Inheritance diagram for dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper:

Collaboration diagram for dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper:

Public Member Functions
def	__init__ (self, jsonData)

def	setProxyData (self, proxyData)

def	getProxyData (self, proxyName)

def	getData (self)

def	addProxyList (self, proxyList)

def	getProxyList (self, state=None)

def	addFrequency (self, proxyName, incrementSize=DEFAULT_VALUE_INCREMENT_FREQUENCY)

def	addFaults (self, proxyName, incrementSize=DEFAULT_VALUE_INCREMENT_FAULTS)

Public Attributes
	jsonData

Static Public Attributes
string	PROXIES_HOST_NAME = 'host'

string	PROXIES_DOMAINS_NAME = 'domains'

string	PROXIES_PRIORITY_NAME = 'priority'

string	PROXIES_LIMITS_NAME = 'limits'

string	PROXIES_STATE_NAME = 'state'

string	PROXIES_FREQ_NAME = 'freq'

string	PROXIES_FAULTS_NAME = 'faults'

string	PROXIES_FAULTS_MAX_NAME = 'faultsMax'

list	PROXIES_DATA_LIST_NAMES

int	PROXY_STATE_DISABLED = 0

int	PROXY_STATE_ENABLED = 1

dictionary	DEFAULT_VALUE_JSON_DATA = {}

list	DEFAULT_VALUE_PROXIES_DOMAIN = ['*']

int	DEFAULT_VALUE_PROXIES_PRIORITY = 10

	DEFAULT_VALUE_PROXIES_LIMITS = None

int	DEFAULT_VALUE_PROXIES_STATE = PROXY_STATE_ENABLED

int	DEFAULT_VALUE_PROXIES_FREQ = 0

int	DEFAULT_VALUE_PROXIES_FAULTS = 0

int	DEFAULT_VALUE_PROXIES_FAULTS_MAX = 0

int	DEFAULT_VALUE_INCREMENT_FREQUENCY = 1

int	DEFAULT_VALUE_INCREMENT_FAULTS = 1

int	DEFAULT_VALUE_UNLIMITED_FAULTS_MAX = 0

Detailed Description

Definition at line 19 of file ProxyJsonWrapper.py.

Constructor & Destructor Documentation

◆ init()

def dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.__init__	(	self,
		jsonData
	)

Definition at line 53 of file ProxyJsonWrapper.py.

   def __init__(self, jsonData):
     self.jsonData = (jsonData if isinstance(jsonData, dict) else self.DEFAULT_VALUE_JSON_DATA)

Member Function Documentation

◆ addFaults()

def dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.addFaults	(	self,
		proxyName,
		incrementSize = `DEFAULT_VALUE_INCREMENT_FAULTS`
	)

Definition at line 175 of file ProxyJsonWrapper.py.

   def addFaults(self, proxyName, incrementSize=DEFAULT_VALUE_INCREMENT_FAULTS):
     proxyData = self.getProxyData(proxyName)
     if proxyData is not None:
       # increment faults counter
       faults = int(proxyData[self.PROXIES_FAULTS_NAME]) if self.PROXIES_FAULTS_NAME in proxyData \
                else self.DEFAULT_VALUE_PROXIES_FAULTS
       faults += incrementSize
       proxyData[self.PROXIES_FAULTS_NAME] = faults
 
       # get faults max value
       faultsMax = int(proxyData[self.PROXIES_FAULTS_MAX_NAME]) if self.PROXIES_FAULTS_MAX_NAME in proxyData \
                   else self.DEFAULT_VALUE_PROXIES_FAULTS_MAX
 
       # check max allowed faults and disable if necessary
       if faultsMax != self.DEFAULT_VALUE_UNLIMITED_FAULTS_MAX and faults >= faultsMax:
         # disable proxy
         proxyData[self.PROXIES_STATE_NAME] = self.PROXY_STATE_DISABLED
 
       # apply changes
       self.setProxyData(proxyData)
 

Here is the call graph for this function:

◆ addFrequency()

def dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.addFrequency	(	self,
		proxyName,
		incrementSize = `DEFAULT_VALUE_INCREMENT_FREQUENCY`
	)

Definition at line 157 of file ProxyJsonWrapper.py.

   def addFrequency(self, proxyName, incrementSize=DEFAULT_VALUE_INCREMENT_FREQUENCY):
     proxyData = self.getProxyData(proxyName)
     if proxyData is not None:
       # increment frequency counter
       frequency = int(proxyData[self.PROXIES_FREQ_NAME]) if self.PROXIES_FREQ_NAME in proxyData \
                   else self.DEFAULT_VALUE_PROXIES_FREQ
       frequency += int(incrementSize)
       proxyData[self.PROXIES_FREQ_NAME] = frequency
 
       # apply changes
       self.setProxyData(proxyData)
 
 

Here is the call graph for this function:

◆ addProxyList()

def dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.addProxyList	(	self,
		proxyList
	)

Definition at line 116 of file ProxyJsonWrapper.py.

   def addProxyList(self, proxyList):
     if isinstance(proxyList, list):
       for proxy in proxyList:
         if isinstance(proxy, Proxy):
           proxyData = {}
           for name in self.PROXIES_DATA_LIST_NAMES:
             if hasattr(proxy, name):
               proxyData[name] = getattr(proxy, name)
 
           self.setProxyData(proxyData)
 
 

Here is the call graph for this function:

◆ getData()

def dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.getData ( self )

Definition at line 108 of file ProxyJsonWrapper.py.

   def getData(self):
     return self.jsonData

◆ getProxyData()

def dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.getProxyData	(	self,
		proxyName
	)

Definition at line 95 of file ProxyJsonWrapper.py.

   def getProxyData(self, proxyName):
     # variable for result
     ret = None
     if proxyName in self.jsonData:
       ret = self.jsonData[proxyName]
 
     return ret
 
 

Here is the caller graph for this function:

◆ getProxyList()

def dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.getProxyList	(	self,
		state = `None`
	)

Definition at line 132 of file ProxyJsonWrapper.py.

   def getProxyList(self, state=None):
     # variable for result
     proxyList = []
 
     for proxyName in self.jsonData.keys():
       proxyData = self.getProxyData(proxyName)
 
       proxy = Proxy(siteId='0', host='')
       for name in self.PROXIES_DATA_LIST_NAMES:
         if name in proxyData:
           setattr(proxy, name, proxyData[name])
 
       if self.PROXIES_FREQ_NAME in proxyData:
         setattr(proxy, self.PROXIES_FREQ_NAME, int(proxyData[self.PROXIES_FREQ_NAME]))
 
       if proxy.host != "" and (proxy.state == state if state is not None else True):
         proxyList.append(proxy)
 
     return proxyList
 
 

Here is the call graph for this function:

◆ setProxyData()

def dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.setProxyData	(	self,
		proxyData
	)

Definition at line 61 of file ProxyJsonWrapper.py.

   def setProxyData(self, proxyData):
     if isinstance(proxyData, dict):
       if self.PROXIES_HOST_NAME in proxyData.keys() and proxyData[self.PROXIES_HOST_NAME] not in self.jsonData:
 
         proxyData[self.PROXIES_DOMAINS_NAME] = proxyData[self.PROXIES_DOMAINS_NAME] \
           if self.PROXIES_PRIORITY_NAME in proxyData.keys() and proxyData[self.PROXIES_DOMAINS_NAME] is not None \
             and len(proxyData[self.PROXIES_DOMAINS_NAME]) > 0 else self.DEFAULT_VALUE_PROXIES_DOMAIN
 
         proxyData[self.PROXIES_PRIORITY_NAME] = proxyData[self.PROXIES_PRIORITY_NAME] \
           if self.PROXIES_PRIORITY_NAME in proxyData.keys() and proxyData[self.PROXIES_PRIORITY_NAME] is not None \
           and int(proxyData[self.PROXIES_PRIORITY_NAME]) > 0 else self.DEFAULT_VALUE_PROXIES_PRIORITY
 
         proxyData[self.PROXIES_LIMITS_NAME] = proxyData[self.PROXIES_LIMITS_NAME] \
           if self.PROXIES_LIMITS_NAME in proxyData.keys() else self.DEFAULT_VALUE_PROXIES_LIMITS
 
         proxyData[self.PROXIES_STATE_NAME] = proxyData[self.PROXIES_STATE_NAME] \
           if self.PROXIES_STATE_NAME in proxyData.keys() else self.DEFAULT_VALUE_PROXIES_STATE
 
         proxyData[self.PROXIES_FREQ_NAME] = proxyData[self.PROXIES_FREQ_NAME] \
           if self.PROXIES_FREQ_NAME in proxyData.keys() else self.DEFAULT_VALUE_PROXIES_FREQ
 
         proxyData[self.PROXIES_FAULTS_NAME] = proxyData[self.PROXIES_FAULTS_NAME] \
           if self.PROXIES_FAULTS_NAME in proxyData.keys() else self.DEFAULT_VALUE_PROXIES_FAULTS
 
         proxyData[self.PROXIES_FAULTS_MAX_NAME] = proxyData[self.PROXIES_FAULTS_MAX_NAME] \
           if self.PROXIES_FAULTS_MAX_NAME in proxyData.keys() else self.DEFAULT_VALUE_PROXIES_FAULTS_MAX
 
         self.jsonData[proxyData[self.PROXIES_HOST_NAME]] = proxyData
 
 

Here is the caller graph for this function:

Member Data Documentation

◆ DEFAULT_VALUE_INCREMENT_FAULTS

int dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.DEFAULT_VALUE_INCREMENT_FAULTS = 1

static

Definition at line 46 of file ProxyJsonWrapper.py.

◆ DEFAULT_VALUE_INCREMENT_FREQUENCY

int dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.DEFAULT_VALUE_INCREMENT_FREQUENCY = 1

static

Definition at line 45 of file ProxyJsonWrapper.py.

◆ DEFAULT_VALUE_JSON_DATA

dictionary dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.DEFAULT_VALUE_JSON_DATA = {}

static

Definition at line 36 of file ProxyJsonWrapper.py.

◆ DEFAULT_VALUE_PROXIES_DOMAIN

list dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.DEFAULT_VALUE_PROXIES_DOMAIN = ['*']

static

Definition at line 37 of file ProxyJsonWrapper.py.

◆ DEFAULT_VALUE_PROXIES_FAULTS

int dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.DEFAULT_VALUE_PROXIES_FAULTS = 0

static

Definition at line 42 of file ProxyJsonWrapper.py.

◆ DEFAULT_VALUE_PROXIES_FAULTS_MAX

int dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.DEFAULT_VALUE_PROXIES_FAULTS_MAX = 0

static

Definition at line 43 of file ProxyJsonWrapper.py.

◆ DEFAULT_VALUE_PROXIES_FREQ

int dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.DEFAULT_VALUE_PROXIES_FREQ = 0

static

Definition at line 41 of file ProxyJsonWrapper.py.

◆ DEFAULT_VALUE_PROXIES_LIMITS

dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.DEFAULT_VALUE_PROXIES_LIMITS = None

static

Definition at line 39 of file ProxyJsonWrapper.py.

◆ DEFAULT_VALUE_PROXIES_PRIORITY

int dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.DEFAULT_VALUE_PROXIES_PRIORITY = 10

static

Definition at line 38 of file ProxyJsonWrapper.py.

◆ DEFAULT_VALUE_PROXIES_STATE

int dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.DEFAULT_VALUE_PROXIES_STATE = PROXY_STATE_ENABLED

static

Definition at line 40 of file ProxyJsonWrapper.py.

◆ DEFAULT_VALUE_UNLIMITED_FAULTS_MAX

int dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.DEFAULT_VALUE_UNLIMITED_FAULTS_MAX = 0

static

Definition at line 47 of file ProxyJsonWrapper.py.

◆ jsonData

dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.jsonData

Definition at line 54 of file ProxyJsonWrapper.py.

◆ PROXIES_DATA_LIST_NAMES

list dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.PROXIES_DATA_LIST_NAMES

static

Initial value:

= [PROXIES_HOST_NAME, PROXIES_DOMAINS_NAME, PROXIES_PRIORITY_NAME, PROXIES_LIMITS_NAME, \

PROXIES_STATE_NAME, PROXIES_FREQ_NAME, PROXIES_FAULTS_NAME, PROXIES_FAULTS_MAX_NAME]

Definition at line 30 of file ProxyJsonWrapper.py.

◆ PROXIES_DOMAINS_NAME

string dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.PROXIES_DOMAINS_NAME = 'domains'

static

Definition at line 22 of file ProxyJsonWrapper.py.

◆ PROXIES_FAULTS_MAX_NAME

string dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.PROXIES_FAULTS_MAX_NAME = 'faultsMax'

static

Definition at line 28 of file ProxyJsonWrapper.py.

◆ PROXIES_FAULTS_NAME

string dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.PROXIES_FAULTS_NAME = 'faults'

static

Definition at line 27 of file ProxyJsonWrapper.py.

◆ PROXIES_FREQ_NAME

string dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.PROXIES_FREQ_NAME = 'freq'

static

Definition at line 26 of file ProxyJsonWrapper.py.

◆ PROXIES_HOST_NAME

string dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.PROXIES_HOST_NAME = 'host'

static

Definition at line 21 of file ProxyJsonWrapper.py.

◆ PROXIES_LIMITS_NAME

string dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.PROXIES_LIMITS_NAME = 'limits'

static

Definition at line 24 of file ProxyJsonWrapper.py.

◆ PROXIES_PRIORITY_NAME

string dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.PROXIES_PRIORITY_NAME = 'priority'

static

Definition at line 23 of file ProxyJsonWrapper.py.

◆ PROXIES_STATE_NAME

string dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.PROXIES_STATE_NAME = 'state'

static

Definition at line 25 of file ProxyJsonWrapper.py.

◆ PROXY_STATE_DISABLED

int dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.PROXY_STATE_DISABLED = 0

static

Definition at line 33 of file ProxyJsonWrapper.py.

◆ PROXY_STATE_ENABLED

int dc_crawler.ProxyJsonWrapper.ProxyJsonWrapper.PROXY_STATE_ENABLED = 1

static

Definition at line 34 of file ProxyJsonWrapper.py.

The documentation for this class was generated from the following file:

sources/hce/dc_crawler/ProxyJsonWrapper.py

Public Member Functions

Public Attributes

Static Public Attributes

Detailed Description

Constructor & Destructor Documentation

◆ __init__()

Member Function Documentation

◆ addFaults()

◆ addFrequency()

◆ addProxyList()

◆ getData()

◆ getProxyData()

◆ getProxyList()

◆ setProxyData()

Member Data Documentation

◆ DEFAULT_VALUE_INCREMENT_FAULTS

◆ DEFAULT_VALUE_INCREMENT_FREQUENCY

◆ DEFAULT_VALUE_JSON_DATA

◆ DEFAULT_VALUE_PROXIES_DOMAIN

◆ DEFAULT_VALUE_PROXIES_FAULTS

◆ DEFAULT_VALUE_PROXIES_FAULTS_MAX

◆ DEFAULT_VALUE_PROXIES_FREQ

◆ DEFAULT_VALUE_PROXIES_LIMITS

◆ DEFAULT_VALUE_PROXIES_PRIORITY

◆ DEFAULT_VALUE_PROXIES_STATE

◆ DEFAULT_VALUE_UNLIMITED_FAULTS_MAX

◆ jsonData

◆ PROXIES_DATA_LIST_NAMES

◆ PROXIES_DOMAINS_NAME

◆ PROXIES_FAULTS_MAX_NAME

◆ PROXIES_FAULTS_NAME

◆ PROXIES_FREQ_NAME

◆ PROXIES_HOST_NAME

◆ PROXIES_LIMITS_NAME

◆ PROXIES_PRIORITY_NAME

◆ PROXIES_STATE_NAME

◆ PROXY_STATE_DISABLED

◆ PROXY_STATE_ENABLED

◆ init()