HCE Project Python language Distributed Tasks Manager Application, Distributed Crawler Application and client API bindings.  2.0.0-chaika
Hierarchical Cluster Engine Python language binding
ProxyStatusTask.py
Go to the documentation of this file.
1 '''
2 @package: dc
3 @author scorp
4 @link: http://hierarchical-cluster-engine.com/
5 @copyright: Copyright © 2013-2014 IOIX Ukraine
6 @license: http://hierarchical-cluster-engine.com/license/
7 @since: 0.1
8 '''
9 
10 import copy
11 import collections
12 
13 from dc_db.BaseTask import BaseTask
14 import dc_db.Constants as Constants
15 import dc.EventObjects
16 import app.SQLCriterions
17 import app.Utils as Utils # pylint: disable=F0401
18 
19 logger = Utils.MPLogger().getLogger()
20 
21 
22 # # ProxyStatusTask Class, implements Proxy Status task functionality
23 #
25 
26  # #Class constructor
27  #
28  def __init__(self):
29  super(ProxyStatusTask, self).__init__()
30 
31 
32  # #make all necessary actions to fetch Proxies from mysql db
33  #
34  # @param urls list of Proxies objects
35  # @param queryCallback function for queries execution
36  # @return list of fetching proxies
37  def process(self, proxyStatuses, queryCallback):
38  proxies = []
39  for proxyStatus in proxyStatuses:
40  localProxyStatuses = []
41  if proxyStatus.siteId is None or proxyStatus.host is None and proxyStatus.criterions is not None:
42  result = ProxyStatusTask.execCriterion(proxyStatus, queryCallback, proxyStatus.siteId)
43  for elem in result:
44  localProxy = copy.deepcopy(proxyStatus)
45  # if localProxy.siteId is None:
46  localProxy.siteId = elem["Site_Id"]
47  # if localProxy.host is None:
48  localProxy.host = elem["Host"]
49  localProxyStatuses.append(localProxy)
50  else:
51  localProxyStatuses.append(proxyStatus)
52  logger.debug(">>> len(localProxyStatuses) = " + str(len(localProxyStatuses)))
53  for localProxy in localProxyStatuses:
54  proxy = self.fetchProxy(localProxy, queryCallback)
55  if proxy is not None:
56  proxies.append(proxy)
57  return proxies
58 
59 
60  # #execCriterion fetches proxy DB records by criterions
61  #
62  # @param proxyStatus incoming proxyStatus element
63  # @param queryCallback function for queries execution
64  # @param siteId incoming siteId, may be None
65  # @return list of fetching proxies
66  @staticmethod
67  def execCriterion(proxyStatus, queryCallback, siteId, additionWhere=None):
68  ret = []
69  PROXY_SELECT_TEMPLATE = "SELECT * FROM `sites_proxy`"
70  criterionString = app.SQLCriterions.generateCriterionSQL(proxyStatus.criterions, additionWhere, siteId)
71  if criterionString is not None and criterionString != "":
72  query = PROXY_SELECT_TEMPLATE + criterionString
73  res = queryCallback(query, Constants.PRIMARY_DB_ID, Constants.EXEC_NAME)
74  if isinstance(res, collections.Iterable):
75  ret = res
76 
77  return ret
78 
79 
80  # #fetchProxy fetches one proxy element
81  #
82  # @param queryCallback function for queries execution
83  # @return Proxy object or None
84  def fetchProxy(self, localProxy, queryCallback):
85  ret = None
86  if localProxy.siteId is not None and localProxy.host is not None:
87  try:
88  PROXY_FETCH_TEMPLATE = "SELECT * FROM `sites_proxy` WHERE `Site_Id` = '%s' AND `Host` = '%s'"
89  query = PROXY_FETCH_TEMPLATE % (localProxy.siteId, localProxy.host)
90  res = queryCallback(query, Constants.PRIMARY_DB_ID, Constants.EXEC_NAME)
91 
92  if isinstance(res, collections.Iterable) and len(res) > 0:
93  ret = dc.EventObjects.Proxy(res[0]["Site_Id"], res[0]["Host"])
94  ret.id = res[0]["Id"]
95  ret.domains = res[0]["Domains"]
96  ret.priority = res[0]["Priority"]
97  ret.state = res[0]["State"]
98  ret.countryCode = res[0]["CountryCode"]
99  ret.countryName = res[0]["CountryName"]
100  ret.regionCode = res[0]["RegionCode"]
101  ret.regionName = res[0]["RegionName"]
102  ret.cityName = res[0]["CityName"]
103  ret.zipCode = res[0]["ZipCode"]
104  ret.timeZone = res[0]["TimeZone"]
105  ret.latitude = res[0]["Latitude"]
106  ret.longitude = res[0]["Longitude"]
107  ret.metroCode = res[0]["MetroCode"]
108  ret.faults = res[0]["Faults"]
109  ret.faultsMax = res[0]["FaultsMax"]
110  ret.categoryId = res[0]["Category_Id"]
111  ret.limits = res[0]["Limits"]
112  ret.description = res[0]["Description"]
113  ret.cDate = res[0]["CDate"]
114  ret.uDate = res[0]["UDate"]
115  except Exception as excp:
116  logger.debug(">>> ProxyStatusTask.fetchProxy some exception = " + str(excp))
117  return ret
def fetchProxy(self, localProxy, queryCallback)
def process(self, proxyStatuses, queryCallback)
def generateCriterionSQL(criterions, additionWhere=None, siteId=None)
def execCriterion(proxyStatus, queryCallback, siteId, additionWhere=None)