HCE Project Python language Distributed Tasks Manager Application, Distributed Crawler Application and client API bindings.  2.0.0-chaika
Hierarchical Cluster Engine Python language binding
ProxyNewTask.py
Go to the documentation of this file.
1 '''
2 @package: dc_db
3 @author scorp
4 @link: http://hierarchical-cluster-engine.com/
5 @copyright: Copyright © 2013-2014 IOIX Ukraine
6 @license: http://hierarchical-cluster-engine.com/license/
7 @since: 0.1
8 '''
9 
10 from dc_db.BaseTask import BaseTask
11 import dc_db.Constants as Constants
12 from dtm.EventObjects import GeneralResponse
13 import app.Utils as Utils # pylint: disable=F0401
14 
15 logger = Utils.MPLogger().getLogger()
16 
17 
18 # # ProxyNewTask Class, implements Proxy New task functionality
19 #
21 
22 
23  # #Class constructor
24  #
25  def __init__(self):
26  super(ProxyNewTask, self).__init__()
27 
28 
29  # #Method lookProxyInDB looks current proxy in th db
30  #
31  # @param proxy - incoming proxy object
32  # @param queryCallback ueryCallback function for queries execution
33  # @return bool value, is current proxy in db or not
34  @staticmethod
35  def lookProxyInDB(proxy, queryCallback):
36  ret = False
37  FETCH_PROXY_SQL = "SELECT * FROM `sites_proxy` WHERE `Site_Id` = '%s' AND `Host` = '%s'"
38  query = FETCH_PROXY_SQL % (proxy.siteId, proxy.host)
39  res = queryCallback(query, Constants.PRIMARY_DB_ID)
40  if hasattr(res, '__iter__') and len(res) > 0 and res[0] is not None:
41  ret = True
42  return ret
43 
44 
45  # #make all necessary actions to add new Proxies into mysql db
46  #
47  # @param urls list of Proxies objects
48  # @param queryCallback function for queries execution
49  # @return generalResponse instance of GeneralResponse object
50  def process(self, proxies, queryCallback):
51  ret = GeneralResponse()
52  for proxy in proxies:
53  if not ProxyNewTask.lookProxyInDB(proxy, queryCallback):
54  if self.insertProxy(proxy, queryCallback):
55  status = 0
56  else:
57  status = 1
58  else:
59  status = 2
60  ret.statuses.append(status)
61  return ret
62 
63 
64  # #Method insertProxy inserts new proxy record in the db
65  #
66  # @param proxy - incoming proxy object
67  # @param queryCallback ueryCallback function for queries execution
68  # @return bool value, is current insertProxy was successful or wasn't
69  def insertProxy(self, proxy, queryCallback):
70  ret = False
71  fields, values = Constants.getFieldsValuesTuple(proxy, Constants.ProxyTableDict)
72  fieldValueString = Constants.createFieldsValuesString(fields, values)
73  if fieldValueString is not None and fieldValueString != "":
74  query = Constants.INSERT_COMMON_TEMPLATE % ("sites_proxy", fieldValueString)
75  queryCallback(query, Constants.PRIMARY_DB_ID)
76  ret = True
77  return ret
GeneralResponse event object, represents general state response for multipurpose usage.
def process(self, proxies, queryCallback)
Definition: ProxyNewTask.py:50
def insertProxy(self, proxy, queryCallback)
Definition: ProxyNewTask.py:69
def lookProxyInDB(proxy, queryCallback)
Definition: ProxyNewTask.py:35