HCE Project Python language Distributed Tasks Manager Application, Distributed Crawler Application and client API bindings.  2.0.0-chaika
Hierarchical Cluster Engine Python language binding
Public Member Functions | Public Attributes | Static Public Attributes | List of all members
dc_db.TasksManager.TasksManager Class Reference
Inheritance diagram for dc_db.TasksManager.TasksManager:
Inheritance graph
[legend]
Collaboration diagram for dc_db.TasksManager.TasksManager:
Collaboration graph
[legend]

Public Member Functions

def __init__ (self, configParser, dbTaskMode=dc_event.Batch.DB_MODE_RW)
 
def __del__ (self)
 
def process (self, drceSyncTasksCover)
 
def executeQuery (self, query, dbConnectionName, executeType=Constants.EXEC_INDEX, SQLErrorClear=False)
 
def commonQueryErrorHandler (self, err, dbConnection, SQLErrorClear=False)
 
def checkQueryInReadOnly (self, query)
 
def executeQueryByIndex (self, query, dbConnectionName, SQLErrorClear=False)
 
def executeQueryByName (self, query, dbConnectionName, SQLErrorClear=False)
 
def checkInputData (self, drceSyncTasksCover)
 
def createDRCESyncTasksCover (self, eventType, eventObject)
 
def backDBResolve (self, dbName)
 

Public Attributes

 dbTaskMode
 
 mutexLockTTL
 
 contentStorageType
 
 dbConnections
 
 dbNames
 
 dcSiteTemplate
 
 dcStatTemplate
 
 dcLogTemplate
 
 dcAttrTemplate
 
 keyValueStorageDir
 
 rawDataDir
 
 keyValueDefaultFile
 
 dcContentTemplate
 
 SQLErrorCode
 
 SQLErrorString
 
 dBDataTask
 

Static Public Attributes

string DB_HOST = "db_host"
 
string DB_PORT = "db_port"
 
string DB_USER = "db_user"
 
string DB_PWD = "db_pwd"
 
string PRIMARY_DB = "primary_db"
 
string SECONDARY_DB = "secondary_db"
 
string THIRD_DB = "third_db"
 
string FOURTH_DB = "fourth_db"
 
string FIFTH_DB = "fifth_db"
 
string STAT_DB = "stat_db"
 
string LOG_DB = "log_db"
 
string ATT_DB = "att_db"
 
string STAT_DOMAINS_DB = "stat_domains_db"
 
string MUTEX_LOCK_TTL = "mutexLockTTL"
 
string SITE_TEMPLATE = "dc_site_template"
 
string KEY_VALUE_STORAGE_DIR = "key_value_storage_dir"
 
string KEY_VALUE_DEFAULT_FILE = "key_value_default_file"
 
string RAW_DATA_DIR = "raw_data_dir"
 
string CONTENT_STORAGE_TYPE = "content_storage_type"
 
string CONTENT_TEMPLATE = "dc_content_template"
 
string STAT_TEMPLATE = "dc_statistics_freq_template"
 
string LOG_TEMPLATE = "dc_statistics_log_template"
 
string ATTR_TEMPLATE = "dc_attribute_template"
 
string SITE_PROP_NAME_FREQ = "stats_freq_enabled"
 
string SITE_PROP_NAME_LOG = "stats_log_enabled"
 
list SQL_OPERATION_LIST = ["update", "insert", "delete", "drop"]
 

Detailed Description

Definition at line 90 of file TasksManager.py.

Constructor & Destructor Documentation

◆ __init__()

def dc_db.TasksManager.TasksManager.__init__ (   self,
  configParser,
  dbTaskMode = dc_event.Batch.DB_MODE_RW 
)

Definition at line 123 of file TasksManager.py.

123  def __init__(self, configParser, dbTaskMode=dc_event.Batch.DB_MODE_RW):
124  self.dbTaskMode = dbTaskMode
125  className = self.__class__.__name__
126  # Get configuration settings
127  dbHost = configParser.get(className, self.DB_HOST)
128  dbPort = int(configParser.get(className, self.DB_PORT))
129  dbUser = configParser.get(className, self.DB_USER)
130  dbPWD = configParser.get(className, self.DB_PWD)
131  dbPrimary = configParser.get(className, self.PRIMARY_DB)
132  dbSecondary = configParser.get(className, self.SECONDARY_DB)
133  dbThird = configParser.get(className, self.THIRD_DB)
134  dbFourth = configParser.get(className, self.FOURTH_DB)
135  dbFifth = configParser.get(className, self.FIFTH_DB)
136  dbStat = configParser.get(className, self.STAT_DB)
137  dbLog = configParser.get(className, self.LOG_DB)
138  dbAtt = configParser.get(className, self.ATT_DB)
139  dbStatDomains = configParser.get(className, self.STAT_DOMAINS_DB)
140 
141  try:
142  self.mutexLockTTL = int(configParser.get(className, self.MUTEX_LOCK_TTL))
143  except Exception:
144  self.mutexLockTTL = Constants.DEFAULT_LOCK_TTL
145  self.contentStorageType = int(configParser.get(className, self.CONTENT_STORAGE_TYPE))
146  self.dbConnections = {}
147  self.dbNames = {}
148 
149  self.dcSiteTemplate = configParser.get(className, self.SITE_TEMPLATE)
150  self.dcStatTemplate = configParser.get(className, self.STAT_TEMPLATE)
151  self.dcLogTemplate = configParser.get(className, self.LOG_TEMPLATE)
152  self.dcAttrTemplate = configParser.get(className, self.ATTR_TEMPLATE)
153  self.keyValueStorageDir = configParser.get(className, self.KEY_VALUE_STORAGE_DIR)
154  self.rawDataDir = configParser.get(className, self.RAW_DATA_DIR)
155  self.keyValueDefaultFile = configParser.get(className, self.KEY_VALUE_DEFAULT_FILE)
156  self.dcContentTemplate = configParser.get(className, self.CONTENT_TEMPLATE)
157  self.dbConnections[Constants.PRIMARY_DB_ID] = mdb.connect(dbHost, dbUser, dbPWD, dbPrimary, dbPort, use_unicode=True, charset='utf8')
158  self.dbNames[dbPrimary] = Constants.PRIMARY_DB_ID
159  self.dbConnections[Constants.SECONDARY_DB_ID] = mdb.connect(dbHost, dbUser, dbPWD, dbSecondary, dbPort, use_unicode=True, charset='utf8')
160  self.dbNames[dbSecondary] = Constants.SECONDARY_DB_ID
161  if dbThird is not None and dbThird != '':
162  self.dbConnections[Constants.THIRD_DB_ID] = mdb.connect(dbHost, dbUser, dbPWD, dbThird, dbPort, use_unicode=True, charset='utf8')
163  self.dbNames[dbThird] = Constants.THIRD_DB_ID
164  else:
165  logger.debug(">>> THIRD_DB_ID empty")
166  if dbFourth is not None and dbFourth != '':
167  self.dbConnections[Constants.FOURTH_DB_ID] = mdb.connect(dbHost, dbUser, dbPWD, dbFourth, dbPort, use_unicode=True, charset='utf8')
168  self.dbNames[dbFourth] = Constants.FOURTH_DB_ID
169  else:
170  logger.debug(">>> FOURTH_DB_ID empty")
171  if dbFifth is not None and dbFifth != '':
172  self.dbConnections[Constants.FIFTH_DB_ID] = mdb.connect(dbHost, dbUser, dbPWD, dbFifth, dbPort, use_unicode=True, charset='utf8')
173  self.dbNames[dbFifth] = Constants.FIFTH_DB_ID
174  else:
175  logger.debug(">>> FIFTH_DB_ID empty")
176  self.dbConnections[Constants.STAT_DB_ID] = mdb.connect(dbHost, dbUser, dbPWD, dbStat, dbPort, use_unicode=True, charset='utf8')
177  self.dbNames[dbStat] = Constants.STAT_DB_ID
178  self.dbConnections[Constants.LOG_DB_ID] = mdb.connect(dbHost, dbUser, dbPWD, dbLog, dbPort, use_unicode=True, charset='utf8')
179  self.dbNames[dbLog] = Constants.LOG_DB_ID
180  self.dbConnections[Constants.LOG_DB_ID] = mdb.connect(dbHost, dbUser, dbPWD, dbLog, dbPort, use_unicode=True, charset='utf8')
181  self.dbNames[dbLog] = Constants.LOG_DB_ID
182  self.dbConnections[Constants.ATT_DB_ID] = mdb.connect(dbHost, dbUser, dbPWD, dbAtt, dbPort, use_unicode=True, charset='utf8')
183  self.dbNames[dbAtt] = Constants.ATT_DB_ID
184  self.dbConnections[Constants.STAT_DOMAINS_DB_ID] = mdb.connect(dbHost, dbUser, dbPWD, dbStatDomains, dbPort, use_unicode=True, charset='utf8')
185  self.dbNames[dbStatDomains] = Constants.STAT_DOMAINS_DB_ID
186 
187  self.SQLErrorCode = Constants.EXIT_CODE_OK
188  self.SQLErrorString = ""
189  self.dBDataTask = DBDataTask(self.contentStorageType, self.keyValueDefaultFile, self.dcContentTemplate,
190  self.keyValueStorageDir, self.rawDataDir)
191  try:
192  tmpVal = int(configParser.get(className, self.SITE_PROP_NAME_FREQ))
193  StatisticLogManager.GLOBAL_FREQ_ENABLED = bool(tmpVal)
194  except ConfigParser.NoOptionError:
195  pass
196  try:
197  tmpVal = int(configParser.get(className, self.SITE_PROP_NAME_LOG))
198  StatisticLogManager.GLOBAL_LOG_ENABLED = bool(tmpVal)
199  except ConfigParser.NoOptionError:
200  pass
201 
202 
transport.UIDGenerator.UIDGenerator.__init__
def __init__(self)
constructor
Definition: UIDGenerator.py:19

◆ __del__()

def dc_db.TasksManager.TasksManager.__del__ (   self)

Definition at line 205 of file TasksManager.py.

205  def __del__(self):
206  for dbConnect in self.dbConnections.values():
207  if dbConnect is not None:
208  dbConnect.close()
209 
210 

Member Function Documentation

◆ backDBResolve()

def dc_db.TasksManager.TasksManager.backDBResolve (   self,
  dbName 
)

Definition at line 517 of file TasksManager.py.

517  def backDBResolve(self, dbName):
518  ret = None
519  if dbName in self.dbNames:
520  ret = self.dbNames[dbName]
521  return ret
522 
523 
Here is the caller graph for this function:

◆ checkInputData()

def dc_db.TasksManager.TasksManager.checkInputData (   self,
  drceSyncTasksCover 
)

Definition at line 494 of file TasksManager.py.

494  def checkInputData(self, drceSyncTasksCover):
495  for inputType in eventCheckDict:
496  if drceSyncTasksCover.eventType is inputType:
497  if isinstance(eventCheckDict[inputType], tuple):
498  for localType in eventCheckDict[inputType]:
499  if isinstance(drceSyncTasksCover.eventObject, localType):
500  return True
501  elif isinstance(drceSyncTasksCover.eventObject, eventCheckDict[inputType]):
502  return True
503  raise Exception("Incorrect types of input data")
504 
505 
Here is the caller graph for this function:

◆ checkQueryInReadOnly()

def dc_db.TasksManager.TasksManager.checkQueryInReadOnly (   self,
  query 
)

Definition at line 428 of file TasksManager.py.

428  def checkQueryInReadOnly(self, query):
429  ret = True
430  if self.dbTaskMode & dc_event.Batch.DB_MODE_W == 0:
431  low_query = query.lower()
432  for elem in self.SQL_OPERATION_LIST:
433  if low_query.startswith(elem):
434  logger.debug(">>> QUERY = " + query)
435  logger.debug(">>> NOT SUPPORT in read only mode")
436  ret = False
437  break
438  return ret
439 
440 
Here is the caller graph for this function:

◆ commonQueryErrorHandler()

def dc_db.TasksManager.TasksManager.commonQueryErrorHandler (   self,
  err,
  dbConnection,
  SQLErrorClear = False 
)

Definition at line 414 of file TasksManager.py.

414  def commonQueryErrorHandler(self, err, dbConnection, SQLErrorClear=False):
415  dbConnection.rollback()
416  self.SQLErrorString = "%s %s" % (err.args[0], err.args[1])
417  if not SQLErrorClear:
418  ExceptionLog.handler(logger, err, self.SQLErrorString)
419  self.SQLErrorCode = Constants.EXIT_CODE_MYSQL_ERROR
420  else:
421  logger.debug(self.SQLErrorString)
422 
423 
Here is the caller graph for this function:

◆ createDRCESyncTasksCover()

def dc_db.TasksManager.TasksManager.createDRCESyncTasksCover (   self,
  eventType,
  eventObject 
)

Definition at line 510 of file TasksManager.py.

510  def createDRCESyncTasksCover(self, eventType, eventObject):
511  return DRCESyncTasksCover(eventType, eventObject)
512 
513 
Here is the caller graph for this function:

◆ executeQuery()

def dc_db.TasksManager.TasksManager.executeQuery (   self,
  query,
  dbConnectionName,
  executeType = Constants.EXEC_INDEX,
  SQLErrorClear = False 
)

Definition at line 400 of file TasksManager.py.

400  def executeQuery(self, query, dbConnectionName, executeType=Constants.EXEC_INDEX, SQLErrorClear=False):
401  ret = None
402  if executeType == Constants.EXEC_INDEX:
403  ret = self.executeQueryByIndex(query, dbConnectionName, SQLErrorClear)
404  elif executeType == Constants.EXEC_NAME:
405  ret = self.executeQueryByName(query, dbConnectionName, SQLErrorClear)
406  return ret
407 
408 
generate_sites_jsons.executeQuery
def executeQuery(db_connector, query)
Definition: generate_sites_jsons.py:101
Here is the call graph for this function:
Here is the caller graph for this function:

◆ executeQueryByIndex()

def dc_db.TasksManager.TasksManager.executeQueryByIndex (   self,
  query,
  dbConnectionName,
  SQLErrorClear = False 
)

Definition at line 446 of file TasksManager.py.

446  def executeQueryByIndex(self, query, dbConnectionName, SQLErrorClear=False):
447  ret = None
448  if self.checkQueryInReadOnly(query):
449  dbConnection = None
450  if dbConnectionName in self.dbConnections:
451  dbConnection = self.dbConnections[dbConnectionName]
452  if dbConnection:
453  try:
454  with closing(dbConnection.cursor()) as cursor:
455  # logger.debug(query)
456  cursor.execute(query)
457  dbConnection.commit()
458  ret = cursor.fetchall()
459  except mdb.Error as err: # pylint: disable=E1101
460  self.commonQueryErrorHandler(err, dbConnection, SQLErrorClear)
461  else:
462  logger.debug(">>> dbConnection Not Found = " + str(dbConnectionName))
463  return ret
464 
465 
Here is the call graph for this function:
Here is the caller graph for this function:

◆ executeQueryByName()

def dc_db.TasksManager.TasksManager.executeQueryByName (   self,
  query,
  dbConnectionName,
  SQLErrorClear = False 
)

Definition at line 471 of file TasksManager.py.

471  def executeQueryByName(self, query, dbConnectionName, SQLErrorClear=False):
472  ret = None
473  if self.checkQueryInReadOnly(query):
474  dbConnection = None
475  if dbConnectionName in self.dbConnections:
476  dbConnection = self.dbConnections[dbConnectionName]
477  if dbConnection:
478  try:
479  with closing(dbConnection.cursor(MySQLdb.cursors.DictCursor)) as cursor:
480  # logger.debug(query)
481  cursor.execute(query)
482  dbConnection.commit()
483  ret = cursor.fetchall()
484  except mdb.Error as err: # pylint: disable=E1101
485  self.commonQueryErrorHandler(err, dbConnection, SQLErrorClear)
486  else:
487  logger.debug(">>> dbConnection Not Found = " + str(dbConnectionName))
488  return ret
489 
490 
Here is the call graph for this function:
Here is the caller graph for this function:

◆ process()

def dc_db.TasksManager.TasksManager.process (   self,
  drceSyncTasksCover 
)

Definition at line 215 of file TasksManager.py.

215  def process(self, drceSyncTasksCover):
216  logger.info("Request eventType: %s, eventObject type: %s",
217  str(drceSyncTasksCover.eventType), str(type(drceSyncTasksCover.eventObject)))
218  if isinstance(drceSyncTasksCover.eventObject, list):
219  if len(drceSyncTasksCover.eventObject) > 0:
220  itemType = str(drceSyncTasksCover.eventObject[0])
221  else:
222  itemType = ""
223  logger.info("Request eventObject items: %s, item type: %s", str(len(drceSyncTasksCover.eventObject)), itemType)
224  # if isinstance(drceSyncTasksCover.eventObject, dc_event.URLFetch):
225  # logger.debug('Event object: %s', varDump(drceSyncTasksCover.eventObject))
226 
227  self.checkInputData(drceSyncTasksCover)
228  responseObject = None
229  responseEvent = None
230  if drceSyncTasksCover.eventType == EVENT_TYPES.SITE_DELETE:
231  siteDeleteTask = SiteDeleteTask(self.keyValueStorageDir, self.rawDataDir, self.dBDataTask)
232  responseObject = siteDeleteTask.process(drceSyncTasksCover.eventObject, self.executeQuery)
233  responseEvent = EVENT_TYPES.SITE_DELETE_RESPONSE
234  self.SQLErrorCode = Constants.EXIT_CODE_OK
235 
236  elif drceSyncTasksCover.eventType == EVENT_TYPES.SITE_NEW:
237  siteTask = SiteTask(self.dcSiteTemplate, self.keyValueDefaultFile, self.keyValueStorageDir, self.dBDataTask,
238  self.dcStatTemplate, self.dcLogTemplate, self.dcAttrTemplate, self)
239  responseObject = siteTask.process(drceSyncTasksCover.eventObject, self.executeQuery)
240  responseEvent = EVENT_TYPES.SITE_NEW_RESPONSE
241 
242  elif drceSyncTasksCover.eventType == EVENT_TYPES.SITE_UPDATE:
243  siteUpdateTask = SiteUpdateTask(self.dcSiteTemplate, self.keyValueDefaultFile, self.keyValueStorageDir,
244  self.rawDataDir, self.dBDataTask, self.dcStatTemplate, self.dcLogTemplate,
245  self.dcAttrTemplate)
246  responseObject = siteUpdateTask.process(drceSyncTasksCover.eventObject, self.executeQuery)
247  responseEvent = EVENT_TYPES.SITE_UPDATE_RESPONSE
248 
249  elif drceSyncTasksCover.eventType == EVENT_TYPES.SITE_CLEANUP:
250  siteCleanUpTask = SiteCleanUpTask(self.keyValueStorageDir, self.rawDataDir, self.dBDataTask)
251  responseObject = siteCleanUpTask.process(drceSyncTasksCover.eventObject, self.executeQuery)
252  responseEvent = EVENT_TYPES.SITE_CLEANUP_RESPONSE
253  self.SQLErrorCode = Constants.EXIT_CODE_OK
254 
255  elif drceSyncTasksCover.eventType == EVENT_TYPES.SITE_STATUS:
256  siteStatusTask = SiteStatusTask()
257  responseObject = siteStatusTask.process(drceSyncTasksCover.eventObject, self.executeQuery) # pylint:disable=R0204
258  responseEvent = EVENT_TYPES.SITE_STATUS_RESPONSE
259 
260  # find site by root url and return list of siteId's with root urls
261  elif drceSyncTasksCover.eventType == EVENT_TYPES.SITE_FIND:
262  siteFindTask = SiteFindTask(self.dcSiteTemplate, self.keyValueDefaultFile, self.keyValueStorageDir,
263  self.dBDataTask, self.dcStatTemplate, self.dcLogTemplate, self.dcAttrTemplate)
264  responseObject = siteFindTask.process(drceSyncTasksCover.eventObject, self.executeQuery)
265  responseEvent = EVENT_TYPES.SITE_FIND_RESPONSE
266 
267  elif drceSyncTasksCover.eventType == EVENT_TYPES.URL_NEW:
268  urlNewTask = URLNewTask(self.keyValueStorageDir, self.rawDataDir, self.dBDataTask)
269  urlNewTask.siteTask = SiteTask(self.dcSiteTemplate, self.keyValueDefaultFile, self.keyValueStorageDir,
270  self.dBDataTask, self.dcStatTemplate, self.dcLogTemplate, self.dcAttrTemplate)
271  responseObject = urlNewTask.process(drceSyncTasksCover.eventObject, self.executeQuery)
272  responseEvent = EVENT_TYPES.URL_NEW_RESPONSE
273 
274  elif drceSyncTasksCover.eventType == EVENT_TYPES.URL_STATUS:
275  urlStatusTask = URLStatusTask()
276  responseObject = urlStatusTask.process(drceSyncTasksCover.eventObject, self.executeQuery)
277  responseEvent = EVENT_TYPES.URL_STATUS_RESPONSE
278 
279  elif drceSyncTasksCover.eventType == EVENT_TYPES.URL_UPDATE:
280  urlUpdateTask = URLUpdateTask(self.keyValueStorageDir, self.rawDataDir, self.dBDataTask)
281  responseObject = urlUpdateTask.process(drceSyncTasksCover.eventObject, self.executeQuery)
282  responseEvent = EVENT_TYPES.URL_UPDATE_RESPONSE
283 
284  elif drceSyncTasksCover.eventType == EVENT_TYPES.URL_FETCH:
285  urlFetchTask = URLFetchTask(self.keyValueStorageDir, self.rawDataDir, self.dBDataTask, self.dcSiteTemplate,
286  self.keyValueDefaultFile, self.dcStatTemplate, self.dcLogTemplate, self.mutexLockTTL)
287  responseObject = urlFetchTask.process(drceSyncTasksCover.eventObject, self.executeQuery)
288  responseEvent = EVENT_TYPES.URL_FETCH_RESPONSE
289 
290  elif drceSyncTasksCover.eventType == EVENT_TYPES.URL_CONTENT:
291  urlContentTask = URLContentTask(self.keyValueStorageDir, self.rawDataDir, self.dBDataTask, self.dcSiteTemplate,
292  self.keyValueDefaultFile, self.dcStatTemplate, self.dcLogTemplate)
293  responseObject = urlContentTask.process(drceSyncTasksCover.eventObject, self.executeQuery)
294  responseEvent = EVENT_TYPES.URL_CONTENT_RESPONSE
295 
296  elif drceSyncTasksCover.eventType == EVENT_TYPES.URL_DELETE:
297  urlDeleteTask = URLDeleteTask(self.keyValueStorageDir, self.rawDataDir, self.dBDataTask)
298  responseObject = urlDeleteTask.process(drceSyncTasksCover.eventObject, self.executeQuery)
299  responseEvent = EVENT_TYPES.URL_DELETE_RESPONSE
300 
301  elif drceSyncTasksCover.eventType == EVENT_TYPES.URL_CLEANUP:
302  urlCleanUpTask = URLCleanUpTask(self.keyValueStorageDir, self.rawDataDir, self.dBDataTask)
303  responseObject = urlCleanUpTask.process(drceSyncTasksCover.eventObject, self.executeQuery)
304  responseEvent = EVENT_TYPES.URL_CLEANUP_RESPONSE
305 
306  elif drceSyncTasksCover.eventType == EVENT_TYPES.SQL_CUSTOM:
307  sqlCustomTask = SQLCustomTask()
308  responseObject = sqlCustomTask.process(drceSyncTasksCover.eventObject, self.executeQuery, self.backDBResolve)
309  responseEvent = EVENT_TYPES.SQL_CUSTOM_RESPONSE
310 
311  elif drceSyncTasksCover.eventType == EVENT_TYPES.URL_PURGE:
312  urlPurgeTask = URLPurgeTask(self.keyValueStorageDir, self.rawDataDir, self.dBDataTask)
313  responseObject = urlPurgeTask.process(drceSyncTasksCover.eventObject, self.executeQuery)
314  responseEvent = EVENT_TYPES.URL_PURGE_RESPONSE
315 
316  elif drceSyncTasksCover.eventType == EVENT_TYPES.FIELD_RECALCULATE:
317  fieldRecalculatorTask = FieldRecalculatorTask()
318  responseObject = fieldRecalculatorTask.process(drceSyncTasksCover.eventObject, self.executeQuery)
319  responseEvent = EVENT_TYPES.FIELD_RECALCULATE_RESPONSE
320 
321  elif drceSyncTasksCover.eventType == EVENT_TYPES.URL_VERIFY:
322  urlVarifyTask = URLVerifyTask()
323  responseObject = urlVarifyTask.process(drceSyncTasksCover.eventObject, self.executeQuery, self.backDBResolve)
324  responseEvent = EVENT_TYPES.URL_VERIFY_RESPONSE
325 
326  elif drceSyncTasksCover.eventType == EVENT_TYPES.URL_AGE:
327  urlAgeTask = URLAgeTask(self.keyValueStorageDir, self.rawDataDir, self.backDBResolve)
328  responseObject = urlAgeTask.process(drceSyncTasksCover.eventObject, self.executeQuery)
329  responseEvent = EVENT_TYPES.URL_AGE_RESPONSE
330 
331  elif drceSyncTasksCover.eventType == EVENT_TYPES.URL_PUT:
332  urlPutTask = URLPutTask(self.keyValueStorageDir, self.rawDataDir, self.dBDataTask)
333  responseObject = urlPutTask.process(drceSyncTasksCover.eventObject, self.executeQuery)
334  responseEvent = EVENT_TYPES.URL_PUT_RESPONSE
335 
336  elif drceSyncTasksCover.eventType == EVENT_TYPES.URL_HISTORY:
337  urlHistoryTask = URLHistoryTask(self.keyValueStorageDir, self.rawDataDir, self.dBDataTask)
338  responseObject = urlHistoryTask.process(drceSyncTasksCover.eventObject, self.executeQuery)
339  responseEvent = EVENT_TYPES.URL_HISTORY_RESPONSE
340 
341  elif drceSyncTasksCover.eventType == EVENT_TYPES.URL_STATS:
342  urlStatsTask = URLStatsTask(self.keyValueStorageDir, self.rawDataDir, self.dBDataTask)
343  responseObject = urlStatsTask.process(drceSyncTasksCover.eventObject, self.executeQuery)
344  responseEvent = EVENT_TYPES.URL_STATS_RESPONSE
345 
346  elif drceSyncTasksCover.eventType == EVENT_TYPES.PROXY_NEW:
347  proxyNewTask = ProxyNewTask()
348  responseObject = proxyNewTask.process(drceSyncTasksCover.eventObject, self.executeQuery)
349  responseEvent = EVENT_TYPES.PROXY_NEW_RESPONSE
350 
351  elif drceSyncTasksCover.eventType == EVENT_TYPES.PROXY_UPDATE:
352  proxyUpdateTask = ProxyUpdateTask()
353  responseObject = proxyUpdateTask.process(drceSyncTasksCover.eventObject, self.executeQuery)
354  responseEvent = EVENT_TYPES.PROXY_UPDATE_RESPONSE
355 
356  elif drceSyncTasksCover.eventType == EVENT_TYPES.PROXY_DELETE:
357  proxyDeleteTask = ProxyDeleteTask()
358  responseObject = proxyDeleteTask.process(drceSyncTasksCover.eventObject, self.executeQuery)
359  responseEvent = EVENT_TYPES.PROXY_DELETE_RESPONSE
360 
361  elif drceSyncTasksCover.eventType == EVENT_TYPES.PROXY_STATUS:
362  proxyStatusTask = ProxyStatusTask()
363  responseObject = proxyStatusTask.process(drceSyncTasksCover.eventObject, self.executeQuery)
364  responseEvent = EVENT_TYPES.PROXY_STATUS_RESPONSE
365 
366  elif drceSyncTasksCover.eventType == EVENT_TYPES.PROXY_FIND:
367  proxyFindTask = ProxyFindTask()
368  responseObject = proxyFindTask.process(drceSyncTasksCover.eventObject, self.executeQuery)
369  responseEvent = EVENT_TYPES.PROXY_FIND_RESPONSE
370 
371  elif drceSyncTasksCover.eventType == EVENT_TYPES.ATTR_SET:
372  attrSetTask = AttrSetTask()
373  responseObject = attrSetTask.process(drceSyncTasksCover.eventObject, self.executeQuery)
374  responseEvent = EVENT_TYPES.ATTR_SET_RESPONSE
375 
376  elif drceSyncTasksCover.eventType == EVENT_TYPES.ATTR_UPDATE:
377  attrUpdateTask = AttrUpdateTask()
378  responseObject = attrUpdateTask.process(drceSyncTasksCover.eventObject, self.executeQuery)
379  responseEvent = EVENT_TYPES.ATTR_UPDATE_RESPONSE
380 
381  elif drceSyncTasksCover.eventType == EVENT_TYPES.ATTR_DELETE:
382  attrDeleteTask = AttrDeleteTask()
383  responseObject = attrDeleteTask.process(drceSyncTasksCover.eventObject, self.executeQuery)
384  responseEvent = EVENT_TYPES.ATTR_DELETE_RESPONSE
385 
386  elif drceSyncTasksCover.eventType == EVENT_TYPES.ATTR_FETCH:
387  attrFetchTask = AttrFetchTask()
388  responseObject = attrFetchTask.process(drceSyncTasksCover.eventObject, self.executeQuery)
389  responseEvent = EVENT_TYPES.ATTR_FETCH_RESPONSE
390 
391  return self.createDRCESyncTasksCover(responseEvent, responseObject)
392 
393 
ftests.ftest_popen.process
process
Definition: ftest_popen.py:37
integrity-urlContent.type
type
Definition: integrity-urlContent.py:27
Here is the call graph for this function:
Here is the caller graph for this function:

Member Data Documentation

◆ ATT_DB

string dc_db.TasksManager.TasksManager.ATT_DB = "att_db"
static

Definition at line 103 of file TasksManager.py.

◆ ATTR_TEMPLATE

string dc_db.TasksManager.TasksManager.ATTR_TEMPLATE = "dc_attribute_template"
static

Definition at line 114 of file TasksManager.py.

◆ CONTENT_STORAGE_TYPE

string dc_db.TasksManager.TasksManager.CONTENT_STORAGE_TYPE = "content_storage_type"
static

Definition at line 110 of file TasksManager.py.

◆ CONTENT_TEMPLATE

string dc_db.TasksManager.TasksManager.CONTENT_TEMPLATE = "dc_content_template"
static

Definition at line 111 of file TasksManager.py.

◆ contentStorageType

dc_db.TasksManager.TasksManager.contentStorageType

Definition at line 145 of file TasksManager.py.

◆ DB_HOST

string dc_db.TasksManager.TasksManager.DB_HOST = "db_host"
static

Definition at line 92 of file TasksManager.py.

◆ DB_PORT

string dc_db.TasksManager.TasksManager.DB_PORT = "db_port"
static

Definition at line 93 of file TasksManager.py.

◆ DB_PWD

string dc_db.TasksManager.TasksManager.DB_PWD = "db_pwd"
static

Definition at line 95 of file TasksManager.py.

◆ DB_USER

string dc_db.TasksManager.TasksManager.DB_USER = "db_user"
static

Definition at line 94 of file TasksManager.py.

◆ dbConnections

dc_db.TasksManager.TasksManager.dbConnections

Definition at line 146 of file TasksManager.py.

◆ dBDataTask

dc_db.TasksManager.TasksManager.dBDataTask

Definition at line 189 of file TasksManager.py.

◆ dbNames

dc_db.TasksManager.TasksManager.dbNames

Definition at line 147 of file TasksManager.py.

◆ dbTaskMode

dc_db.TasksManager.TasksManager.dbTaskMode

Definition at line 124 of file TasksManager.py.

◆ dcAttrTemplate

dc_db.TasksManager.TasksManager.dcAttrTemplate

Definition at line 152 of file TasksManager.py.

◆ dcContentTemplate

dc_db.TasksManager.TasksManager.dcContentTemplate

Definition at line 156 of file TasksManager.py.

◆ dcLogTemplate

dc_db.TasksManager.TasksManager.dcLogTemplate

Definition at line 151 of file TasksManager.py.

◆ dcSiteTemplate

dc_db.TasksManager.TasksManager.dcSiteTemplate

Definition at line 149 of file TasksManager.py.

◆ dcStatTemplate

dc_db.TasksManager.TasksManager.dcStatTemplate

Definition at line 150 of file TasksManager.py.

◆ FIFTH_DB

string dc_db.TasksManager.TasksManager.FIFTH_DB = "fifth_db"
static

Definition at line 100 of file TasksManager.py.

◆ FOURTH_DB

string dc_db.TasksManager.TasksManager.FOURTH_DB = "fourth_db"
static

Definition at line 99 of file TasksManager.py.

◆ KEY_VALUE_DEFAULT_FILE

string dc_db.TasksManager.TasksManager.KEY_VALUE_DEFAULT_FILE = "key_value_default_file"
static

Definition at line 108 of file TasksManager.py.

◆ KEY_VALUE_STORAGE_DIR

string dc_db.TasksManager.TasksManager.KEY_VALUE_STORAGE_DIR = "key_value_storage_dir"
static

Definition at line 107 of file TasksManager.py.

◆ keyValueDefaultFile

dc_db.TasksManager.TasksManager.keyValueDefaultFile

Definition at line 155 of file TasksManager.py.

◆ keyValueStorageDir

dc_db.TasksManager.TasksManager.keyValueStorageDir

Definition at line 153 of file TasksManager.py.

◆ LOG_DB

string dc_db.TasksManager.TasksManager.LOG_DB = "log_db"
static

Definition at line 102 of file TasksManager.py.

◆ LOG_TEMPLATE

string dc_db.TasksManager.TasksManager.LOG_TEMPLATE = "dc_statistics_log_template"
static

Definition at line 113 of file TasksManager.py.

◆ MUTEX_LOCK_TTL

string dc_db.TasksManager.TasksManager.MUTEX_LOCK_TTL = "mutexLockTTL"
static

Definition at line 105 of file TasksManager.py.

◆ mutexLockTTL

dc_db.TasksManager.TasksManager.mutexLockTTL

Definition at line 142 of file TasksManager.py.

◆ PRIMARY_DB

string dc_db.TasksManager.TasksManager.PRIMARY_DB = "primary_db"
static

Definition at line 96 of file TasksManager.py.

◆ RAW_DATA_DIR

string dc_db.TasksManager.TasksManager.RAW_DATA_DIR = "raw_data_dir"
static

Definition at line 109 of file TasksManager.py.

◆ rawDataDir

dc_db.TasksManager.TasksManager.rawDataDir

Definition at line 154 of file TasksManager.py.

◆ SECONDARY_DB

string dc_db.TasksManager.TasksManager.SECONDARY_DB = "secondary_db"
static

Definition at line 97 of file TasksManager.py.

◆ SITE_PROP_NAME_FREQ

string dc_db.TasksManager.TasksManager.SITE_PROP_NAME_FREQ = "stats_freq_enabled"
static

Definition at line 115 of file TasksManager.py.

◆ SITE_PROP_NAME_LOG

string dc_db.TasksManager.TasksManager.SITE_PROP_NAME_LOG = "stats_log_enabled"
static

Definition at line 116 of file TasksManager.py.

◆ SITE_TEMPLATE

string dc_db.TasksManager.TasksManager.SITE_TEMPLATE = "dc_site_template"
static

Definition at line 106 of file TasksManager.py.

◆ SQL_OPERATION_LIST

list dc_db.TasksManager.TasksManager.SQL_OPERATION_LIST = ["update", "insert", "delete", "drop"]
static

Definition at line 117 of file TasksManager.py.

◆ SQLErrorCode

dc_db.TasksManager.TasksManager.SQLErrorCode

Definition at line 187 of file TasksManager.py.

◆ SQLErrorString

dc_db.TasksManager.TasksManager.SQLErrorString

Definition at line 188 of file TasksManager.py.

◆ STAT_DB

string dc_db.TasksManager.TasksManager.STAT_DB = "stat_db"
static

Definition at line 101 of file TasksManager.py.

◆ STAT_DOMAINS_DB

string dc_db.TasksManager.TasksManager.STAT_DOMAINS_DB = "stat_domains_db"
static

Definition at line 104 of file TasksManager.py.

◆ STAT_TEMPLATE

string dc_db.TasksManager.TasksManager.STAT_TEMPLATE = "dc_statistics_freq_template"
static

Definition at line 112 of file TasksManager.py.

◆ THIRD_DB

string dc_db.TasksManager.TasksManager.THIRD_DB = "third_db"
static

Definition at line 98 of file TasksManager.py.

The documentation for this class was generated from the following file: