dc_db.URLDeleteTask.URLDeleteTask Class Reference

Inheritance diagram for dc_db.URLDeleteTask.URLDeleteTask:

Collaboration diagram for dc_db.URLDeleteTask.URLDeleteTask:

Public Member Functions
def	__init__ (self, keyValueStorageDir, rawDataDir, dBDataTask)
def	process (self, urlDeletes, queryCallback)
def	deleteFromMysqlDB (self, urlDelete, queryCallback)
def	copyUrlToDeleteDB (self, urlDelete, queryCallback)
Public Member Functions inherited from dc_db.BaseTask.BaseTask
def	isSiteExist (self, siteId, queryCallback, userId=None)
def	generateCriterionSQL (self, criterions, additionWhere=None, siteId=None)
def	fetchByCriterions (self, criterions, queryCallback)
def	dbLock (self, mutexName, queryCallback, sleepTime=1, mutexLockTTL=Constants.DEFAULT_LOCK_TTL)
def	dbUnlock (self, mutexName, queryCallback)
def	createUrlsInsertQuery (self, siteId, localKeys, localValues)
def	copyUrlsToDcUrls (self, siteId, queryCallback)
def	statisticLogUpdate (self, localObj, urlMd5, siteId, status, queryCallback, isInsert=False)
def	calculateMd5FormUrl (self, url, urlType, useNormilize=False)

Public Attributes
	uRLCleanUpTask
	recalculator
	urlMd5

Additional Inherited Members
Static Public Member Functions inherited from dc_db.BaseTask.BaseTask
def	readValueFromSiteProp (siteId, propName, queryCallback, urlMd5=None)

Detailed Description

Definition at line 25 of file URLDeleteTask.py.

Constructor & Destructor Documentation

__init__()

def dc_db.URLDeleteTask.URLDeleteTask.__init__	(		self,
			keyValueStorageDir,
			rawDataDir,
			dBDataTask
	)

Definition at line 32 of file URLDeleteTask.py.

  def __init__(self, keyValueStorageDir, rawDataDir, dBDataTask):
    super(URLDeleteTask, self).__init__()
    self.uRLCleanUpTask = URLCleanUpTask(keyValueStorageDir, rawDataDir, dBDataTask)
    self.recalculator = FieldRecalculator()
    self.urlMd5 = None
    # self.dBDataTask = dBDataTask

Member Function Documentation

copyUrlToDeleteDB()

def dc_db.URLDeleteTask.URLDeleteTask.copyUrlToDeleteDB	(		self,
			urlDelete,
			queryCallback
	)

Definition at line 108 of file URLDeleteTask.py.

  def copyUrlToDeleteDB(self, urlDelete, queryCallback):
    if urlDelete.urlType == dc.EventObjects.URLStatus.URL_TYPE_URL:
      localMd5 = hashlib.md5(urlDelete.url).hexdigest()
    else:
      localMd5 = urlDelete.url
    SQL_COPY_QUERY_TEMPLATE = "INSERT INTO %s SELECT * FROM dc_urls.%s WHERE `URLMd5` = '%s'"
    tbName = Constants.DC_URLS_TABLE_NAME_TEMPLATE % urlDelete.siteId
    # TODO: One more query for each delete request is too heavy for DB
    # query = Constants.SQL_CREATE_QUERY_TEMPLATE % (tbName, tbName)
    # queryCallback(query, Constants.FOURTH_DB_ID)
    query = SQL_COPY_QUERY_TEMPLATE % (tbName, tbName, localMd5)
    queryCallback(query, Constants.FOURTH_DB_ID, Constants.EXEC_INDEX, True)

deleteFromMysqlDB()

def dc_db.URLDeleteTask.URLDeleteTask.deleteFromMysqlDB	(		self,
			urlDelete,
			queryCallback
	)

Definition at line 95 of file URLDeleteTask.py.

  def deleteFromMysqlDB(self, urlDelete, queryCallback):
    self.urlMd5 = self.calculateMd5FormUrl(urlDelete.url, urlDelete.urlType, True)
    SQL_CLAUSE = ("`URLMd5` = '%s'" % self.urlMd5)
    UPDATE_SQL_TEMPLATE = "DELETE FROM `%s` WHERE %s"
    tableName = Constants.DC_URLS_TABLE_NAME_TEMPLATE % urlDelete.siteId
    query = UPDATE_SQL_TEMPLATE % (tableName, SQL_CLAUSE)
    queryCallback(query, Constants.SECONDARY_DB_ID)

Here is the call graph for this function:

Here is the caller graph for this function:

process()

def dc_db.URLDeleteTask.URLDeleteTask.process	(		self,
			urlDeletes,
			queryCallback
	)

Definition at line 45 of file URLDeleteTask.py.

  def process(self, urlDeletes, queryCallback):
    ret = GeneralResponse()
    for urlDelete in urlDeletes:
      if urlDelete.siteId == "":
        urlDelete.siteId = "0"
      if self.isSiteExist(urlDelete.siteId, queryCallback):
        try:
          localUrls = []
          if urlDelete.url is None:
            isUrlExtract = False
            if urlDelete.urlType == dc.EventObjects.URLStatus.URL_TYPE_URL:
              isUrlExtract = True
            localUrls = self.uRLCleanUpTask.extractUrlByCriterions(urlDelete.siteId, isUrlExtract,
                                                                   urlDelete.criterions, queryCallback)
          else:
            localUrls.append(urlDelete.url)
          logger.debug(">>> [URLDelete] localUrls size = " + str(len(localUrls)))
          for localUrl in localUrls:
            try:
              urlDelete.url = localUrl
              if urlDelete.delayedType == dc.EventObjects.NOT_DELAYED_OPERATION:
                self.uRLCleanUpTask.deleteFromDataStorage(urlDelete, queryCallback)
                self.uRLCleanUpTask.deleteFromRawStorage(urlDelete)
              elif urlDelete.delayedType == dc.EventObjects.DELAYED_OPERATION:
                self.copyUrlToDeleteDB(urlDelete, queryCallback)
              self.deleteFromMysqlDB(urlDelete, queryCallback)
              AttrDeleteTask.deleteUrlsAttributes(urlDelete.siteId, self.urlMd5, queryCallback)
              if self.urlMd5 is not None:
                StatisticLogManager.statisticUpdate(queryCallback, Constants.StatFreqConstants.FREQ_DELETE, 1,
                                                    urlDelete.siteId, self.urlMd5)
                StatisticLogManager.statisticUpdate(queryCallback, Constants.StatFreqConstants.FREQ_DELETED_STATE, 1,
                                                    urlDelete.siteId, self.urlMd5)
                StatisticLogManager.logUpdate(queryCallback, "LOG_DELETE", urlDelete, urlDelete.siteId, self.urlMd5)
            except Exception as ex:
              logger.debug(">>> [URLDelete] Some Type Exception [LOOP] = " + str(type(ex)) + " " + str(ex))
          ret.statuses.append(True)
          self.recalculator.commonRecalc(urlDelete.siteId, queryCallback)
        except Exception as excp:
          logger.debug(">>> [URLDelete] Some Type Exception = " + str(type(excp)) + " " + str(excp))
          ret.statuses.append(False)
      else:
        ret.statuses.append(False)

    return ret

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

recalculator

dc_db.URLDeleteTask.URLDeleteTask.recalculator

Definition at line 35 of file URLDeleteTask.py.

uRLCleanUpTask

dc_db.URLDeleteTask.URLDeleteTask.uRLCleanUpTask

Definition at line 34 of file URLDeleteTask.py.

urlMd5

dc_db.URLDeleteTask.URLDeleteTask.urlMd5

Definition at line 36 of file URLDeleteTask.py.

---

The documentation for this class was generated from the following file:

• sources/hce/dc_db/URLDeleteTask.py