dc_db.URLPurgeTask.URLPurgeTask Class Reference

Inheritance diagram for dc_db.URLPurgeTask.URLPurgeTask:

Collaboration diagram for dc_db.URLPurgeTask.URLPurgeTask:

Public Member Functions
def	__init__ (self, keyValueStorageDir, rawDataDir, dBDataTask)
def	isAvailableUrls (self, urlPurge, tbName, queryCallback)
def	getAdditionPurges (self, urlPurge, siteLimits, queryCallback)
def	isDeleteTableExist (self, siteId, queryCallback)
def	process (self, urlPurges, queryCallback)
def	deleteUrlDBField (self, urlPurge, queryCallback)
def	checkUrlInDcUrls (self, urlPurge, queryCallback)
Public Member Functions inherited from dc_db.BaseTask.BaseTask
def	isSiteExist (self, siteId, queryCallback, userId=None)
def	generateCriterionSQL (self, criterions, additionWhere=None, siteId=None)
def	fetchByCriterions (self, criterions, queryCallback)
def	dbLock (self, mutexName, queryCallback, sleepTime=1, mutexLockTTL=Constants.DEFAULT_LOCK_TTL)
def	dbUnlock (self, mutexName, queryCallback)
def	createUrlsInsertQuery (self, siteId, localKeys, localValues)
def	copyUrlsToDcUrls (self, siteId, queryCallback)
def	statisticLogUpdate (self, localObj, urlMd5, siteId, status, queryCallback, isInsert=False)
def	calculateMd5FormUrl (self, url, urlType, useNormilize=False)

Public Attributes
	uRLCleanUpTask
	dBDataTask
	urlMd5

Additional Inherited Members
Static Public Member Functions inherited from dc_db.BaseTask.BaseTask
def	readValueFromSiteProp (siteId, propName, queryCallback, urlMd5=None)

Detailed Description

Definition at line 24 of file URLPurgeTask.py.

Constructor & Destructor Documentation

__init__()

def dc_db.URLPurgeTask.URLPurgeTask.__init__	(		self,
			keyValueStorageDir,
			rawDataDir,
			dBDataTask
	)

Definition at line 30 of file URLPurgeTask.py.

  def __init__(self, keyValueStorageDir, rawDataDir, dBDataTask):
    super(URLPurgeTask, self).__init__()
    self.uRLCleanUpTask = URLCleanUpTask(keyValueStorageDir, rawDataDir, dBDataTask)
    self.dBDataTask = dBDataTask
    self.urlMd5 = None

Member Function Documentation

checkUrlInDcUrls()

def dc_db.URLPurgeTask.URLPurgeTask.checkUrlInDcUrls	(		self,
			urlPurge,
			queryCallback
	)

Definition at line 185 of file URLPurgeTask.py.

  def checkUrlInDcUrls(self, urlPurge, queryCallback):
    ret = False
    SQL_DELETE_TEMPLATE = "SELECT url FROM %s WHERE `UrlMd5` = '%s' AND `tcDate` NOT IN " + \
                          "(SELECT `tcDate` FROM dc_urls_deleted.%s WHERE `UrlMd5` = '%s') LIMIT 1"
    dbName = Constants.DC_URLS_TABLE_NAME_TEMPLATE % urlPurge.siteId
    if urlPurge.urlType == dc.EventObjects.URLStatus.URL_TYPE_URL:
      urlMd5 = hashlib.md5(urlPurge.url).hexdigest()
    else:
      urlMd5 = urlPurge.url
    query = SQL_DELETE_TEMPLATE % (dbName, urlMd5, dbName, urlMd5)
    res = queryCallback(query, Constants.SECONDARY_DB_ID)
    if res is not None and len(res) > 0:
      ret = True
    logger.debug(">>> [PURGE] checkUrlInDcUrls 'UrlMd5' = " + urlMd5)
    if ret:
      logger.debug(" has record in dc_urls")
    else:
      logger.debug(" DOESN'T has record in dc_urls")
    return ret

deleteUrlDBField()

def dc_db.URLPurgeTask.URLPurgeTask.deleteUrlDBField	(		self,
			urlPurge,
			queryCallback
	)

Definition at line 170 of file URLPurgeTask.py.

  def deleteUrlDBField(self, urlPurge, queryCallback):
    SQL_DELETE_TEMPLATE = "DELETE FROM %s WHERE `UrlMd5` = '%s'"
    dbName = Constants.DC_URLS_TABLE_NAME_TEMPLATE % urlPurge.siteId
    if urlPurge.urlType == dc.EventObjects.URLStatus.URL_TYPE_URL:
      self.urlMd5 = hashlib.md5(urlPurge.url).hexdigest()
    else:
      self.urlMd5 = urlPurge.url
    query = SQL_DELETE_TEMPLATE % (dbName, self.urlMd5)
    queryCallback(query, Constants.FOURTH_DB_ID)

Here is the caller graph for this function:

getAdditionPurges()

def dc_db.URLPurgeTask.URLPurgeTask.getAdditionPurges	(		self,
			urlPurge,
			siteLimits,
			queryCallback
	)

Definition at line 63 of file URLPurgeTask.py.

  def getAdditionPurges(self, urlPurge, siteLimits, queryCallback):
    ret = []
    if siteLimits is not None and hasattr(siteLimits, '__iter__') and len(siteLimits) >= 2 and int(siteLimits[0]) >= 0:
      query = "SHOW TABLES"
      res = queryCallback(query, Constants.FOURTH_DB_ID)
      if res is not None:
        startLimit = int(siteLimits[0])
        countLimit = int(siteLimits[1])
        if countLimit == dc.EventObjects.URLPurge.ALL_SITES:
          countLimit = len(res)
        i = startLimit
        for num in xrange(i, len(res)):
          if len(ret) >= countLimit:
            break
          if res[num] is not None and res[num][0] is not None and \
          self.isAvailableUrls(urlPurge, res[num][0], queryCallback):
            localPurge = copy.deepcopy(urlPurge)
            localPurge.siteId = res[num][0][5:]
            localPurge.url = None
            ret.append(localPurge)
    else:
      logger.error(">>> siteLimits field must be type of [x, x] and not None")
    return ret

Here is the call graph for this function:

Here is the caller graph for this function:

isAvailableUrls()

def dc_db.URLPurgeTask.URLPurgeTask.isAvailableUrls	(		self,
			urlPurge,
			tbName,
			queryCallback
	)

Definition at line 43 of file URLPurgeTask.py.

  def isAvailableUrls(self, urlPurge, tbName, queryCallback):
    ret = False
    localUrls = self.uRLCleanUpTask.extractUrlByCriterions(tbName[5:], True, urlPurge.criterions,
                                                           queryCallback, Constants.FOURTH_DB_ID)
    if localUrls is not None and len(localUrls) > 0:
      ret = True
    if ret:
      logger.debug(">>> Has urls by criterions, bdName = " + tbName)
    else:
      logger.debug(">>> Not content urls by criterions, bdName = " + tbName)
    return ret

Here is the caller graph for this function:

isDeleteTableExist()

def dc_db.URLPurgeTask.URLPurgeTask.isDeleteTableExist	(		self,
			siteId,
			queryCallback
	)

Definition at line 93 of file URLPurgeTask.py.

  def isDeleteTableExist(self, siteId, queryCallback):
    ret = False
    query = "SHOW TABLES"
    dbName = Constants.DC_URLS_TABLE_NAME_TEMPLATE % siteId
    res = queryCallback(query, Constants.FOURTH_DB_ID)
    logger.debug(">>> Delete tables = " + str(res))
    if res is not None and hasattr(res, '__iter__'):
      for table in res:
        if table is not None and hasattr(table, '__iter__') and dbName in table:
          ret = True
          break
    return ret

Here is the caller graph for this function:

process()

def dc_db.URLPurgeTask.URLPurgeTask.process	(		self,
			urlPurges,
			queryCallback
	)

Definition at line 112 of file URLPurgeTask.py.

  def process(self, urlPurges, queryCallback):
    generalResponse = GeneralResponse()

    newPurges = copy.deepcopy(urlPurges)
    for urlPurge in urlPurges:
      if urlPurge.siteId is None:
        logger.debug(">>> Site Limits = " + str(urlPurge.siteLimits))
        newPurges = newPurges + self.getAdditionPurges(urlPurge, urlPurge.siteLimits, queryCallback)

    if len(urlPurges) != len(newPurges):
      logger.debug(">>> Purges reassign")
      urlPurges = newPurges

    for urlPurge in urlPurges:
      # @todo add more complex case
      urlsCount = 0
      if urlPurge.siteId == "":
        urlPurge.siteId = "0"
      if self.isDeleteTableExist(urlPurge.siteId, queryCallback):
        try:
          localUrls = []
          if urlPurge.url is None:
            isUrlExtract = False
            logger.debug(">>> UrlType = " + str(urlPurge.urlType))
            if urlPurge.urlType == dc.EventObjects.URLStatus.URL_TYPE_URL:
              isUrlExtract = True
            localUrls = self.uRLCleanUpTask.extractUrlByCriterions(urlPurge.siteId, isUrlExtract, urlPurge.criterions,
                                                                   queryCallback, Constants.FOURTH_DB_ID)
          else:
            localUrls.append(urlPurge.url)
          logger.debug(">>> [PURGE] localUrls size = " + str(len(localUrls)))
          for localUrl in localUrls:
            try:
              urlPurge.url = localUrl
              if not self.checkUrlInDcUrls(urlPurge, queryCallback):
                self.uRLCleanUpTask.deleteFromDataStorage(urlPurge, queryCallback)
                self.uRLCleanUpTask.deleteFromRawStorage(urlPurge)
              self.deleteUrlDBField(urlPurge, queryCallback)
              if self.urlMd5 is not None:
                StatisticLogManager.statisticUpdate(queryCallback, Constants.StatFreqConstants.FREQ_PURGED_STATE, 1,
                                                    urlPurge.siteId, self.urlMd5)
              urlsCount = urlsCount + 1
            except Exception as ex:
              logger.debug(">>> [PURGE] Some Type Exception [LOOP] = " + str(type(ex)) + " " + str(ex))
        except Exception as ex:
          logger.debug(">>> [PURGE] Some Type Exception = " + str(type(ex)) + " " + str(ex))
      else:
        logger.debug(">>> [PURGE] Table not found, SiteId = " + str(urlPurge.siteId))

      generalResponse.statuses.append([urlPurge.siteId, urlsCount])
      logger.debug(">>> [PURGE] Rsult = " + str([urlPurge.siteId, urlsCount]))
    return generalResponse

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

dBDataTask

dc_db.URLPurgeTask.URLPurgeTask.dBDataTask

Definition at line 33 of file URLPurgeTask.py.

uRLCleanUpTask

dc_db.URLPurgeTask.URLPurgeTask.uRLCleanUpTask

Definition at line 32 of file URLPurgeTask.py.

urlMd5

dc_db.URLPurgeTask.URLPurgeTask.urlMd5

Definition at line 34 of file URLPurgeTask.py.

---

The documentation for this class was generated from the following file:

• sources/hce/dc_db/URLPurgeTask.py