Inheritance diagram for dc_db.URLCleanupTask.URLCleanUpTask:

Collaboration diagram for dc_db.URLCleanupTask.URLCleanUpTask:

Public Member Functions
def	__init__ (self, keyValueStorageDir, rawDataDir, dBDataTask)

def	process (self, urlCleanups, queryCallback)

def	extractUrlByCriterions (self, siteId, isUrlExtract, criterions, queryCallback, dbName=Constants.SECONDARY_DB_ID, tablePrefix=Constants.DC_URLS_TABLE_NAME_TEMPLATE)

def	getSiteFields (self, siteId, queryCallback, dbName=Constants.PRIMARY_DB_ID)

def	deleteFromDataStorage (self, urlCleanup, queryCallback)

def	deleteFromRawStorage (self, urlCleanup)

def	updateMysqlDB (self, urlCleanup, queryCallback, siteId)

def	copyUrlToDeleteDB (self, urlCleanup, queryCallback)

Public Member Functions inherited from dc_db.BaseTask.BaseTask
def	isSiteExist (self, siteId, queryCallback, userId=None)

def	generateCriterionSQL (self, criterions, additionWhere=None, siteId=None)

def	fetchByCriterions (self, criterions, queryCallback)

def	dbLock (self, mutexName, queryCallback, sleepTime=1, mutexLockTTL=Constants.DEFAULT_LOCK_TTL)

def	dbUnlock (self, mutexName, queryCallback)

def	createUrlsInsertQuery (self, siteId, localKeys, localValues)

def	copyUrlsToDcUrls (self, siteId, queryCallback)

def	statisticLogUpdate (self, localObj, urlMd5, siteId, status, queryCallback, isInsert=False)

def	calculateMd5FormUrl (self, url, urlType, useNormilize=False)

Public Attributes
	keyValueStorageDir

	rawDataDir

	recalculator

	dBDataTask

	urlMd5

Additional Inherited Members
Static Public Member Functions inherited from dc_db.BaseTask.BaseTask
def	readValueFromSiteProp (siteId, propName, queryCallback, urlMd5=None)

Detailed Description

Definition at line 29 of file URLCleanupTask.py.

Constructor & Destructor Documentation

◆ init()

def dc_db.URLCleanupTask.URLCleanUpTask.__init__	(	self,
		keyValueStorageDir,
		rawDataDir,
		dBDataTask
	)

Definition at line 36 of file URLCleanupTask.py.

   def __init__(self, keyValueStorageDir, rawDataDir, dBDataTask):
     super(URLCleanUpTask, self).__init__()
     self.keyValueStorageDir = keyValueStorageDir
     self.rawDataDir = rawDataDir
     self.recalculator = FieldRecalculator.FieldRecalculator()
     self.dBDataTask = dBDataTask
     self.urlMd5 = None
 
 

Member Function Documentation

◆ copyUrlToDeleteDB()

def dc_db.URLCleanupTask.URLCleanUpTask.copyUrlToDeleteDB	(	self,
		urlCleanup,
		queryCallback
	)

Definition at line 190 of file URLCleanupTask.py.

190 def copyUrlToDeleteDB(self, urlCleanup, queryCallback):

191 SQL_COPY_QUERY_TEMPLATE = "INSERT INTO %s SELECT * FROM `dc_urls`.`%s` WHERE `URLMD5` = '%s'"

192 tbName = Constants.DC_URLS_TABLE_NAME_TEMPLATE % urlCleanup.siteId

193 query = Constants.SQL_CREATE_QUERY_TEMPLATE % (tbName, tbName)

194 queryCallback(query, Constants.FOURTH_DB_ID)

195 query = SQL_COPY_QUERY_TEMPLATE % (tbName, tbName, self.urlMd5)

196 queryCallback(query, Constants.FOURTH_DB_ID)

197

Here is the caller graph for this function:

◆ deleteFromDataStorage()

def dc_db.URLCleanupTask.URLCleanUpTask.deleteFromDataStorage	(	self,
		urlCleanup,
		queryCallback
	)

Definition at line 136 of file URLCleanupTask.py.

   def deleteFromDataStorage(self, urlCleanup, queryCallback):
     # @todo try block + check table name
     ret = None
     if self.dBDataTask is not None:
       localUrlMd5 = self.calculateMd5FormUrl(urlCleanup.url, urlCleanup.urlType)
       dataDeleteRequest = dc.EventObjects.DataDeleteRequest(urlCleanup.siteId, localUrlMd5, None)
       ret = self.dBDataTask.process(dataDeleteRequest, queryCallback)
     return ret
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ deleteFromRawStorage()

def dc_db.URLCleanupTask.URLCleanUpTask.deleteFromRawStorage	(	self,
		urlCleanup
	)

Definition at line 150 of file URLCleanupTask.py.

   def deleteFromRawStorage(self, urlCleanup):
     localUrlMd5 = self.calculateMd5FormUrl(urlCleanup.url, urlCleanup.urlType)
     dataDir = self.rawDataDir + '/' + urlCleanup.siteId + '/' + PathMaker(localUrlMd5).getDir()
     logger.debug(">>> CLEANUP DIR = " + str(dataDir))
     if os.path.isdir(dataDir):
       try:
         shutil.rmtree(dataDir)
         hiLevelDir = dataDir[0: dataDir.rfind('/') if dataDir.rfind('/') >= 0 else len(dataDir)]
         if len(os.listdir(hiLevelDir)) == 0:
           shutil.rmtree(hiLevelDir)
       except OSError as ex:
         logger.debug(">>> [%s] Dir delete error - MSG [%s]", dataDir, str(ex.message))
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ extractUrlByCriterions()

def dc_db.URLCleanupTask.URLCleanUpTask.extractUrlByCriterions	(	self,
		siteId,
		isUrlExtract,
		criterions,
		queryCallback,
		dbName = `Constants.SECONDARY_DB_ID`,
		tablePrefix = `Constants.DC_URLS_TABLE_NAME_TEMPLATE`
	)

Definition at line 99 of file URLCleanupTask.py.

                              tablePrefix=Constants.DC_URLS_TABLE_NAME_TEMPLATE):
     retUrls = []
     tableName = tablePrefix % siteId
     if isUrlExtract:
       SQLUrlExtractor = "SELECT `URL` FROM `%s`" % tableName
     else:
       SQLUrlExtractor = "SELECT `URLMd5` FROM `%s`" % tableName
     query = SQLUrlExtractor + self.generateCriterionSQL(criterions)
     res = queryCallback(query, dbName)
     if hasattr(res, '__iter__'):
       logger.debug(">>> Select URL len(res) = " + str(len(res)))
       for row in res:
         retUrls.append(row[0])
 
     return retUrls
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ getSiteFields()

def dc_db.URLCleanupTask.URLCleanUpTask.getSiteFields	(	self,
		siteId,
		queryCallback,
		dbName = `Constants.PRIMARY_DB_ID`
	)

Definition at line 120 of file URLCleanupTask.py.

   def getSiteFields(self, siteId, queryCallback, dbName=Constants.PRIMARY_DB_ID):
     res = None
 
     query = "SELECT * FROM `dc_sites`.`sites` WHERE `Id` = '%s' LIMIT 1" % siteId
     res = queryCallback(query, dbName, Constants.EXEC_NAME)
     if len(res) > 0:
       res = res[0]
 
     return res
 
 

Here is the caller graph for this function:

◆ process()

def dc_db.URLCleanupTask.URLCleanUpTask.process	(	self,
		urlCleanups,
		queryCallback
	)

Definition at line 50 of file URLCleanupTask.py.

   def process(self, urlCleanups, queryCallback):
     generalResponse = GeneralResponse()
     for urlCleanup in urlCleanups:
       # @todo add more complex case
       if urlCleanup.siteId == "":
         urlCleanup.siteId = "0"
       if self.isSiteExist(urlCleanup.siteId, queryCallback):
         try:
           localUrls = []
           if urlCleanup.url is None:
             localUrls = self.extractUrlByCriterions(urlCleanup.siteId, (urlCleanup.urlType == \
                                                     dc.EventObjects.URLStatus.URL_TYPE_URL), urlCleanup.criterions,
                                                     queryCallback)
           else:
             localUrls.append(urlCleanup.url)
           for localUrl in localUrls:
             urlCleanup.url = localUrl
             self.urlMd5 = self.calculateMd5FormUrl(urlCleanup.url, urlCleanup.urlType)
             StatisticLogManager.logUpdate(queryCallback, "LOG_URL_CLEANUP", urlCleanup, urlCleanup.siteId, self.urlMd5)
             if urlCleanup.delayedType == dc.EventObjects.NOT_DELAYED_OPERATION:
               # self.deleteFromKeyValue(urlCleanup)
               self.deleteFromDataStorage(urlCleanup, queryCallback)
               self.deleteFromRawStorage(urlCleanup)
             self.updateMysqlDB(urlCleanup, queryCallback, urlCleanup.siteId)
             if urlCleanup.delayedType == dc.EventObjects.DELAYED_OPERATION:
               self.copyUrlToDeleteDB(urlCleanup, queryCallback)
 
             # Call remove attributes
             AttrDeleteTask.deleteUrlsAttributes(urlCleanup.siteId, self.urlMd5, queryCallback)
 
           generalResponse.statuses.append(True)
           self.recalculator.commonRecalc(urlCleanup.siteId, queryCallback, \
                                          dc.EventObjects.FieldRecalculatorObj.PARTITION_RECALC)
         except Exception:
           generalResponse.statuses.append(False)
           type_, value_, traceback_ = sys.exc_info()  # pylint: disable=unused-variable
           stack = traceback.format_tb(traceback_)
           logger.error(str(stack.pop()))
       else:
         generalResponse.statuses.append(False)
     return generalResponse
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ updateMysqlDB()

def dc_db.URLCleanupTask.URLCleanUpTask.updateMysqlDB	(	self,
		urlCleanup,
		queryCallback,
		siteId
	)

Definition at line 169 of file URLCleanupTask.py.

   def updateMysqlDB(self, urlCleanup, queryCallback, siteId):
     localState = urlCleanup.state if urlCleanup.state is not None else "state"
     localStatus = urlCleanup.status if urlCleanup.status is not None else "status"
     uDate = ''
     if localStatus == dc.EventObjects.URL.STATUS_NEW:
       sf = self.getSiteFields(siteId, queryCallback)
       if sf is not None and 'RecrawlPeriod' in sf:
         uDate = ", `UDate`=DATE_SUB(`UDate`,  INTERVAL %s MINUTE)" % sf['RecrawlPeriod']
     else:
       uDate = ", `UDate`=NOW()"
     sqlt = "UPDATE `%s` SET `TcDate`=NOW()%s, `state` = '%s', `status` = '%s' WHERE `URLMD5` = '%s' LIMIT 1"
     localUrlMd5 = self.calculateMd5FormUrl(urlCleanup.url, urlCleanup.urlType)
     tableName = Constants.DC_URLS_TABLE_NAME_TEMPLATE % urlCleanup.siteId
     query = sqlt % (tableName, uDate, localState, localStatus, localUrlMd5)
     queryCallback(query, Constants.SECONDARY_DB_ID)
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

◆ dBDataTask

dc_db.URLCleanupTask.URLCleanUpTask.dBDataTask

Definition at line 41 of file URLCleanupTask.py.

◆ keyValueStorageDir

dc_db.URLCleanupTask.URLCleanUpTask.keyValueStorageDir

Definition at line 38 of file URLCleanupTask.py.

◆ rawDataDir

dc_db.URLCleanupTask.URLCleanUpTask.rawDataDir

Definition at line 39 of file URLCleanupTask.py.

◆ recalculator

dc_db.URLCleanupTask.URLCleanUpTask.recalculator

Definition at line 40 of file URLCleanupTask.py.

◆ urlMd5

dc_db.URLCleanupTask.URLCleanUpTask.urlMd5

Definition at line 42 of file URLCleanupTask.py.

The documentation for this class was generated from the following file:

sources/hce/dc_db/URLCleanupTask.py

Public Member Functions

Public Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

◆ __init__()

Member Function Documentation

◆ copyUrlToDeleteDB()

◆ deleteFromDataStorage()

◆ deleteFromRawStorage()

◆ extractUrlByCriterions()

◆ getSiteFields()

◆ process()

◆ updateMysqlDB()

Member Data Documentation

◆ dBDataTask

◆ keyValueStorageDir

◆ rawDataDir

◆ recalculator

◆ urlMd5

◆ init()