dc_db.SiteCleanUpTask.SiteCleanUpTask Class Reference

Inheritance diagram for dc_db.SiteCleanUpTask.SiteCleanUpTask:

Collaboration diagram for dc_db.SiteCleanUpTask.SiteCleanUpTask:

Public Member Functions
def	__init__ (self, keyValueStorageDir, rawDataDir, dBDataTask)
def	process (self, siteCleanups, queryCallback)
def	staticUpdate (self, sqlTemplate, siteCleanup, queryCallback)
def	cleanUpMysqlStorage (self, siteCleanup, queryCallback)
def	trancateArbitraryTable (self, tablePrefix, siteCleanup, dbId, queryCallback)
def	cleanUpMysqlSiteTable (self, siteCleanup, queryCallback)
def	cleanUpDBStorage (self, siteCleanup, filesSuffix, queryCallback)
def	cleanUpRawDataStorage (self, siteCleanup)
Public Member Functions inherited from dc_db.BaseTask.BaseTask
def	isSiteExist (self, siteId, queryCallback, userId=None)
def	generateCriterionSQL (self, criterions, additionWhere=None, siteId=None)
def	fetchByCriterions (self, criterions, queryCallback)
def	dbLock (self, mutexName, queryCallback, sleepTime=1, mutexLockTTL=Constants.DEFAULT_LOCK_TTL)
def	dbUnlock (self, mutexName, queryCallback)
def	createUrlsInsertQuery (self, siteId, localKeys, localValues)
def	copyUrlsToDcUrls (self, siteId, queryCallback)
def	statisticLogUpdate (self, localObj, urlMd5, siteId, status, queryCallback, isInsert=False)
def	calculateMd5FormUrl (self, url, urlType, useNormilize=False)

Public Attributes
	keyValueStorageDir
	rawDataDir
	errorCode
	errorMessage
	dBDataTask
	fieldRecalculator

Additional Inherited Members
Static Public Member Functions inherited from dc_db.BaseTask.BaseTask
def	readValueFromSiteProp (siteId, propName, queryCallback, urlMd5=None)

Detailed Description

Definition at line 33 of file SiteCleanUpTask.py.

Constructor & Destructor Documentation

__init__()

def dc_db.SiteCleanUpTask.SiteCleanUpTask.__init__	(		self,
			keyValueStorageDir,
			rawDataDir,
			dBDataTask
	)

Definition at line 40 of file SiteCleanUpTask.py.

  def __init__(self, keyValueStorageDir, rawDataDir, dBDataTask):
    self.keyValueStorageDir = keyValueStorageDir
    self.rawDataDir = rawDataDir
    self.errorCode = 0
    self.errorMessage = "OK"
    self.dBDataTask = dBDataTask
    self.fieldRecalculator = FieldRecalculator()

Member Function Documentation

cleanUpDBStorage()

def dc_db.SiteCleanUpTask.SiteCleanUpTask.cleanUpDBStorage	(		self,
			siteCleanup,
			filesSuffix,
			queryCallback
	)

Definition at line 168 of file SiteCleanUpTask.py.

  def cleanUpDBStorage(self, siteCleanup, filesSuffix, queryCallback):
    ret = None
    if self.dBDataTask is not None:
      dataDeleteRequest = dc.EventObjects.DataDeleteRequest(siteCleanup.id, None, filesSuffix)
      ret = self.dBDataTask.process(dataDeleteRequest, queryCallback)
    return ret

Here is the call graph for this function:

Here is the caller graph for this function:

cleanUpMysqlSiteTable()

def dc_db.SiteCleanUpTask.SiteCleanUpTask.cleanUpMysqlSiteTable	(		self,
			siteCleanup,
			queryCallback
	)

Definition at line 158 of file SiteCleanUpTask.py.

  def cleanUpMysqlSiteTable(self, siteCleanup, queryCallback):
    CLEAR_SITE_RECORS_SQL = ("UPDATE `sites` SET TcDate=NOW(), Resources=0, Iterations=0, State=%s, " +
                             "ErrorMask=0, Errors=0, Contents=0, CollectedURLs=0 WHERE id = '%s'")
    query = CLEAR_SITE_RECORS_SQL % (str(siteCleanup.state), siteCleanup.id)
    queryCallback(query, Constants.PRIMARY_DB_ID)

Here is the caller graph for this function:

cleanUpMysqlStorage()

def dc_db.SiteCleanUpTask.SiteCleanUpTask.cleanUpMysqlStorage	(		self,
			siteCleanup,
			queryCallback
	)

Definition at line 117 of file SiteCleanUpTask.py.

  def cleanUpMysqlStorage(self, siteCleanup, queryCallback):
    if siteCleanup.saveRootUrls:
      SQL_COPY_QUERY_TEMPLATE = "INSERT INTO %s SELECT * FROM dc_urls.%s WHERE dc_urls.%s.ParentMd5 != ''"
      SQL_DEL_QUERY_TEMPLATE = "DELETE FROM `%s` WHERE ParentMd5 != ''"
      self.staticUpdate("SELECT `URLMd5` FROM %s WHERE ParentMd5 != ''", siteCleanup, queryCallback)
    else:
      SQL_COPY_QUERY_TEMPLATE = "INSERT INTO %s SELECT * FROM dc_urls.%s"
      SQL_DEL_QUERY_TEMPLATE = "TRUNCATE TABLE `%s`"
      self.staticUpdate("SELECT `URLMd5` FROM %s", siteCleanup, queryCallback)
    tbName = Constants.DC_URLS_TABLE_NAME_TEMPLATE % siteCleanup.id
    query = SQL_DEL_QUERY_TEMPLATE % tbName
    if siteCleanup.delayedType == dc.EventObjects.NOT_DELAYED_OPERATION:
      queryCallback(query, Constants.SECONDARY_DB_ID)
    elif siteCleanup.delayedType == dc.EventObjects.DELAYED_OPERATION:
      query = Constants.SQL_CREATE_QUERY_TEMPLATE % (tbName, tbName)
      queryCallback(query, Constants.FOURTH_DB_ID)
      if siteCleanup.saveRootUrls:
        query = SQL_COPY_QUERY_TEMPLATE % (tbName, tbName, tbName)
      else:
        query = SQL_COPY_QUERY_TEMPLATE % (tbName, tbName)
      queryCallback(query, Constants.FOURTH_DB_ID)
      query = SQL_DEL_QUERY_TEMPLATE % tbName
      queryCallback(query, Constants.SECONDARY_DB_ID)

Here is the call graph for this function:

Here is the caller graph for this function:

cleanUpRawDataStorage()

def dc_db.SiteCleanUpTask.SiteCleanUpTask.cleanUpRawDataStorage	(		self,
			siteCleanup
	)

Definition at line 179 of file SiteCleanUpTask.py.

  def cleanUpRawDataStorage(self, siteCleanup):
    try:
      tmpDirName = self.rawDataDir + "/" + os.path.basename(tempfile.NamedTemporaryFile().name)
      originDirName = self.rawDataDir + "/" + siteCleanup.id
      logger.debug(">>> originDir = %s", str(originDirName))
      os.rename(originDirName, tmpDirName)
      shutil.rmtree(tmpDirName)
    except Exception as err:
      type_, value_, traceback_ = sys.exc_info()
      logger.debug("type_ = %s, value_ = %s", str(type_), str(value_))
      stack = traceback.format_tb(traceback_)
      logger.debug("Error: %s\n%s", str(err), str(stack.pop()))
      logger.debug(">>> [cleanUpRawDataStorage] CURRENT DIR " + str(os.getcwd()))
#      self.errorCode = 2
#      self.errorMessage = (">>> cleanUpRawDataStorage Error")

process()

def dc_db.SiteCleanUpTask.SiteCleanUpTask.process	(		self,
			siteCleanups,
			queryCallback
	)

Definition at line 54 of file SiteCleanUpTask.py.

  def process(self, siteCleanups, queryCallback):
    ret = GeneralResponse()
    if not isinstance(siteCleanups, list):
      siteCleanups = [siteCleanups]

    for siteCleanup in siteCleanups:
      self.errorCode = 0
      self.errorMessage = "OK"
      if self.isSiteExist(siteCleanup.id, queryCallback):
        self.cleanUpMysqlStorage(siteCleanup, queryCallback)
        if siteCleanup.historyCleanUp == dc.EventObjects.SiteCleanup.HISTORY_CLEANUP_LOG or \
        siteCleanup.historyCleanUp == dc.EventObjects.SiteCleanup.HISTORY_CLEANUP_FULL:
          self.trancateArbitraryTable(Constants.DC_LOG_TABLE_NAME_TEMPLATE, siteCleanup, Constants.LOG_DB_ID,
                                      queryCallback)
        if siteCleanup.historyCleanUp == dc.EventObjects.SiteCleanup.HISTORY_CLEANUP_FULL:
          self.trancateArbitraryTable(Constants.DC_FREQ_TABLE_NAME_TEMPLATE, siteCleanup, Constants.STAT_DB_ID,
                                      queryCallback)
        if siteCleanup.delayedType == dc.EventObjects.NOT_DELAYED_OPERATION:
          self.cleanUpDBStorage(siteCleanup, KEY_VALUE_FILE_NAME_TEMPLATE, queryCallback)
          self.cleanUpDBStorage(siteCleanup, KEY_VALUE_FIELDS_FILE_NAME_TEMPLATE, queryCallback)
          self.cleanUpRawDataStorage(siteCleanup)
        self.cleanUpMysqlSiteTable(siteCleanup, queryCallback)
        if siteCleanup.moveURLs:
          self.copyUrlsToDcUrls(siteCleanup.id, queryCallback)
        self.fieldRecalculator.updateSiteCleanupFields(siteCleanup.id, queryCallback)

        # cleaunup attributes
        self.trancateArbitraryTable(Constants.DC_ATT_TABLE_NAME_TEMPLATE, siteCleanup, Constants.ATT_DB_ID,
                                    queryCallback)
      else:
        self.errorCode = Constants.EXIT_CODE_GLOBAL_ERROR
        self.errorMessage = (">>> Site id [%s] not found" % siteCleanup.id)

      ret.errorCode = self.errorCode
      ret.statuses.append(ret.errorCode)
      if ret.errorMessage is None or ret.errorMessage == "":
        ret.errorMessage = self.errorMessage
      else:
        ret.errorMessage += ("-" + self.errorMessage)

    return ret

Here is the call graph for this function:

Here is the caller graph for this function:

staticUpdate()

def dc_db.SiteCleanUpTask.SiteCleanUpTask.staticUpdate	(		self,
			sqlTemplate,
			siteCleanup,
			queryCallback
	)

Definition at line 102 of file SiteCleanUpTask.py.

  def staticUpdate(self, sqlTemplate, siteCleanup, queryCallback):
    tbName = Constants.DC_URLS_TABLE_NAME_TEMPLATE % siteCleanup.id
    query = sqlTemplate % tbName
    res = queryCallback(query, Constants.SECONDARY_DB_ID)
    if res is not None:
      for elem in res:
        if elem[0] is not None:
          StatisticLogManager.statisticUpdate(queryCallback, Constants.StatFreqConstants.FREQ_DELETED_STATE, 1,
                                              siteCleanup.id, elem[0])

Here is the caller graph for this function:

trancateArbitraryTable()

def dc_db.SiteCleanUpTask.SiteCleanUpTask.trancateArbitraryTable	(		self,
			tablePrefix,
			siteCleanup,
			dbId,
			queryCallback
	)

Definition at line 147 of file SiteCleanUpTask.py.

  def trancateArbitraryTable(self, tablePrefix, siteCleanup, dbId, queryCallback):
    tbName = tablePrefix % siteCleanup.id
    SQL_TRUNCATE_QUERY_TEMPLATE = "TRUNCATE TABLE `%s`"
    query = SQL_TRUNCATE_QUERY_TEMPLATE % tbName
    queryCallback(query, dbId)

Here is the caller graph for this function:

Member Data Documentation

dBDataTask

dc_db.SiteCleanUpTask.SiteCleanUpTask.dBDataTask

Definition at line 45 of file SiteCleanUpTask.py.

errorCode

dc_db.SiteCleanUpTask.SiteCleanUpTask.errorCode

Definition at line 43 of file SiteCleanUpTask.py.

errorMessage

dc_db.SiteCleanUpTask.SiteCleanUpTask.errorMessage

Definition at line 44 of file SiteCleanUpTask.py.

fieldRecalculator

dc_db.SiteCleanUpTask.SiteCleanUpTask.fieldRecalculator

Definition at line 46 of file SiteCleanUpTask.py.

keyValueStorageDir

dc_db.SiteCleanUpTask.SiteCleanUpTask.keyValueStorageDir

Definition at line 41 of file SiteCleanUpTask.py.

rawDataDir

dc_db.SiteCleanUpTask.SiteCleanUpTask.rawDataDir

Definition at line 42 of file SiteCleanUpTask.py.

---

The documentation for this class was generated from the following file:

• sources/hce/dc_db/SiteCleanUpTask.py