dc_db.URLAgeTask.URLAgeTask Class Reference

Inheritance diagram for dc_db.URLAgeTask.URLAgeTask:

Collaboration diagram for dc_db.URLAgeTask.URLAgeTask:

Public Member Functions
def	__init__ (self, keyValueStorageDir, rawDataDir, backDBResolve)
def	urlDeleteOperation (self, queryCallback)
def	addElemInLocalDict (self, siteId, UrlMd5, urlLimit, delayedType)
def	process (self, urlAges, queryCallback)
Public Member Functions inherited from dc_db.BaseTask.BaseTask
def	isSiteExist (self, siteId, queryCallback, userId=None)
def	generateCriterionSQL (self, criterions, additionWhere=None, siteId=None)
def	fetchByCriterions (self, criterions, queryCallback)
def	dbLock (self, mutexName, queryCallback, sleepTime=1, mutexLockTTL=Constants.DEFAULT_LOCK_TTL)
def	dbUnlock (self, mutexName, queryCallback)
def	createUrlsInsertQuery (self, siteId, localKeys, localValues)
def	copyUrlsToDcUrls (self, siteId, queryCallback)
def	statisticLogUpdate (self, localObj, urlMd5, siteId, status, queryCallback, isInsert=False)
def	calculateMd5FormUrl (self, url, urlType, useNormilize=False)

Public Attributes
	uRLDeleteTask
	urlsSelectDict
	gloablLoopExit
	curUrlsCount

Additional Inherited Members
Static Public Member Functions inherited from dc_db.BaseTask.BaseTask
def	readValueFromSiteProp (siteId, propName, queryCallback, urlMd5=None)

Detailed Description

Definition at line 21 of file URLAgeTask.py.

Constructor & Destructor Documentation

__init__()

def dc_db.URLAgeTask.URLAgeTask.__init__	(		self,
			keyValueStorageDir,
			rawDataDir,
			backDBResolve
	)

Definition at line 28 of file URLAgeTask.py.

  def __init__(self, keyValueStorageDir, rawDataDir, backDBResolve):
    super(URLAgeTask, self).__init__()
    self.uRLDeleteTask = URLDeleteTask(keyValueStorageDir, rawDataDir, backDBResolve)
    self.urlsSelectDict = {}
    self.gloablLoopExit = False
    self.curUrlsCount = 0

Member Function Documentation

addElemInLocalDict()

def dc_db.URLAgeTask.URLAgeTask.addElemInLocalDict	(		self,
			siteId,
			UrlMd5,
			urlLimit,
			delayedType
	)

Definition at line 61 of file URLAgeTask.py.

  def addElemInLocalDict(self, siteId, UrlMd5, urlLimit, delayedType):
    if siteId not in self.urlsSelectDict:
      self.urlsSelectDict[siteId] = {}
    if UrlMd5 in self.urlsSelectDict[siteId]:
      logger.debug(">>> " + siteId + "." + UrlMd5 + " Already selected")
    else:
      if self.curUrlsCount < urlLimit:
        self.urlsSelectDict[siteId][UrlMd5] = delayedType
        logger.debug(">>> " + siteId + "." + UrlMd5 + " Added")
        self.curUrlsCount += 1
      else:
        logger.debug(">>> UrlLimit reached = " + str(urlLimit))

Here is the caller graph for this function:

process()

def dc_db.URLAgeTask.URLAgeTask.process	(		self,
			urlAges,
			queryCallback
	)

Definition at line 80 of file URLAgeTask.py.

  def process(self, urlAges, queryCallback):
    self.curUrlsCount = 0
    generalResponse = GeneralResponse()
    self.urlsSelectDict = {}
    URL_SELECT_TEMPL = "SELECT `UrlMd5` FROM %s"
    for urlAge in urlAges:
      defaultUrlsCriterions = urlAge.urlsCriterions[dc.EventObjects.URLAge.CRITERION_WHERE]
      if self.gloablLoopExit:
        break
      query = "SELECT `Id` FROM `sites`"
      sitesCriterionStr = self.generateCriterionSQL(urlAge.sitesCriterions)
      if len(sitesCriterionStr) > 0:
        query += " " + sitesCriterionStr
      sitesRes = queryCallback(query, Constants.PRIMARY_DB_ID)
      if sitesRes is not None:
        for sitesElem in sitesRes:
          if self.gloablLoopExit:
            break
          if sitesElem is not None and len(sitesElem) > 0:
            # StatisticLogManager.logUpdate(queryCallback, "LOG_URL_AGING", urlAge, sitesElem[0], "")
            # Get the alternate URLs select criterion from the sites_properties table
            queryAltURLsCrit = \
              "SELECT `Value` FROM `sites_properties` WHERE `Site_Id`='%s' AND `Name`='AGING_URL_CRITERION' LIMIT 1"\
              % sitesElem[0]
            altURLsCritRes = queryCallback(queryAltURLsCrit, Constants.PRIMARY_DB_ID)
            criterionsSubstituted = False
            if altURLsCritRes is not None:
              for altURLsCritItem in altURLsCritRes:
                if altURLsCritItem is not None and len(altURLsCritItem) > 0:
                  # Overwrite criterion WHERE with value from dc_sites.sites_properties
                  urlAge.urlsCriterions[dc.EventObjects.URLAge.CRITERION_WHERE] = altURLsCritItem[0]
                  criterionsSubstituted = True
            if not criterionsSubstituted:
              urlAge.urlsCriterions[dc.EventObjects.URLAge.CRITERION_WHERE] = defaultUrlsCriterions
            # Make criterion for URLs select
            tableName = Constants.DC_URLS_TABLE_NAME_TEMPLATE % sitesElem[0]
            query = URL_SELECT_TEMPL % tableName
            urlsCriterionStr = self.generateCriterionSQL(urlAge.urlsCriterions, None, sitesElem[0])
            if len(urlsCriterionStr) > 0:
              query += " " + urlsCriterionStr
            # Select URLs
            urlsRes = queryCallback(query, Constants.SECONDARY_DB_ID)
            for urlsRes in urlsRes:
              if self.gloablLoopExit:
                break
              if urlsRes is not None and len(urlsRes) > 0:
                self.addElemInLocalDict(sitesElem[0], urlsRes[0], urlAge.maxURLs, urlAge.delayedType)
                StatisticLogManager.logUpdate(queryCallback, "LOG_URL_AGING", urlAge, sitesElem[0], urlsRes[0])
    if len(self.urlsSelectDict) > 0:
      self.urlDeleteOperation(queryCallback)
    return generalResponse

urlDeleteOperation()

def dc_db.URLAgeTask.URLAgeTask.urlDeleteOperation	(		self,
			queryCallback
	)

Definition at line 39 of file URLAgeTask.py.

  def urlDeleteOperation(self, queryCallback):
    urlsDeleteObjs = []
    localUrlDelete = None
    for siteId in self.urlsSelectDict:
      for urlMd5 in self.urlsSelectDict[siteId]:
        localUrlDelete = dc.EventObjects.URLDelete(siteId, urlMd5, reason=dc.EventObjects.URLDelete.REASON_AGING)
        localUrlDelete.urlType = dc.EventObjects.URLStatus.URL_TYPE_MD5
        localUrlDelete.delayedType = self.urlsSelectDict[siteId][urlMd5]
        urlsDeleteObjs.append(localUrlDelete)
        StatisticLogManager.statisticUpdate(queryCallback, Constants.StatFreqConstants.FREQ_AGED_STATE, 1,
                                            siteId, urlMd5)
    if len(urlsDeleteObjs) > 0:
      logger.debug(">>> URLAge started URLDelete count = " + str(len(urlsDeleteObjs)))
      self.uRLDeleteTask.process(urlsDeleteObjs, queryCallback)

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

curUrlsCount

dc_db.URLAgeTask.URLAgeTask.curUrlsCount

Definition at line 33 of file URLAgeTask.py.

gloablLoopExit

dc_db.URLAgeTask.URLAgeTask.gloablLoopExit

Definition at line 32 of file URLAgeTask.py.

uRLDeleteTask

dc_db.URLAgeTask.URLAgeTask.uRLDeleteTask

Definition at line 30 of file URLAgeTask.py.

urlsSelectDict

dc_db.URLAgeTask.URLAgeTask.urlsSelectDict

Definition at line 31 of file URLAgeTask.py.

---

The documentation for this class was generated from the following file:

• sources/hce/dc_db/URLAgeTask.py