dc_db.URLStatsTask.URLStatsTask Class Reference

Inheritance diagram for dc_db.URLStatsTask.URLStatsTask:

Collaboration diagram for dc_db.URLStatsTask.URLStatsTask:

Public Member Functions
def	__init__ (self, keyValueStorageDir, rawDataDir, dBDataTask)
def	process (self, urlStatses, queryCallback)
def	fetchStatsFromDB (self, urlStats, queryCallback)
Public Member Functions inherited from dc_db.BaseTask.BaseTask
def	isSiteExist (self, siteId, queryCallback, userId=None)
def	generateCriterionSQL (self, criterions, additionWhere=None, siteId=None)
def	fetchByCriterions (self, criterions, queryCallback)
def	dbLock (self, mutexName, queryCallback, sleepTime=1, mutexLockTTL=Constants.DEFAULT_LOCK_TTL)
def	dbUnlock (self, mutexName, queryCallback)
def	createUrlsInsertQuery (self, siteId, localKeys, localValues)
def	copyUrlsToDcUrls (self, siteId, queryCallback)
def	statisticLogUpdate (self, localObj, urlMd5, siteId, status, queryCallback, isInsert=False)
def	calculateMd5FormUrl (self, url, urlType, useNormilize=False)

Public Attributes
	uRLCleanUpTask

Static Public Attributes
string	SQL_STATS_TEMPLATE = "SELECT * FROM %s WHERE `URLMd5`='%s'"

Additional Inherited Members
Static Public Member Functions inherited from dc_db.BaseTask.BaseTask
def	readValueFromSiteProp (siteId, propName, queryCallback, urlMd5=None)

Detailed Description

Definition at line 20 of file URLStatsTask.py.

Constructor & Destructor Documentation

__init__()

def dc_db.URLStatsTask.URLStatsTask.__init__	(		self,
			keyValueStorageDir,
			rawDataDir,
			dBDataTask
	)

Definition at line 26 of file URLStatsTask.py.

  def __init__(self, keyValueStorageDir, rawDataDir, dBDataTask):
    super(URLStatsTask, self).__init__()
    self.uRLCleanUpTask = URLCleanUpTask(keyValueStorageDir, rawDataDir, dBDataTask)

Member Function Documentation

fetchStatsFromDB()

def dc_db.URLStatsTask.URLStatsTask.fetchStatsFromDB	(		self,
			urlStats,
			queryCallback
	)

Definition at line 112 of file URLStatsTask.py.

  def fetchStatsFromDB(self, urlStats, queryCallback):
    tableName = Constants.DC_FREQ_TABLE_NAME_TEMPLATE % urlStats.siteId
    query = self.SQL_STATS_TEMPLATE % (tableName, urlStats.urlMd5)
    ret = queryCallback(query, Constants.STAT_DB_ID, Constants.EXEC_NAME)
    if ret is not None:
      for elem in ret:
        if "CDate" in elem:
          elem["CDate"] = str(elem["CDate"])
        if "MDate" in elem:
          elem["MDate"] = str(elem["MDate"])
    return ret

process()

def dc_db.URLStatsTask.URLStatsTask.process	(		self,
			urlStatses,
			queryCallback
	)

Definition at line 36 of file URLStatsTask.py.

  def process(self, urlStatses, queryCallback):
    uRLStatsResponses = []
    for urlStats in urlStatses:
      uRLStatsResponse = None
      localMd5s = []
      if urlStats is not None:
        if urlStats.urlMd5 is None:
          if urlStats.urlCriterions is not None and len(urlStats.urlCriterions) > 0:
            localMd5s = self.uRLCleanUpTask.extractUrlByCriterions(urlStats.siteId, False,
                                                                   urlStats.urlCriterions, queryCallback)
          if urlStats.statsCriterions is not None:
            statsMd5s = []
            if urlStats.urlCriterions is None or len(urlStats.urlCriterions) == 0:
              statsMd5s = self.uRLCleanUpTask.extractUrlByCriterions(urlStats.siteId,
                                                                     False,
                                                                     urlStats.statsCriterions, queryCallback,
                                                                     Constants.STAT_DB_ID,
                                                                     Constants.DC_FREQ_TABLE_NAME_TEMPLATE)
            else:
              SQL_WHERE_TMPL = "`UrlMd5` = '%s'"
              statsCriterionCopy = copy.deepcopy(urlStats.statsCriterions)
              for localMd5 in localMd5s:
                urlStats.statsCriterions = copy.deepcopy(statsCriterionCopy)
                if dc.EventObjects.URLFetch.CRITERION_WHERE in urlStats.statsCriterions \
                and urlStats.statsCriterions[dc.EventObjects.URLFetch.CRITERION_WHERE] is not None:
                  urlStats.statsCriterions[dc.EventObjects.URLFetch.CRITERION_WHERE] = ' AND ' + \
                  (SQL_WHERE_TMPL % localMd5)
                else:
                  urlStats.statsCriterions[dc.EventObjects.URLFetch.CRITERION_WHERE] = (SQL_WHERE_TMPL % localMd5)
                statsMd5s += self.uRLCleanUpTask.extractUrlByCriterions(urlStats.siteId,
                                                                        False,
                                                                        urlStats.statsCriterions,
                                                                        queryCallback,
                                                                        Constants.STAT_DB_ID,
                                                                        Constants.DC_FREQ_TABLE_NAME_TEMPLATE)
            localMd5s = statsMd5s

#           '''
#           if urlStats.urlCriterions is not None:
#             urlsMd5s = self.uRLCleanUpTask.extractUrlByCriterions(urlStats.siteId, False,
#                         urlStats.urlCriterions, queryCallback)
#
#           if urlStats.statsCriterions is not None:
#             statsMd5s = self.uRLCleanUpTask.extractUrlByCriterions(urlStats.siteId, False,
#                         urlStats.statsCriterions, queryCallback, Constants.STAT_DB_ID,
#                         Constants.DC_FREQ_TABLE_NAME_TEMPLATE)
#           if len(statsMd5s) > 0 and len(urlsMd5s) > 0:
#             localMd5s = [x for x in statsMd5s if x in urlsMd5s]
#           elif len(statsMd5s) > 0:
#             localMd5s = statsMd5s
#           elif len(urlsMd5s) > 0:
#             localMd5s = urlsMd5s
#           '''
        else:
          localMd5s.append(urlStats.urlMd5)
      logger.debug(">>> [URLStatsTask] localUrls size = " + str(len(localMd5s)))
      for localMd5 in localMd5s:
        try:
          urlStats.urlMd5 = localMd5
          res = self.fetchStatsFromDB(urlStats, queryCallback)
          if uRLStatsResponse is None:
            uRLStatsResponse = dc.EventObjects.URLStatsResponse([], urlStats.siteId)
          if res is not None and len(res) > 0:
            uRLStatsResponse.freqRows.extend(res)
        except Exception as ex:
          logger.debug(">>> [URLStatsTask] Some Type Exception = " + str(type(ex)) + " " + str(ex))
      uRLStatsResponses.append(uRLStatsResponse)

    return uRLStatsResponses

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

SQL_STATS_TEMPLATE

string dc_db.URLStatsTask.URLStatsTask.SQL_STATS_TEMPLATE = "SELECT * FROM %s WHERE `URLMd5`='%s'"

static

Definition at line 22 of file URLStatsTask.py.

uRLCleanUpTask

dc_db.URLStatsTask.URLStatsTask.uRLCleanUpTask

Definition at line 28 of file URLStatsTask.py.

---

The documentation for this class was generated from the following file:

• sources/hce/dc_db/URLStatsTask.py