dc_db.URLHistoryTask.URLHistoryTask Class Reference

Inheritance diagram for dc_db.URLHistoryTask.URLHistoryTask:

Collaboration diagram for dc_db.URLHistoryTask.URLHistoryTask:

Public Member Functions
def	__init__ (self, keyValueStorageDir, rawDataDir, dBDataTask)
def	process (self, urlHistories, queryCallback)
def	fetchLogsFromDB (self, urlHistory, queryCallback, logCriterions=None)
Public Member Functions inherited from dc_db.BaseTask.BaseTask
def	isSiteExist (self, siteId, queryCallback, userId=None)
def	generateCriterionSQL (self, criterions, additionWhere=None, siteId=None)
def	fetchByCriterions (self, criterions, queryCallback)
def	dbLock (self, mutexName, queryCallback, sleepTime=1, mutexLockTTL=Constants.DEFAULT_LOCK_TTL)
def	dbUnlock (self, mutexName, queryCallback)
def	createUrlsInsertQuery (self, siteId, localKeys, localValues)
def	copyUrlsToDcUrls (self, siteId, queryCallback)
def	statisticLogUpdate (self, localObj, urlMd5, siteId, status, queryCallback, isInsert=False)
def	calculateMd5FormUrl (self, url, urlType, useNormilize=False)

Public Attributes
	uRLCleanUpTask

Static Public Attributes
string	SQL_LOG_TEMPLATE = "SELECT * FROM %s WHERE `URLMd5`='%s'"
string	SQL_LOG_TEMPLATE_SHORT = "SELECT * FROM %s"

Additional Inherited Members
Static Public Member Functions inherited from dc_db.BaseTask.BaseTask
def	readValueFromSiteProp (siteId, propName, queryCallback, urlMd5=None)

Detailed Description

Definition at line 19 of file URLHistoryTask.py.

Constructor & Destructor Documentation

__init__()

def dc_db.URLHistoryTask.URLHistoryTask.__init__	(		self,
			keyValueStorageDir,
			rawDataDir,
			dBDataTask
	)

Definition at line 26 of file URLHistoryTask.py.

  def __init__(self, keyValueStorageDir, rawDataDir, dBDataTask):
    super(URLHistoryTask, self).__init__()
    self.uRLCleanUpTask = URLCleanUpTask(keyValueStorageDir, rawDataDir, dBDataTask)

Member Function Documentation

fetchLogsFromDB()

def dc_db.URLHistoryTask.URLHistoryTask.fetchLogsFromDB	(		self,
			urlHistory,
			queryCallback,
			logCriterions = None
	)

Definition at line 69 of file URLHistoryTask.py.

  def fetchLogsFromDB(self, urlHistory, queryCallback, logCriterions=None):
    tableName = Constants.DC_LOG_TABLE_NAME_TEMPLATE % urlHistory.siteId
    if logCriterions is None:
      query = self.SQL_LOG_TEMPLATE % (tableName, urlHistory.urlMd5)
    else:
      additionWere = "`URLMd5` = '%s'"
      additionWere = (additionWere % urlHistory.urlMd5)
      query = self.SQL_LOG_TEMPLATE_SHORT % tableName
      query += self.generateCriterionSQL(logCriterions, additionWere)
    ret = queryCallback(query, Constants.LOG_DB_ID, Constants.EXEC_NAME)
    if ret is not None:
      for elem in ret:
        if "CDate" in elem:
          elem["CDate"] = str(elem["CDate"])
        if "ODate" in elem:
          elem["ODate"] = str(elem["ODate"])
    return ret

process()

def dc_db.URLHistoryTask.URLHistoryTask.process	(		self,
			urlHistories,
			queryCallback
	)

Definition at line 36 of file URLHistoryTask.py.

  def process(self, urlHistories, queryCallback):
    uRLHistoryResponses = []
    for urlHistory in urlHistories:
      uRLHistoryResponse = None
      if urlHistory is not None:
        localMd5s = []
        if urlHistory.urlMd5 is None:
          if urlHistory.urlCriterions is not None:
            localMd5s = self.uRLCleanUpTask.extractUrlByCriterions(urlHistory.siteId, False,
                                                                      urlHistory.urlCriterions, queryCallback)
        else:
          localMd5s.append(urlHistory.urlMd5)
      logger.debug(">>> [URLHistoryTask] localUrls size = " + str(len(localMd5s)))
      for localMd5 in localMd5s:
        try:
          urlHistory.urlMd5 = localMd5
          res = self.fetchLogsFromDB(urlHistory, queryCallback, urlHistory.logCriterions)
          if uRLHistoryResponse is None:
            uRLHistoryResponse = dc.EventObjects.URLHistoryResponse([], urlHistory.siteId)
          if res is not None and len(res) > 0:
            uRLHistoryResponse.logRows.extend(res)
        except Exception as ex:
          logger.debug(">>> [URLHistoryTask] Some Type Exception = " + str(type(ex)) + " " + str(ex))
      uRLHistoryResponses.append(uRLHistoryResponse)
    return uRLHistoryResponses

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

SQL_LOG_TEMPLATE

string dc_db.URLHistoryTask.URLHistoryTask.SQL_LOG_TEMPLATE = "SELECT * FROM %s WHERE `URLMd5`='%s'"

static

Definition at line 21 of file URLHistoryTask.py.

SQL_LOG_TEMPLATE_SHORT

string dc_db.URLHistoryTask.URLHistoryTask.SQL_LOG_TEMPLATE_SHORT = "SELECT * FROM %s"

static

Definition at line 22 of file URLHistoryTask.py.

uRLCleanUpTask

dc_db.URLHistoryTask.URLHistoryTask.uRLCleanUpTask

Definition at line 28 of file URLHistoryTask.py.

---

The documentation for this class was generated from the following file:

• sources/hce/dc_db/URLHistoryTask.py