dc_db.URLUpdateTask.URLUpdateTask Class Reference

Inheritance diagram for dc_db.URLUpdateTask.URLUpdateTask:

Collaboration diagram for dc_db.URLUpdateTask.URLUpdateTask:

Public Member Functions
def	__init__ (self, keyValueStorageDir, rawDataDir, dBDataTask)
def	process (self, urlUpdates, queryCallback)
def	statisticUpdate (self, urlUpdate, queryCallback)
def	updateURL (self, urlUpdate, queryCallback)
def	urlPutOperation (self, urlObject, urlPutObject, queryCallback)
def	attributesUpdate (self, attributes, queryCallback)
Public Member Functions inherited from dc_db.BaseTask.BaseTask
def	isSiteExist (self, siteId, queryCallback, userId=None)
def	generateCriterionSQL (self, criterions, additionWhere=None, siteId=None)
def	fetchByCriterions (self, criterions, queryCallback)
def	dbLock (self, mutexName, queryCallback, sleepTime=1, mutexLockTTL=Constants.DEFAULT_LOCK_TTL)
def	dbUnlock (self, mutexName, queryCallback)
def	createUrlsInsertQuery (self, siteId, localKeys, localValues)
def	copyUrlsToDcUrls (self, siteId, queryCallback)
def	statisticLogUpdate (self, localObj, urlMd5, siteId, status, queryCallback, isInsert=False)
def	calculateMd5FormUrl (self, url, urlType, useNormilize=False)

Public Attributes
	urlPutTask

Additional Inherited Members
Static Public Member Functions inherited from dc_db.BaseTask.BaseTask
def	readValueFromSiteProp (siteId, propName, queryCallback, urlMd5=None)

Detailed Description

Definition at line 21 of file URLUpdateTask.py.

Constructor & Destructor Documentation

__init__()

def dc_db.URLUpdateTask.URLUpdateTask.__init__	(		self,
			keyValueStorageDir,
			rawDataDir,
			dBDataTask
	)

Definition at line 25 of file URLUpdateTask.py.

  def __init__(self, keyValueStorageDir, rawDataDir, dBDataTask):
    super(URLUpdateTask, self).__init__()
    self.urlPutTask = URLPutTask(keyValueStorageDir, rawDataDir, dBDataTask)

Member Function Documentation

attributesUpdate()

def dc_db.URLUpdateTask.URLUpdateTask.attributesUpdate	(		self,
			attributes,
			queryCallback
	)

Definition at line 120 of file URLUpdateTask.py.

  def attributesUpdate(self, attributes, queryCallback):
    logger.debug(">>> URLUpdateTask.attributesUpdate (len) = " + str(len(attributes)))
    attrUpdateTask = AttrUpdateTask()
    res = attrUpdateTask.process(attributes, queryCallback)
    logger.debug(">>> URLUpdateTask.attributesUpdate (res) == " + str(res))

process()

def dc_db.URLUpdateTask.URLUpdateTask.process	(		self,
			urlUpdates,
			queryCallback
	)

Definition at line 35 of file URLUpdateTask.py.

  def process(self, urlUpdates, queryCallback):
    ret = GeneralResponse()
    status = False
    for urlUpdate in urlUpdates:
      status = False
      if urlUpdate.siteId == "":
        urlUpdate.siteId = "0"
      if not hasattr(urlUpdate, "urlMd5"):
        urlUpdate.fillMD5(urlUpdate.url, urlUpdate.type)
      if self.isSiteExist(urlUpdate.siteId, queryCallback):
        self.statisticUpdate(urlUpdate, queryCallback)
        status = self.updateURL(urlUpdate, queryCallback)
        if status and urlUpdate.attributes is not None and len(urlUpdate.attributes) > 0:
          self.attributesUpdate(urlUpdate.attributes, queryCallback)
      ret.statuses.append(status)
      if "urlPut" in urlUpdate.__dict__ and urlUpdate.urlPut is not None:
        self.urlPutOperation(urlUpdate, urlUpdate.urlPut, queryCallback)
    return ret

Here is the call graph for this function:

Here is the caller graph for this function:

statisticUpdate()

def dc_db.URLUpdateTask.URLUpdateTask.statisticUpdate	(		self,
			urlUpdate,
			queryCallback
	)

Definition at line 59 of file URLUpdateTask.py.

  def statisticUpdate(self, urlUpdate, queryCallback):
    prevStatus = None
    SQL_SELECT_STATUS_TEMPLATE = "SELECT `Status` FROM `%s` WHERE `URLMD5` = '%s'"
    tableName = Constants.DC_URLS_TABLE_NAME_TEMPLATE % urlUpdate.siteId
    query = SQL_SELECT_STATUS_TEMPLATE % (tableName, urlUpdate.urlMd5)
    ret = queryCallback(query, Constants.SECONDARY_DB_ID)
    if ret is not None and len(ret) > 0 and len(ret[0]) > 0 and ret[0][0] is not None:
      prevStatus = int(ret[0][0])
    StatisticLogManager.statisticUpdate(queryCallback, Constants.StatFreqConstants.FREQ_UPDATE, 1,
                                        urlUpdate.siteId, urlUpdate.urlMd5)
    StatisticLogManager.logUpdate(queryCallback, "LOG_UPDATE", urlUpdate, urlUpdate.siteId, urlUpdate.urlMd5)
    if prevStatus is None or prevStatus != urlUpdate.status:
      self.statisticLogUpdate(urlUpdate, urlUpdate.urlMd5, urlUpdate.siteId, urlUpdate.status, queryCallback)

Here is the call graph for this function:

Here is the caller graph for this function:

updateURL()

def dc_db.URLUpdateTask.URLUpdateTask.updateURL	(		self,
			urlUpdate,
			queryCallback
	)

Definition at line 78 of file URLUpdateTask.py.

  def updateURL(self, urlUpdate, queryCallback):
    ret = False
    SQL_UPDATE_URLSITE_TEMPLATE = "UPDATE IGNORE `%s` SET %s"
    if urlUpdate.eTag is not None:
      urlUpdate.eTag = urlUpdate.eTag.strip("\"'")
    fields, values = Constants.getFieldsValuesTuple(urlUpdate, Constants.URLTableDict)
    fieldValueString = Constants.createFieldsValuesString(fields, values, Constants.urlExcludeList)
    if fieldValueString and len(fieldValueString) > 0:
      tableName = Constants.DC_URLS_TABLE_NAME_TEMPLATE % urlUpdate.siteId
      query = SQL_UPDATE_URLSITE_TEMPLATE % (tableName, fieldValueString)
      additionWhere = None
      if urlUpdate.urlMd5 is not None:
        additionWhere = ("`URLMD5` = '%s'" % urlUpdate.urlMd5)
      additionQueryStr = self.generateCriterionSQL(urlUpdate.criterions, additionWhere)
      if len(additionQueryStr) > 0:
        query += " "
        query += additionQueryStr
      queryCallback(query, Constants.SECONDARY_DB_ID)
      ret = True
    return ret

Here is the call graph for this function:

Here is the caller graph for this function:

urlPutOperation()

def dc_db.URLUpdateTask.URLUpdateTask.urlPutOperation	(		self,
			urlObject,
			urlPutObject,
			queryCallback
	)

Definition at line 105 of file URLUpdateTask.py.

  def urlPutOperation(self, urlObject, urlPutObject, queryCallback):
    if urlPutObject.siteId is None and urlObject.siteId is not None:
      urlPutObject.siteId = urlObject.siteId
      logger.debug(">>> URLPut.siteId is None and set to the = " + urlPutObject.siteId)
    if urlPutObject.urlMd5 is None and urlObject.urlMd5 is not None:
      urlPutObject.urlMd5 = urlObject.urlMd5
      logger.debug(">>> URLPut.urlMd5 is None and set to the = " + urlPutObject.urlMd5)
    logger.debug(">>> Call internal URLPut")
    self.urlPutTask.process([urlPutObject], queryCallback)

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

urlPutTask

dc_db.URLUpdateTask.URLUpdateTask.urlPutTask

Definition at line 27 of file URLUpdateTask.py.

---

The documentation for this class was generated from the following file:

• sources/hce/dc_db/URLUpdateTask.py