dc_db.FieldRecalculator.FieldRecalculator Class Reference

Inheritance diagram for dc_db.FieldRecalculator.FieldRecalculator:

Collaboration diagram for dc_db.FieldRecalculator.FieldRecalculator:

Public Member Functions
def	__init__ (self)
def	commonSiteRecalculate (self, defaultCritName, fieldName, siteId, queryCallback)
def	siteResourcesRecalculate (self, siteId, queryCallback)
def	siteContentsRecalculate (self, siteId, queryCallback)
def	updateCollectedURLs (self, siteId, queryCallback)
def	updateNewURLs (self, siteId, queryCallback)
def	updateErrors (self, siteId, queryCallback)
def	updateDeletedURLs (self, siteId, queryCallback)
def	commonRecalc (self, siteId, queryCallback, recalcType=dc.EventObjects.FieldRecalculatorObj.FULL_RECALC)
def	updateSiteCleanupFields (self, siteId, queryCallback)

Detailed Description

Definition at line 19 of file FieldRecalculator.py.

Constructor & Destructor Documentation

__init__()

def dc_db.FieldRecalculator.FieldRecalculator.__init__

(

self

)

Definition at line 21 of file FieldRecalculator.py.

  def __init__(self):
    pass

Member Function Documentation

commonRecalc()

def dc_db.FieldRecalculator.FieldRecalculator.commonRecalc	(		self,
			siteId,
			queryCallback,
			recalcType = dc.EventObjects.FieldRecalculatorObj.FULL_RECALC
	)

Definition at line 111 of file FieldRecalculator.py.

  def commonRecalc(self, siteId, queryCallback, recalcType=dc.EventObjects.FieldRecalculatorObj.FULL_RECALC):
    self.siteResourcesRecalculate(siteId, queryCallback)
    self.siteContentsRecalculate(siteId, queryCallback)
    if recalcType == dc.EventObjects.FieldRecalculatorObj.FULL_RECALC:
      self.updateCollectedURLs(siteId, queryCallback)
    self.updateNewURLs(siteId, queryCallback)
    self.updateDeletedURLs(siteId, queryCallback)
    self.updateSiteCleanupFields(siteId, queryCallback)

Here is the call graph for this function:

commonSiteRecalculate()

def dc_db.FieldRecalculator.FieldRecalculator.commonSiteRecalculate	(		self,
			defaultCritName,
			fieldName,
			siteId,
			queryCallback
	)

Definition at line 31 of file FieldRecalculator.py.

  def commonSiteRecalculate(self, defaultCritName, fieldName, siteId, queryCallback):
    UPDATE_SQL_TEMPLATE = "UPDATE `sites` SET `%s`=(SELECT COUNT(*) FROM dc_urls.%s %s) WHERE `id` = '%s'"
    tableName = Constants.DC_URLS_TABLE_NAME_TEMPLATE % siteId
    criterionsString = DefCriterions.getDefaultCriterions(defaultCritName, siteId, queryCallback)
    query = UPDATE_SQL_TEMPLATE % (fieldName, tableName, criterionsString, siteId)
    queryCallback(query, Constants.PRIMARY_DB_ID)

Here is the caller graph for this function:

siteContentsRecalculate()

def dc_db.FieldRecalculator.FieldRecalculator.siteContentsRecalculate	(		self,
			siteId,
			queryCallback
	)

Definition at line 48 of file FieldRecalculator.py.

  def siteContentsRecalculate(self, siteId, queryCallback):
    # self.commonSiteRecalculate(queryCallback, "State=7 AND Crawled>0 AND Processed>0", "Contents", siteId)
    self.commonSiteRecalculate(DefCriterions.CRIT_CONTENTS, "Contents", siteId, queryCallback)

Here is the call graph for this function:

Here is the caller graph for this function:

siteResourcesRecalculate()

def dc_db.FieldRecalculator.FieldRecalculator.siteResourcesRecalculate	(		self,
			siteId,
			queryCallback
	)

Definition at line 40 of file FieldRecalculator.py.

  def siteResourcesRecalculate(self, siteId, queryCallback):
    # self.commonSiteRecalculate(queryCallback, "State>3 AND Crawled>0", "Resources", siteId)
    # self.commonSiteRecalculate("Crawled>0 AND Size>0", "Resources", siteId, queryCallback)
    self.commonSiteRecalculate(DefCriterions.CRIT_RESOURCES, "Resources", siteId, queryCallback)

Here is the call graph for this function:

Here is the caller graph for this function:

updateCollectedURLs()

def dc_db.FieldRecalculator.FieldRecalculator.updateCollectedURLs	(		self,
			siteId,
			queryCallback
	)

Definition at line 57 of file FieldRecalculator.py.

  def updateCollectedURLs(self, siteId, queryCallback):
    QUERY_TEMPLATE = "UPDATE `sites` SET `CollectedURLs`=(SELECT count(*) FROM dc_urls.%s %s) WHERE `Id`='%s'"
    tableName = Constants.DC_URLS_TABLE_NAME_TEMPLATE % siteId
    criterionsString = DefCriterions.getDefaultCriterions(DefCriterions.CRIT_CLURLS, siteId, queryCallback)
    query = QUERY_TEMPLATE % (tableName, criterionsString, siteId)
    queryCallback(query, Constants.PRIMARY_DB_ID)

Here is the caller graph for this function:

updateDeletedURLs()

def dc_db.FieldRecalculator.FieldRecalculator.updateDeletedURLs	(		self,
			siteId,
			queryCallback
	)

Definition at line 93 of file FieldRecalculator.py.

  def updateDeletedURLs(self, siteId, queryCallback):
    QUERY_TEMPLATE_SELECT = "SELECT count(*) FROM %s %s"
    tableName = Constants.DC_URLS_TABLE_NAME_TEMPLATE % siteId
    criterionsString = DefCriterions.getDefaultCriterions(DefCriterions.CRIT_DURLS, siteId, queryCallback)
    query = QUERY_TEMPLATE_SELECT % (tableName, criterionsString)
    res = queryCallback(query, Constants.FOURTH_DB_ID, Constants.EXEC_INDEX, True)
    if res is not None and len(res) > 0 and len(res[0]) > 0:
      count = res[0][0]
      QUERY_TEMPLATE_UPDATE = "UPDATE `sites` SET `DeletedURLs`=%s WHERE `Id`='%s'"
      query = QUERY_TEMPLATE_UPDATE % (str(count), siteId)
      queryCallback(query, Constants.PRIMARY_DB_ID)

Here is the caller graph for this function:

updateErrors()

def dc_db.FieldRecalculator.FieldRecalculator.updateErrors	(		self,
			siteId,
			queryCallback
	)

Definition at line 81 of file FieldRecalculator.py.

  def updateErrors(self, siteId, queryCallback):
    QUERY_TEMPLATE = "UPDATE `sites` SET `Errors`=(SELECT count(*) FROM dc_urls.%s %s) WHERE `Id`='%s'"
    tableName = Constants.DC_URLS_TABLE_NAME_TEMPLATE % siteId
    criterionsString = DefCriterions.getDefaultCriterions(DefCriterions.CRIT_ERRORS, siteId, queryCallback)
    query = QUERY_TEMPLATE % (tableName, criterionsString, siteId)
    queryCallback(query, Constants.PRIMARY_DB_ID)

updateNewURLs()

def dc_db.FieldRecalculator.FieldRecalculator.updateNewURLs	(		self,
			siteId,
			queryCallback
	)

Definition at line 69 of file FieldRecalculator.py.

  def updateNewURLs(self, siteId, queryCallback):
    QUERY_TEMPLATE = "UPDATE `sites` SET `NewURLs`=(SELECT count(*) FROM dc_urls.%s %s) WHERE `Id`='%s'"
    tableName = Constants.DC_URLS_TABLE_NAME_TEMPLATE % siteId
    criterionsString = DefCriterions.getDefaultCriterions(DefCriterions.CRIT_NURLS, siteId, queryCallback)
    query = QUERY_TEMPLATE % (tableName, criterionsString, siteId)
    queryCallback(query, Constants.PRIMARY_DB_ID)

Here is the caller graph for this function:

updateSiteCleanupFields()

def dc_db.FieldRecalculator.FieldRecalculator.updateSiteCleanupFields	(		self,
			siteId,
			queryCallback
	)

Definition at line 125 of file FieldRecalculator.py.

  def updateSiteCleanupFields(self, siteId, queryCallback):
    QUERY_TEMPLATE = "UPDATE `sites` SET `Size`=%s, `Errors`=%s, `ErrorMask`=%s, `AVGSpeed`=%s WHERE `Id`='%s'"
    tableName = Constants.DC_URLS_TABLE_NAME_TEMPLATE % siteId
    localSize = "`Size`"
    localErrors = "`Errors`"
    localErrorMask = "`ErrorMask`"
    localSpeed = "`AVGSpeed`"
    TMP_QUERY_TEMPLATE = "SELECT SUM(`Size`) FROM %s WHERE " + DefCriterions.CRIT_CRAWLED_THIS_NODE
    query = TMP_QUERY_TEMPLATE % tableName
    res = queryCallback(query, Constants.SECONDARY_DB_ID)
    if res is not None and len(res) > 0 and res[0] is not None and len(res[0]) > 0 and res[0][0] is not None:
      localSize = str(res[0][0])
    TMP_QUERY_TEMPLATE = "SELECT COUNT(*) FROM %s WHERE `errorMask` > 0 AND " + DefCriterions.CRIT_CRAWLED_THIS_NODE
    query = TMP_QUERY_TEMPLATE % tableName
    res = queryCallback(query, Constants.SECONDARY_DB_ID)
    if res is not None and len(res) > 0 and res[0] is not None and len(res[0]) > 0 and res[0][0] is not None:
      localErrors = str(res[0][0])
    TMP_QUERY_TEMPLATE = "SELECT BIT_OR(`errorMask`) FROM %s WHERE " + DefCriterions.CRIT_CRAWLED_THIS_NODE
    query = TMP_QUERY_TEMPLATE % tableName
    res = queryCallback(query, Constants.SECONDARY_DB_ID)
    if res is not None and len(res) > 0 and res[0] is not None and len(res[0]) > 0 and res[0][0] is not None:
      localErrorMask = str(res[0][0])
    TMP_QUERY_TEMPLATE = "SELECT AVG(`size`/`crawlingTime`*1000) FROM %s WHERE `crawlingTime` > 0 AND " + \
                         DefCriterions.CRIT_CRAWLED_THIS_NODE
    query = TMP_QUERY_TEMPLATE % tableName
    res = queryCallback(query, Constants.SECONDARY_DB_ID)
    if res is not None and len(res) > 0 and res[0] is not None and len(res[0]) > 0 and res[0][0] is not None:
      localSpeed = str(res[0][0])
    query = QUERY_TEMPLATE % (localSize, localErrors, localErrorMask, localSpeed, siteId)
    queryCallback(query, Constants.PRIMARY_DB_ID)

---

The documentation for this class was generated from the following file:

• sources/hce/dc_db/FieldRecalculator.py