Inheritance diagram for dc_db.SiteUpdateTask.SiteUpdateTask:

Collaboration diagram for dc_db.SiteUpdateTask.SiteUpdateTask:

Public Member Functions
def	__init__ (self, dcSiteTemplate, keyValueDefaultFile, keyValueStorageDir, rawDataDir, dBDataTask, dcStatTemplate, dcLogTemplate, dcAttrTemplate)

def	process (self, siteUpdate, queryCallback)

def	updateSite (self, siteUpdate, queryCallback)

def	updateSitesFilter (self, siteUpdate, queryCallback)

def	updateSiteProperties (self, siteUpdate, queryCallback)

def	updateSiteURL (self, siteUpdate, queryCallback)

def	updateURlsURL (self, urls, siteId, queryCallback)

def	restartSite (self, siteUpdate, queryCallback)

def	deleteRootUrlsFromURLsUrls (self, siteId, queryCallback)

def	copyURLsFromSiteURlsToURLsURLs (self, siteId, queryCallback)

Public Member Functions inherited from dc_db.SiteTask.SiteTask
def	__init__ (self, dcSiteTemplate, keyValueDefaultFile, keyValueStorageDir, dBDataTask, dcStatTemplates, dcLogTemplate, dcAttrTemplate, tasksManager=None)

def	process (self, site, queryCallback)

def	addSite (self, site, queryCallback)

def	addSitesFilter (self, site, queryCallback)

def	updateSitesFilter (self, site, queryCallback)

def	createPropDict (self, field, site)

def	isIsoFormatDate (self, dateValue)

def	addSiteProperties (self, site, queryCallback)

def	updateSiteProperties (self, site, queryCallback)

def	addSiteURLSites (self, site, queryCallback)

def	updateSiteURLSites (self, site, queryCallback)

def	addSiteURLURLs (self, site, queryCallback)

def	createTableFromTemplate (self, site, template, dbId, queryCallback, replaceDic=None)

def	addSiteInKVDB (self, site, queryCallback)

def	siteDelete (self, site, queryCallback)

Public Member Functions inherited from dc_db.BaseTask.BaseTask
def	isSiteExist (self, siteId, queryCallback, userId=None)

def	generateCriterionSQL (self, criterions, additionWhere=None, siteId=None)

def	fetchByCriterions (self, criterions, queryCallback)

def	dbLock (self, mutexName, queryCallback, sleepTime=1, mutexLockTTL=Constants.DEFAULT_LOCK_TTL)

def	dbUnlock (self, mutexName, queryCallback)

def	createUrlsInsertQuery (self, siteId, localKeys, localValues)

def	copyUrlsToDcUrls (self, siteId, queryCallback)

def	statisticLogUpdate (self, localObj, urlMd5, siteId, status, queryCallback, isInsert=False)

def	calculateMd5FormUrl (self, url, urlType, useNormilize=False)

Public Attributes
	urlDeleteTask

	fieldRecalculator

Public Attributes inherited from dc_db.SiteTask.SiteTask
	dcSiteTemplate

	keyValueDefaultFile

	keyValueStorageDir

	dBDataTask

	dcStatTemplates

	dcLogTemplate

	dcAttrTemplate

	siteDeleteTask

	tasksManager

Additional Inherited Members
Static Public Member Functions inherited from dc_db.SiteTask.SiteTask
def	execSiteCriterions (criterions, queryCallback)

Static Public Member Functions inherited from dc_db.BaseTask.BaseTask
def	readValueFromSiteProp (siteId, propName, queryCallback, urlMd5=None)

Static Public Attributes inherited from dc_db.SiteTask.SiteTask
string	FIELD_NAME_URLS = "urls"

string	FIELD_NAME_PROPERTIES = "properties"

string	FIELD_NAME_FILTERS = "filters"

Detailed Description

Definition at line 32 of file SiteUpdateTask.py.

Constructor & Destructor Documentation

◆ init()

def dc_db.SiteUpdateTask.SiteUpdateTask.__init__	(	self,
		dcSiteTemplate,
		keyValueDefaultFile,
		keyValueStorageDir,
		rawDataDir,
		dBDataTask,
		dcStatTemplate,
		dcLogTemplate,
		dcAttrTemplate
	)

Definition at line 39 of file SiteUpdateTask.py.

                dcLogTemplate, dcAttrTemplate):
     super(SiteUpdateTask, self).__init__(dcSiteTemplate, keyValueDefaultFile, keyValueStorageDir, dBDataTask,
                                          dcStatTemplate, dcLogTemplate, dcAttrTemplate)
     self.urlDeleteTask = URLDeleteTask(keyValueStorageDir, rawDataDir, dBDataTask)
     self.fieldRecalculator = FieldRecalculator()
 
 

Member Function Documentation

◆ copyURLsFromSiteURlsToURLsURLs()

def dc_db.SiteUpdateTask.SiteUpdateTask.copyURLsFromSiteURlsToURLsURLs	(	self,
		siteId,
		queryCallback
	)

Definition at line 243 of file SiteUpdateTask.py.

   def copyURLsFromSiteURlsToURLsURLs(self, siteId, queryCallback):
     if len(Constants.URLTableDict) > 0:
       tbName = Constants.DC_URLS_TABLE_NAME_TEMPLATE % siteId
       query = "INSERT IGNORE INTO `%s` ("
       query = query % tbName
       fields = ''
       for elem in Constants.URLTableDict.values():
         fields += elem
         fields += ','
 
       fields = fields[:-1]
       query += fields
       query += (") SELECT ")
       query += fields
       query += (" FROM dc_sites.sites_urls WHERE `Site_Id`='%s' GROUP BY `URLMd5`" % siteId)
 
       queryCallback(query, Constants.SECONDARY_DB_ID)
 

Here is the caller graph for this function:

◆ deleteRootUrlsFromURLsUrls()

def dc_db.SiteUpdateTask.SiteUpdateTask.deleteRootUrlsFromURLsUrls	(	self,
		siteId,
		queryCallback
	)

Definition at line 228 of file SiteUpdateTask.py.

   def deleteRootUrlsFromURLsUrls(self, siteId, queryCallback):
     logger.debug("Delete all root URLs from dc_urls.urls_* table")
     urlDeleteObj = dc.EventObjects.URLDelete(siteId, None,
                                              reason=dc.EventObjects.URLDelete.REASON_SITE_UPDATE_ROOT_URLS)
     # whereCriterionStr = \
     #      "`URLMd5` IN (SELECT `URLMd5` FROM dc_sites.sites_urls WHERE `Site_Id` = '%s')" % siteId
     whereCriterionStr = "`ParentMd5`=''"
     urlDeleteObj.criterions = {app.SQLCriterions.CRITERION_WHERE : whereCriterionStr}
     self.urlDeleteTask.process([urlDeleteObj], queryCallback)
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ process()

def dc_db.SiteUpdateTask.SiteUpdateTask.process	(	self,
		siteUpdate,
		queryCallback
	)

Definition at line 51 of file SiteUpdateTask.py.

   def process(self, siteUpdate, queryCallback):
     siteIds = []
     response = GeneralResponse()
     if siteUpdate.id is None:
       siteIds.extend(self.fetchByCriterions(siteUpdate.criterions, queryCallback))
     else:
       siteIds.append(siteUpdate.id)
 
     for siteId in siteIds:
       siteUpdate.id = siteId
       if super(SiteUpdateTask, self).isSiteExist(siteUpdate.id, queryCallback):
         self.updateSite(siteUpdate, queryCallback)
         self.updateSitesFilter(siteUpdate, queryCallback)
         self.updateSiteProperties(siteUpdate, queryCallback)
         sitePropValue = BaseTask.readValueFromSiteProp(siteUpdate.id, "UPDATE_NOT_INSERT_ROOT_URLS", queryCallback)
         if siteUpdate.updateType != dc.EventObjects.SiteUpdate.UPDATE_TYPE_UPDATE and\
           (sitePropValue is None or not bool(int(sitePropValue))) and siteUpdate.urls is not None:
           self.deleteRootUrlsFromURLsUrls(siteUpdate.id, queryCallback)
         elif siteUpdate.updateType == dc.EventObjects.SiteUpdate.UPDATE_TYPE_UPDATE and siteUpdate.urls is not None:
           self.updateURlsURL(siteUpdate.urls, siteId, queryCallback)
         self.updateSiteURL(siteUpdate, queryCallback)
         self.restartSite(siteUpdate, queryCallback)
         if siteUpdate.updateType != dc.EventObjects.SiteUpdate.UPDATE_TYPE_UPDATE and\
           (sitePropValue is None or not bool(int(sitePropValue))) and siteUpdate.urls is not None:
           logger.debug("Copy root URLs from dc_sites.site_urls table to dc_urls.urls_*")
           self.copyURLsFromSiteURlsToURLsURLs(siteUpdate.id, queryCallback)
         self.fieldRecalculator.updateSiteCleanupFields(siteId, queryCallback)
         response.statuses.append(GeneralResponse.ERROR_OK)
       else:
         response.statuses.append(TASK_NOT_EXIST_ERR)
 
     return response
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ restartSite()

def dc_db.SiteUpdateTask.SiteUpdateTask.restartSite	(	self,
		siteUpdate,
		queryCallback
	)

Definition at line 207 of file SiteUpdateTask.py.

   def restartSite(self, siteUpdate, queryCallback):
     if siteUpdate.state == siteUpdate.STATE_RESTART:
       UPDATE_SITE_SQL_QUERY = ("UPDATE `sites` SET `State`=%s, `UDate`=NOW(), " +
                                "`Iterations`=`Iterations`+1, `ErrorMask`=0, `Errors`=0 WHERE Id='%s'")
       query = UPDATE_SITE_SQL_QUERY % (str(dc.EventObjects.Site.STATE_ACTIVE), siteUpdate.id)
       queryCallback(query, Constants.PRIMARY_DB_ID)
       # TODO: need to remove the URLs from urls_ and then insert from sites_urls
       # siteUpdate.urls = []
       # SELECT_QUERY = "SELECT `URL` FROM `sites_urls` WHERE `Site_Id` = '%s'"
       # query = SELECT_QUERY % siteUpdate.id
       # res = queryCallback(query, Constants.PRIMARY_DB_ID)
       # if hasattr(res, '__iter__'):
       #  for row in res:
       #    siteUpdate.urls.append(dc.EventObjects.URL(siteUpdate.id, row[0]))
       # super(SiteUpdateTask, self).addSiteURLURLs(siteUpdate, queryCallback)
 
 

Here is the caller graph for this function:

◆ updateSite()

def dc_db.SiteUpdateTask.SiteUpdateTask.updateSite	(	self,
		siteUpdate,
		queryCallback
	)

Definition at line 89 of file SiteUpdateTask.py.

   def updateSite(self, siteUpdate, queryCallback):
     SQL_UPDATE_SITE_SQL_TEMPLATE = "UPDATE sites SET %s WHERE `Id` = '%s'"
     fields, values = Constants.getFieldsValuesTuple(siteUpdate, Constants.siteDict)
     fieldValueString = Constants.createFieldsValuesString(fields, values, Constants.siteExcludeList)
     query = SQL_UPDATE_SITE_SQL_TEMPLATE % (fieldValueString, siteUpdate.id)
     logger.debug('>>> updateSite  query: ' + str(query))
     queryCallback(query, Constants.PRIMARY_DB_ID)
 
 

Here is the caller graph for this function:

◆ updateSiteProperties()

def dc_db.SiteUpdateTask.SiteUpdateTask.updateSiteProperties	(	self,
		siteUpdate,
		queryCallback
	)

Definition at line 118 of file SiteUpdateTask.py.

   def updateSiteProperties(self, siteUpdate, queryCallback):
     if siteUpdate.properties is not None:
       if siteUpdate.updateType == dc.EventObjects.SiteUpdate.UPDATE_TYPE_APPEND:
         super(SiteUpdateTask, self).addSiteProperties(siteUpdate, queryCallback)
       if siteUpdate.updateType == dc.EventObjects.SiteUpdate.UPDATE_TYPE_OVERWRITE:
         query = Constants.DEL_BY_ID_QUERY_TEMPLATE % ("sites_properties", siteUpdate.id)
         queryCallback(query, Constants.PRIMARY_DB_ID)
         super(SiteUpdateTask, self).addSiteProperties(siteUpdate, queryCallback)
       if siteUpdate.updateType == dc.EventObjects.SiteUpdate.UPDATE_TYPE_UPDATE:
         super(SiteUpdateTask, self).updateSiteProperties(siteUpdate, queryCallback)
 
 

Here is the call graph for this function:

◆ updateSitesFilter()

def dc_db.SiteUpdateTask.SiteUpdateTask.updateSitesFilter	(	self,
		siteUpdate,
		queryCallback
	)

Definition at line 102 of file SiteUpdateTask.py.

   def updateSitesFilter(self, siteUpdate, queryCallback):
     if siteUpdate.filters is not None:
       if siteUpdate.updateType == dc.EventObjects.SiteUpdate.UPDATE_TYPE_APPEND:
         super(SiteUpdateTask, self).addSitesFilter(siteUpdate, queryCallback)
       if siteUpdate.updateType == dc.EventObjects.SiteUpdate.UPDATE_TYPE_OVERWRITE:
         query = Constants.DEL_BY_ID_QUERY_TEMPLATE % ("sites_filters", siteUpdate.id)
         queryCallback(query, Constants.PRIMARY_DB_ID)
         super(SiteUpdateTask, self).addSitesFilter(siteUpdate, queryCallback)
       if siteUpdate.updateType == dc.EventObjects.SiteUpdate.UPDATE_TYPE_UPDATE:
         super(SiteUpdateTask, self).updateSitesFilter(siteUpdate, queryCallback)
 
 

Here is the call graph for this function:

◆ updateSiteURL()

def dc_db.SiteUpdateTask.SiteUpdateTask.updateSiteURL	(	self,
		siteUpdate,
		queryCallback
	)

Definition at line 134 of file SiteUpdateTask.py.

   def updateSiteURL(self, siteUpdate, queryCallback):
     if siteUpdate.urls is not None:
       site = dc.EventObjects.Site("")
       site.urls = siteUpdate.urls
       site.id = siteUpdate.id
       if siteUpdate.updateType == dc.EventObjects.SiteUpdate.UPDATE_TYPE_APPEND:
         super(SiteUpdateTask, self).addSiteURLSites(site, queryCallback)
       if siteUpdate.updateType == dc.EventObjects.SiteUpdate.UPDATE_TYPE_OVERWRITE:
         URL_TABLE_DEL_TEMPLATE = "DELETE FROM `sites_urls` WHERE `Site_Id` = '%s'"
         query = URL_TABLE_DEL_TEMPLATE % siteUpdate.id
         queryCallback(query, Constants.PRIMARY_DB_ID)
         super(SiteUpdateTask, self).addSiteURLSites(site, queryCallback)
       if siteUpdate.updateType == dc.EventObjects.SiteUpdate.UPDATE_TYPE_UPDATE:
         super(SiteUpdateTask, self).updateSiteURLSites(siteUpdate, queryCallback)
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ updateURlsURL()

def dc_db.SiteUpdateTask.SiteUpdateTask.updateURlsURL	(	self,
		urls,
		siteId,
		queryCallback
	)

Definition at line 154 of file SiteUpdateTask.py.

   def updateURlsURL(self, urls, siteId, queryCallback):
     query = None
     urlMd5Defined = None
     for urlObject in urls:
       try:
         if urlObject is not None:
           if urlObject.url is not None:
             urlMd5 = hashlib.md5(urlObject.url).hexdigest()
           else:
             urlMd5 = None
           # Check is URL already exists with urlMd5 different from got to update
           if urlObject.urlMd5 is not None and urlObject.url is not None:
             query = str(Constants.URL_URL_SQL_SELECT_COUNT + "`URL`='%s' AND `URLMd5`<>'%s'") % \
                        (siteId, Utils.escape(urlObject.url), urlObject.urlMd5)
             r = queryCallback(query, Constants.SECONDARY_DB_ID)
             if r is None or (len(r) > 0) and len(r[0]) > 0 and r[0][0] > 0:
               logger.error("Root URL '%s' already exists in dc_urls.urls_%s !", str(urlObject.url), siteId)
               continue
           if urlObject.url is not None and urlObject.urlMd5 is None:
             urlMd5Defined = False
             urlObject.urlMd5 = urlMd5
           elif urlObject.urlMd5 is not None:
             urlMd5Defined = True
           else:
             urlMd5Defined = False
           fields, values = Constants.getFieldsValuesTuple(urlObject, Constants.URLTableDict)
           fieldValueString = Constants.createFieldsValuesString(fields, values)
           if urlMd5Defined is False and urlObject.url is not None:
             query = str(Constants.URL_URL_SQL_UPDATE + " `URL`='%s'") % (siteId, fieldValueString, urlObject.url)
           elif urlMd5Defined is True:
             query = str(Constants.URL_URL_SQL_UPDATE + " `URLMd5`='%s'") % (siteId, fieldValueString, urlObject.urlMd5)
           else:
             query = None
           if query is not None:
             queryCallback(query, Constants.SECONDARY_DB_ID)
           else:
             logger.error('No url or urlMd5 value in urlObject, update query not created!\n%s' + str(urlObject))
           # Additionally update the urlMd5 after URL update to synch them if urlMd5 not matched
           if urlMd5Defined and urlMd5 != urlObject.urlMd5:
             query = (Constants.URL_URL_SQL_UPDATE + " `URL`='%s'") % (siteId, 'URLMd5="' + str(urlMd5) + '"',
                                                                       Utils.escape(urlObject.url))
             logger.debug('Update old URLMd5 with new: %s for url: %s, query: %s', urlMd5, urlObject.url, str(query))
             queryCallback(query, Constants.SECONDARY_DB_ID)
       except Exception, err:
         logger.error("Error: %s, query:%s, urlMd5Defined: %s, urlObject.url: %s", str(err), str(query),
                      str(urlMd5Defined), str(urlObject.url))
         logger.error(Utils.getTracebackInfo())
 
 

Here is the caller graph for this function:

Member Data Documentation

◆ fieldRecalculator

dc_db.SiteUpdateTask.SiteUpdateTask.fieldRecalculator

Definition at line 43 of file SiteUpdateTask.py.

◆ urlDeleteTask

dc_db.SiteUpdateTask.SiteUpdateTask.urlDeleteTask

Definition at line 42 of file SiteUpdateTask.py.

The documentation for this class was generated from the following file:

sources/hce/dc_db/SiteUpdateTask.py

Public Member Functions

Public Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

◆ __init__()

Member Function Documentation

◆ copyURLsFromSiteURlsToURLsURLs()

◆ deleteRootUrlsFromURLsUrls()

◆ process()

◆ restartSite()

◆ updateSite()

◆ updateSiteProperties()

◆ updateSitesFilter()

◆ updateSiteURL()

◆ updateURlsURL()

Member Data Documentation

◆ fieldRecalculator

◆ urlDeleteTask

◆ init()