Inheritance diagram for dc_db.URLNewTask.URLNewTask:

Collaboration diagram for dc_db.URLNewTask.URLNewTask:

Public Member Functions
def	__init__ (self, keyValueStorageDir, rawDataDir, dBDataTask, siteTask=None)

def	newSiteCreate (self, initUrl, queryCallback)

def	fillSiteRelatedFields (self, urlObj, queryCallback)

def	resolveSiteIdByURL (self, url, queryCallback)

def	calcSiteIdByUrl (self, url)

def	siteTableOperation (self, urlObj, queryCallback)

def	process (self, urls, queryCallback)

def	urlInsertWithGoodSietId (self, urlObj, statusInit, queryCallback)

def	selectURL (self, urlObject, queryCallback)

def	addURL (self, urlObject, queryCallback)

def	attributesSet (self, attributes, queryCallback)

Public Member Functions inherited from dc_db.BaseTask.BaseTask
def	isSiteExist (self, siteId, queryCallback, userId=None)

def	generateCriterionSQL (self, criterions, additionWhere=None, siteId=None)

def	fetchByCriterions (self, criterions, queryCallback)

def	dbLock (self, mutexName, queryCallback, sleepTime=1, mutexLockTTL=Constants.DEFAULT_LOCK_TTL)

def	dbUnlock (self, mutexName, queryCallback)

def	createUrlsInsertQuery (self, siteId, localKeys, localValues)

def	copyUrlsToDcUrls (self, siteId, queryCallback)

def	statisticLogUpdate (self, localObj, urlMd5, siteId, status, queryCallback, isInsert=False)

def	calculateMd5FormUrl (self, url, urlType, useNormilize=False)

Public Attributes
	siteTask

	recalculator

	urlMd5

	urlUpdateTask

Static Public Attributes
int	CODE_GOOD_INSERT = 0

int	CODE_BAD_INSERT = 1

int	CODE_ALREADY_EXIST = 2

Additional Inherited Members
Static Public Member Functions inherited from dc_db.BaseTask.BaseTask
def	readValueFromSiteProp (siteId, propName, queryCallback, urlMd5=None)

Detailed Description

Definition at line 24 of file URLNewTask.py.

Constructor & Destructor Documentation

◆ init()

def dc_db.URLNewTask.URLNewTask.__init__	(	self,
		keyValueStorageDir,
		rawDataDir,
		dBDataTask,
		siteTask = `None`
	)

Definition at line 32 of file URLNewTask.py.

   def __init__(self, keyValueStorageDir, rawDataDir, dBDataTask, siteTask=None):
     super(URLNewTask, self).__init__()
     self.siteTask = siteTask
     self.recalculator = FieldRecalculator()
     self.urlMd5 = None
     self.urlUpdateTask = URLUpdateTask(keyValueStorageDir, rawDataDir, dBDataTask)
 
 

Member Function Documentation

◆ addURL()

def dc_db.URLNewTask.URLNewTask.addURL	(	self,
		urlObject,
		queryCallback
	)

Definition at line 210 of file URLNewTask.py.

   def addURL(self, urlObject, queryCallback):
     self.statisticLogUpdate(urlObject, self.urlMd5, urlObject.siteId, urlObject.status, queryCallback, True)
     ret = False
     fields, values = Constants.getFieldsValuesTuple(urlObject, Constants.URLTableDict)
     fieldValueString = Constants.createFieldsValuesString(fields, values)
     if fieldValueString is not None and fieldValueString != "":
       query = Constants.INSERT_COMMON_TEMPLATE % ((Constants.DC_URLS_TABLE_NAME_TEMPLATE % urlObject.siteId),
                                                   fieldValueString)
       logger.debug(str(query))
       queryCallback(query, Constants.SECONDARY_DB_ID, Constants.EXEC_NAME, True)
       ret = True
 
     return ret
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ attributesSet()

def dc_db.URLNewTask.URLNewTask.attributesSet	(	self,
		attributes,
		queryCallback
	)

Definition at line 229 of file URLNewTask.py.

229 def attributesSet(self, attributes, queryCallback):

230 logger.debug(">>> Add Attributes (len) == " + str(len(attributes)))

231 attrSetTask = AttrSetTask()

232 res = attrSetTask.process(attributes, queryCallback)

233 logger.debug(">>> Add Attributes (res) == " + varDump(res))

234

app.Utils.varDump

def varDump(obj, stringify=True, strTypeMaxLen=256, strTypeCutSuffix='...', stringifyType=1, ignoreErrors=False, objectsHash=None, depth=0, indent=2, ensure_ascii=False, maxDepth=10)

Definition: Utils.py:410

Here is the call graph for this function:

Here is the caller graph for this function:

◆ calcSiteIdByUrl()

def dc_db.URLNewTask.URLNewTask.calcSiteIdByUrl	(	self,
		url
	)

Definition at line 84 of file URLNewTask.py.

   def calcSiteIdByUrl(self, url):
     canonicUrl = Utils.UrlParser.generateDomainUrl(url)
     if canonicUrl is not None and len(canonicUrl) > 0 and canonicUrl[-1] != '/':
       canonicUrl += '/'
     localSiteId = hashlib.md5(canonicUrl).hexdigest()
     return localSiteId
 
 

Here is the caller graph for this function:

◆ fillSiteRelatedFields()

def dc_db.URLNewTask.URLNewTask.fillSiteRelatedFields	(	self,
		urlObj,
		queryCallback
	)

Definition at line 55 of file URLNewTask.py.

   def fillSiteRelatedFields(self, urlObj, queryCallback):
     ret = False
     SITE_EXTRACT_SQL_QUERY = "SELECT `RequestDelay`, `HTTPTimeout`, `URLType` FROM `sites` WHERE id = '%s'"
     res = queryCallback(SITE_EXTRACT_SQL_QUERY % urlObj.siteId, Constants.PRIMARY_DB_ID)
     if hasattr(res, '__iter__') and len(res) > 0:
       ret = True
       if urlObj.requestDelay is None:
         urlObj.requestDelay = res[0][0]
       if urlObj.httpTimeout is None:
         urlObj.httpTimeout = res[0][1]
       if urlObj.type is None:
         urlObj.type = res[0][2]
     return ret
 
 

Here is the caller graph for this function:

◆ newSiteCreate()

def dc_db.URLNewTask.URLNewTask.newSiteCreate	(	self,
		initUrl,
		queryCallback
	)

Definition at line 44 of file URLNewTask.py.

   def newSiteCreate(self, initUrl, queryCallback):
     if self.siteTask is None:
       raise Exception(">>> URLNew.siteTask object is None!")
     localSiteObj = dc.EventObjects.Site(initUrl)
     self.siteTask.process(localSiteObj, queryCallback)
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ process()

def dc_db.URLNewTask.URLNewTask.process	(	self,
		urls,
		queryCallback
	)

Definition at line 134 of file URLNewTask.py.

   def process(self, urls, queryCallback):
     ret = GeneralResponse()
     status = URLNewTask.CODE_BAD_INSERT
     isRelatedSite = False
     for url in urls:
       isRelatedSite = False
       if url.siteId is None and url.siteSelect != dc.EventObjects.URL.SITE_SELECT_TYPE_EXPLICIT:
         url.siteId = self.resolveSiteIdByURL(url.url, queryCallback)
       if self.isSiteExist(url.siteId, queryCallback):
         isRelatedSite = self.fillSiteRelatedFields(url, queryCallback)
       logger.debug(">>> Url New main = " + url.url)
       try:
         if not isRelatedSite:
           logger.debug(">>> Url New before = " + url.url)
           self.siteTableOperation(url, queryCallback)
           logger.debug(">>> Site_Id By URL = %s", str(url.url))
           logger.debug(">>> Url New after = " + url.url)
         if url.siteId is not None and url.siteId != "":
           status = self.urlInsertWithGoodSietId(url, status, queryCallback)
       except Exception as excp:
         logger.debug(">>> Url New operation exception = " + str(excp))
       ret.statuses.append(status)
     return ret
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ resolveSiteIdByURL()

def dc_db.URLNewTask.URLNewTask.resolveSiteIdByURL	(	self,
		url,
		queryCallback
	)

Definition at line 74 of file URLNewTask.py.

   def resolveSiteIdByURL(self, url, queryCallback):
     ret = None
     query = Constants.SELECT_SITE_ID_BY_URL % url
     res = queryCallback(query, Constants.PRIMARY_DB_ID)
     if hasattr(res, '__iter__') and len(res) > 0:
       ret = res[0][0]
       logger.debug(">>> Site_Id By URL = %s", str(ret))
     return ret
 
 

Here is the caller graph for this function:

◆ selectURL()

def dc_db.URLNewTask.URLNewTask.selectURL	(	self,
		urlObject,
		queryCallback
	)

Definition at line 191 of file URLNewTask.py.

   def selectURL(self, urlObject, queryCallback):
     ret = False
     self.urlMd5 = None
     LOCAL_URL_CHECK_QUERY = "SELECT COUNT(*) FROM `urls_%s` WHERE `URLMd5` = '%s'"
     if urlObject.urlMd5 is not None:
       self.urlMd5 = urlObject.urlMd5
     else:
       self.urlMd5 = hashlib.md5(urlObject.url).hexdigest()
     query = LOCAL_URL_CHECK_QUERY % (urlObject.siteId, self.urlMd5)
     res = queryCallback(query, Constants.SECONDARY_DB_ID)
     if hasattr(res, '__iter__') and len(res) > 0 and len(res[0]) > 0 and res[0][0] > 0:
       ret = True
     return ret
 
 

Here is the caller graph for this function:

◆ siteTableOperation()

def dc_db.URLNewTask.URLNewTask.siteTableOperation	(	self,
		urlObj,
		queryCallback
	)

Definition at line 96 of file URLNewTask.py.

   def siteTableOperation(self, urlObj, queryCallback):
     if urlObj.siteSelect == dc.EventObjects.URL.SITE_SELECT_TYPE_EXPLICIT:
       if urlObj.siteId == "" or not self.isSiteExist(urlObj.siteId, queryCallback):
         urlObj.siteId = "0"
     elif urlObj.siteSelect == dc.EventObjects.URL.SITE_SELECT_TYPE_AUTO:
       try:
         canonicUrl = Utils.UrlParser.generateDomainUrl(urlObj.url)
         if canonicUrl is not None and len(canonicUrl) > 0 and canonicUrl[-1] != '/':
           canonicUrl += '/'
         localSiteId = self.calcSiteIdByUrl(urlObj.url)
         logger.debug(">>> S_NEW_ID=" + str(localSiteId))
         if self.isSiteExist(localSiteId, queryCallback):
           urlObj.siteId = localSiteId
           self.fillSiteRelatedFields(urlObj, queryCallback)
         elif canonicUrl is not None:
           self.newSiteCreate(canonicUrl, queryCallback)
           urlObj.siteId = localSiteId
         else:
           raise Exception(">>> canonicUrl is None !!!")
       except app.Exceptions.UrlParseException:
         logger.debug(">>> UrlParseException")
     elif urlObj.siteSelect == dc.EventObjects.URL.SITE_SELECT_TYPE_QUALIFY_URL:
       localSiteId = self.calcSiteIdByUrl(urlObj.url)
       if not self.isSiteExist(localSiteId, queryCallback):
         urlObj.siteId = "0"
     elif urlObj.siteSelect == dc.EventObjects.URL.SITE_SELECT_TYPE_NONE:
       localSiteId = self.calcSiteIdByUrl(urlObj.url)
       if not self.isSiteExist(localSiteId, queryCallback):
         Exception(">>> urlObj operation can't find siteId")
     else:
       raise Exception(">>> urlObj.siteSelect field has wrong value - %s" % str(urlObj.siteSelect))
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urlInsertWithGoodSietId()

def dc_db.URLNewTask.URLNewTask.urlInsertWithGoodSietId	(	self,
		urlObj,
		statusInit,
		queryCallback
	)

Definition at line 165 of file URLNewTask.py.

   def urlInsertWithGoodSietId(self, urlObj, statusInit, queryCallback):
     ret = statusInit
     if not self.selectURL(urlObj, queryCallback):
       if self.addURL(urlObj, queryCallback):
         ret = URLNewTask.CODE_GOOD_INSERT
         if urlObj.attributes is not None and len(urlObj.attributes) > 0:
           self.attributesSet(urlObj.attributes, queryCallback)
     else:
       ret = URLNewTask.CODE_ALREADY_EXIST
       if urlObj.urlUpdate is not None:
         logger.debug(">>> Url New Start Internal urlUpdate")
         self.urlUpdateTask.process([urlObj.urlUpdate], queryCallback)
 
       if urlObj.attributes is not None and len(urlObj.attributes) > 0:
         self.attributesSet(urlObj.attributes, queryCallback)
 
     self.recalculator.commonRecalc(urlObj.siteId, queryCallback)
     if "urlPut" in urlObj.__dict__ and urlObj.urlPut is not None:
       self.urlUpdateTask.urlPutOperation(urlObj, urlObj.urlPut, queryCallback)
     return ret
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

◆ CODE_ALREADY_EXIST

int dc_db.URLNewTask.URLNewTask.CODE_ALREADY_EXIST = 2

static

Definition at line 28 of file URLNewTask.py.

◆ CODE_BAD_INSERT

int dc_db.URLNewTask.URLNewTask.CODE_BAD_INSERT = 1

static

Definition at line 27 of file URLNewTask.py.

◆ CODE_GOOD_INSERT

int dc_db.URLNewTask.URLNewTask.CODE_GOOD_INSERT = 0

static

Definition at line 26 of file URLNewTask.py.

◆ recalculator

dc_db.URLNewTask.URLNewTask.recalculator

Definition at line 35 of file URLNewTask.py.

◆ siteTask

dc_db.URLNewTask.URLNewTask.siteTask

Definition at line 34 of file URLNewTask.py.

◆ urlMd5

dc_db.URLNewTask.URLNewTask.urlMd5

Definition at line 36 of file URLNewTask.py.

◆ urlUpdateTask

dc_db.URLNewTask.URLNewTask.urlUpdateTask

Definition at line 37 of file URLNewTask.py.

The documentation for this class was generated from the following file:

sources/hce/dc_db/URLNewTask.py

Public Member Functions

Public Attributes

Static Public Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

◆ __init__()

Member Function Documentation

◆ addURL()

◆ attributesSet()

◆ calcSiteIdByUrl()

◆ fillSiteRelatedFields()

◆ newSiteCreate()

◆ process()

◆ resolveSiteIdByURL()

◆ selectURL()

◆ siteTableOperation()

◆ urlInsertWithGoodSietId()

Member Data Documentation

◆ CODE_ALREADY_EXIST

◆ CODE_BAD_INSERT

◆ CODE_GOOD_INSERT

◆ recalculator

◆ siteTask

◆ urlMd5

◆ urlUpdateTask

◆ init()