Inheritance diagram for dc_crawler.URLProcess.URLProcess:

Collaboration diagram for dc_crawler.URLProcess.URLProcess:

Public Member Functions
def	__init__ (self, protocols=None)

def	setProtocols (self, protocols=None)

def	checkUrlByPath (self, url)

def	checkUrlByProtocol (self, url)

def	checkFieldsIsNone (self, checkList)

def	resolveTableName (self, localSiteId)

def	readCurrentCnt (self, maxURLs)

def	simpleURLCanonize (self, realUrl)

def	processURL (self, realUrl, internalLinks, externalLinks, filtersApply=None, siteFilters=None, baseUrl=None)

def	isUrlExist (self, recrawlPeriod, urlMd5)

def	updateURLFields (self, urlMd5, wrapper, siteId)

def	recrawlUrlUpdateHandler (self, dbWrapper, recrawlUrlUpdateProperty, urlUpdateObj)

def	detectUrlMime (self, contentTypeMap=None, urlObj=None)

def	getDepthFromUrl (self, urlMd5)

def	updateURLForFailed (self, errorBit, batchItem, httpCode=CONSTS.HTTP_CODE_400, status=dc.EventObjects.URL.STATUS_CRAWLED, updateUdate=True)

def	getRealUrl (self)

def	resolveHTTP (self, postForms, headersDict)

def	updateCrawledURL (self, crawledResource, batchItem, contentSize, status=dc.EventObjects.URL.STATUS_CRAWLED)

def	updateURL (self, batchItem, batchId, status=dc.EventObjects.URL.STATUS_CRAWLING)

def	updateURLStatus (self, urlId, status=dc.EventObjects.URL.STATUS_CRAWLED)

def	resetErrorMask (self, batchItem)

def	addURLFromBatchToDB (self, batchItem, crawlerType, recrawlPeriod, autoRemoveProps)

def	updateCollectTimeAndMime (self, detectedMime, batchItem, crawledTime, autoDetectMime, httpHeaders=None, strContent=None)

def	urlDBSync (self, batchItem, crawlerType, recrawlPeriod, autoRemoveProps)

def	updateAdditionProps (self, internalLinksCount, externalLinksCount, batchItem, size, freq, contentMd5)

def	createUrlObjForCollectURLs (self, urlMd5, formMethods, parentMd5, depth, detectedMime, maxURLsFromPage)

def	createUrlObjForChain (self, pattern, urlMd5, formMethods, parentMd5, depth, detectedMime, maxURLsFromPage)

def	updateTypeForURLObjects (self, urlObjects, typeArg=dc.EventObjects.URL.TYPE_CHAIN)

def	fillRssFieldInUrlObj (self, oldUrl, objectUrlUlr, batchItem, processorName, feed, rootFeed=False)

def	fillRssFieldOneElem (self, entry, urlObj, batchItem, status, crawled, localType)

def	urlTemplateApply (self, url, crawlerType, urlTempalteRegular, urlTempalteRealtime, urlTempalteRegularEncode, urlTempalteRealtimeEncode)

Static Public Member Functions
def	checkDictEmptyStrings (inDict, keys)

def	autoRemoveURL (autoRemoveProps, recrawlPeriod, urlTable, wrapper)

def	conditionEvaluate (condition, conditionalData)

def	additionalUrlObjInit (urlObj, urlInitParam, conditionalData)

Public Attributes
	isUpdateCollection

	urlObj

	url

	dbWrapper

	siteId

	site

	urlTable

	protocolsList

	siteProperties

	normMask

Static Public Attributes
string	DC_URLS_TABLE_PREFIX = "urls_"

int	DETECT_MIME_TIMEOUT = 1

	PATTERN_WITH_PROTOCOL = re.compile('[a-zA-Z]+:(//)?')

string	URL_TEMPLATE_CONST = "%URL%"

string	PROTOCOL_PREFIX = "://"

list	DEFAULT_PROTOCOLS = ["http", "https"]

Detailed Description

Definition at line 48 of file URLProcess.py.

Constructor & Destructor Documentation

◆ init()

def dc_crawler.URLProcess.URLProcess.__init__	(	self,
		protocols = `None`
	)

Definition at line 58 of file URLProcess.py.

   def __init__(self, protocols=None):
     self.isUpdateCollection = False
     self.urlObj = None
     self.url = None
     self.dbWrapper = None
     self.siteId = None
     self.site = None
     self.urlTable = None
     self.protocolsList = self.DEFAULT_PROTOCOLS
     self.siteProperties = None
     self.setProtocols(protocols)
     self.normMask = UrlNormalizator.NORM_DEFAULT
 
 

Member Function Documentation

◆ additionalUrlObjInit()

def dc_crawler.URLProcess.URLProcess.additionalUrlObjInit	(	urlObj,
		urlInitParam,
		conditionalData
	)

static

Definition at line 1061 of file URLProcess.py.

   def additionalUrlObjInit(urlObj, urlInitParam, conditionalData):
     try:
       urlInit = json.loads(urlInitParam)
       for fieldName in urlInit:
         if hasattr(urlObj, fieldName):
           for condition in urlInit[fieldName]["conditions"]:
             if (isinstance(condition, types.BooleanType) and condition) or \
             (isinstance(condition, types.StringTypes) and URLProcess.conditionEvaluate(condition, conditionalData)):
               setattr(urlObj, fieldName, urlInit[fieldName]["value"])
               break
     except Exception as excp:
       logger.debug(">>> some error with URLS_FIELDS_INIT param processing; err=" + str(excp))
 

◆ addURLFromBatchToDB()

def dc_crawler.URLProcess.URLProcess.addURLFromBatchToDB	(	self,
		batchItem,
		crawlerType,
		recrawlPeriod,
		autoRemoveProps
	)

Definition at line 567 of file URLProcess.py.

   def addURLFromBatchToDB(self, batchItem, crawlerType, recrawlPeriod, autoRemoveProps):
     # variable for result
     ret = True
 
     if self.dbWrapper is not None:
       self.checkFieldsIsNone(["dbWrapper", "siteId", "urlTable"])
       try:
         siteStatusObj = dc.EventObjects.SiteStatus(Utils.autoFillSiteId(self.siteId, logger))
         result = self.dbWrapper.siteStatus(siteStatusObj)
         if result is not None:
           maxURLs = result.maxURLs
           if ((crawlerType != dc.EventObjects.Batch.TYPE_REAL_TIME_CRAWLER) and \
           (result.state != dc.EventObjects.Site.STATE_ACTIVE)) or \
           ((crawlerType == dc.EventObjects.Batch.TYPE_REAL_TIME_CRAWLER) and \
           (result.state == dc.EventObjects.Site.STATE_DISABLED)):
             logger.debug("Warning: Batch CrawlerType: %s, site state is %s but not STATE_ACTIVE!", crawlerType,
                          str(result.state))
             raise SyncronizeException("Site state is not active, state=" + str(result.state))
 
           if (result.maxErrors > 0) and (result.errors > result.maxErrors):
             msg = "Site maxErrors limit " + str(result.maxErrors) + " reached " + str(result.errors)
             logger.debug(msg)
             raise SyncronizeException(msg)
 
           # Check the limit of the maxURLs for count of active URLs (not migrated with batches)
           if DC_CONSTS.SITE_PROP_AUTO_REMOVE_WHERE_ACTIVE in autoRemoveProps:
             where = autoRemoveProps[DC_CONSTS.SITE_PROP_AUTO_REMOVE_WHERE_ACTIVE]
           else:
             where = "NOT (`Status`=4 AND `Crawled`=0 AND `Processed`=0)"
           query = "SELECT COUNT(*) FROM `%s` " % self.urlTable
           query += "WHERE " + where
           result = self.dbWrapper.customRequest(query, CrawlerTask.DB_URLS)
           if len(result) > 0 and len(result[0]) > 0:
             activeURLs = result[0][0]
             logger.debug("Active URLs count: " + str(activeURLs) + ", maxURLs: " + str(maxURLs))
             if (maxURLs > 0) and (activeURLs >= maxURLs):
               autoRemoved = URLProcess.autoRemoveURL(autoRemoveProps, recrawlPeriod, self.urlTable, self.dbWrapper)
               if autoRemoved < 1:
                 msg = "Active URLs:" + str(activeURLs) + " > MaxURLs:" + str(maxURLs) + " and no one auto-removed!"
                 logger.debug(msg)
                 raise SyncronizeException(msg)
               else:
                 logger.debug(str(autoRemoved) + " URLs auto-removed to insert new URL from batch")
           else:
             msg = "Error of query processing, no rows returned:\n" + query
             logger.debug(msg)
             raise SyncronizeException(msg)
 
           batchItem.urlObj.CDate = str(datetime.datetime.now())
           batchItem.urlObj.UDate = batchItem.urlObj.CDate
           batchItem.urlObj.tcDate = batchItem.urlObj.CDate
           batchItem.urlObj.batchId = 0  # self.batch.id
           result = self.dbWrapper.urlNew([batchItem.urlObj])
           logger.debug("rows_count: %s", result)
           self.isUpdateCollection = True
           # self.updateCollectedURLs()
         else:
           raise SyncronizeException("Execute last SQL query(SiteStatus), no rows returned:\n")
       except Exception as err:
         logger.debug('Error add new url from batch (another host source):' + str(err))
         ret = False
         raise err
 
     return ret
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ autoRemoveURL()

def dc_crawler.URLProcess.URLProcess.autoRemoveURL	(	autoRemoveProps,
		recrawlPeriod,
		urlTable,
		wrapper
	)

static

Definition at line 660 of file URLProcess.py.

   def autoRemoveURL(autoRemoveProps, recrawlPeriod, urlTable, wrapper):
     ret = 0
     if wrapper is not None:
       try:
         # logger.debug("Auto remove properties:\n%s", varDump(autoRemoveProps))
         # If defined auto remove properties and set in proper values
         if URLProcess.checkDictEmptyStrings(autoRemoveProps, [DC_CONSTS.SITE_PROP_AUTO_REMOVE_RESOURCES,
                                                               DC_CONSTS.SITE_PROP_AUTO_REMOVE_WHERE,
                                                               DC_CONSTS.SITE_PROP_AUTO_REMOVE_ORDER]):
           # Select candidates to remove
           query = "SELECT Site_Id, URLMd5 FROM %s WHERE %s ORDER BY %s LIMIT %s" % \
                   (urlTable,
                    autoRemoveProps[DC_CONSTS.SITE_PROP_AUTO_REMOVE_WHERE].replace("%RecrawlPeriod%", str(recrawlPeriod)),
                    autoRemoveProps[DC_CONSTS.SITE_PROP_AUTO_REMOVE_ORDER],
                    autoRemoveProps[DC_CONSTS.SITE_PROP_AUTO_REMOVE_RESOURCES])
           logger.debug("SQL to select auto remove candidates: %s", query)
           result = wrapper.customRequest(query, CrawlerTask.DB_URLS)
           if len(result) > 0:
             urlsToDelete = []
             for row in result:
               # Create new URLDelete object
               urlDelete = dc.EventObjects.URLDelete(row[0], row[1], dc.EventObjects.URLStatus.URL_TYPE_MD5,
                                                     reason=dc.EventObjects.URLDelete.REASON_CRAWLER_AUTOREMOVE)
               urlsToDelete.append(urlDelete)
               logger.debug("URL added to auto remove URLMd5:[%s]", row[1])
             drceSyncTasksCoverObj = DC_CONSTS.DRCESyncTasksCover(DC_CONSTS.EVENT_TYPES.URL_DELETE, urlsToDelete)
             responseDRCESyncTasksCover = wrapper.process(drceSyncTasksCoverObj)
             logger.debug("Response from db-task module on URLDelete operation:\n%s", \
                               Utils.varDump(responseDRCESyncTasksCover))
             deleted = 0
             if isinstance(responseDRCESyncTasksCover, DC_CONSTS.DRCESyncTasksCover):
               generalResponse = responseDRCESyncTasksCover.eventObject
               if isinstance(generalResponse, GeneralResponse):
                 deleted = sum([el for el in generalResponse.statuses if el])
             ret = deleted
           else:
             logger.debug("No auto remove candidates or SQL query error!")
         else:
           logger.debug("No mandatory auto remove properties in auto_remove_props:\n" + Utils.varDump(autoRemoveProps))
       except Exception as err:
         ExceptionLog.handler(logger, err, 'Error of auto remove operation:')
 
     return ret
 
 

Here is the call graph for this function:

◆ checkDictEmptyStrings()

def dc_crawler.URLProcess.URLProcess.checkDictEmptyStrings	(	inDict,
		keys
	)

static

Definition at line 639 of file URLProcess.py.

   def checkDictEmptyStrings(inDict, keys):
     ret = False
     for key in keys:
       if key in inDict and inDict[key] != '':
         ret = True
       else:
         ret = False
         break
     return ret
 
 
 

◆ checkFieldsIsNone()

def dc_crawler.URLProcess.URLProcess.checkFieldsIsNone	(	self,
		checkList
	)

Definition at line 108 of file URLProcess.py.

   def checkFieldsIsNone(self, checkList):
     # for field in self.__dict__:
     #  if field in checkList and (not hasattr(self, field) or getattr(self, field) is None):
     #    raise Exception(">>> [CollectURLs] Mandatory field must be initialized, field Name = " + field)
     for name in checkList:
       if not hasattr(self, name) or getattr(self, name) is None:
         raise Exception("Some mandatory field `%s` must be initialized!", name)
 
 

Here is the caller graph for this function:

◆ checkUrlByPath()

def dc_crawler.URLProcess.URLProcess.checkUrlByPath	(	self,
		url
	)

Definition at line 85 of file URLProcess.py.

   def checkUrlByPath(self, url):
     ret = False
     position = url.find(self.PROTOCOL_PREFIX)
     if position > 0 and url.find('/') == (position + 1):
       ret = True
     return ret
 
 

Here is the caller graph for this function:

◆ checkUrlByProtocol()

def dc_crawler.URLProcess.URLProcess.checkUrlByProtocol	(	self,
		url
	)

Definition at line 95 of file URLProcess.py.

   def checkUrlByProtocol(self, url):
     ret = False
     for elem in self.protocolsList:
       if url.lower().startswith(elem + self.PROTOCOL_PREFIX):
         ret = True
         break
     if not ret:
       logger.debug(">>> URL skiped by protocol = " + url)
     return ret
 
 

Here is the caller graph for this function:

◆ conditionEvaluate()

def dc_crawler.URLProcess.URLProcess.conditionEvaluate	(	condition,
		conditionalData
	)

static

Definition at line 1020 of file URLProcess.py.

   def conditionEvaluate(condition, conditionalData):
     ret = False
     conditionElements = condition.split(' ', 2)
     if len(conditionElements) == 3:
       objectName = conditionElements[0]
       operationName = conditionElements[1]
       value = conditionElements[2]
       if len(value) > 0 and (value[0] == '"' or value[0] == '\''):
         value = value[1:]
       if len(value) > 0 and (value[-1] == '"' or value[-1] == '\''):
         value = value[0:-1]
       objectName = objectName.strip().split('.')
       if len(objectName) >= 2:
         fieldName = objectName[1]
         objectName = objectName[0]
         if objectName in conditionalData and hasattr(conditionalData[objectName], fieldName):
           if operationName == '=' or operationName == "==":
             if str(getattr(conditionalData[objectName], fieldName)) == value:
               ret = True
           elif operationName == "match":
             if re.compile(value).match(str(getattr(conditionalData[objectName], fieldName))) is not None:
               ret = True
           elif operationName == "search":
             if re.compile(value).search(str(getattr(conditionalData[objectName], fieldName))) is not None:
               ret = True
           elif operationName == "<>" or operationName == "!=":
             if str(getattr(conditionalData[objectName], fieldName)) != value:
               ret = True
           elif operationName == "is" and value == 'empty':
             if str(getattr(conditionalData[objectName], fieldName)) == '':
               ret = True
 
     return ret
 
 

◆ createUrlObjForChain()

def dc_crawler.URLProcess.URLProcess.createUrlObjForChain	(	self,
		pattern,
		urlMd5,
		formMethods,
		parentMd5,
		depth,
		detectedMime,
		maxURLsFromPage
	)

Definition at line 841 of file URLProcess.py.

   def createUrlObjForChain(self, pattern, urlMd5, formMethods, parentMd5, depth, detectedMime, maxURLsFromPage):
     ret = None
     self.checkFieldsIsNone(["url"])
     # logger.debug(">>> chain patter is = " + str(pattern) + " url = " + self.url)
     if re.search(pattern, self.url) is not None:
       ret = self.createUrlObjForCollectURLs(urlMd5, formMethods, parentMd5, depth, detectedMime, maxURLsFromPage)
       ret.type = dc.EventObjects.URL.TYPE_CHAIN
     return ret
 
 

Here is the call graph for this function:

◆ createUrlObjForCollectURLs()

def dc_crawler.URLProcess.URLProcess.createUrlObjForCollectURLs	(	self,
		urlMd5,
		formMethods,
		parentMd5,
		depth,
		detectedMime,
		maxURLsFromPage
	)

Definition at line 809 of file URLProcess.py.

   def createUrlObjForCollectURLs(self, urlMd5, formMethods, parentMd5, depth, detectedMime, maxURLsFromPage):
     self.checkFieldsIsNone(["url", "siteId", "urlObj"])
     ret = dc.EventObjects.URL(self.siteId, self.url, normalizeMask=self.normMask)
     ret.type = self.urlObj.type
     ret.urlMd5 = urlMd5
     ret.requestDelay = self.urlObj.requestDelay
     ret.httpTimeout = self.urlObj.httpTimeout
     ret.httpMethod = formMethods.get(self.url, "get")
     ret.parentMd5 = parentMd5
     ret.maxURLsFromPage = maxURLsFromPage
     ret.tcDate = SQLExpression("NOW()")
     ret.UDate = SQLExpression("NOW()")
     ret.depth = (depth + 1)
     ret.contentType = detectedMime
     ret.priority = self.urlObj.priority
     # TODO Additional URL init
     if self.siteProperties is not None and "URLS_FIELDS_INIT" in self.siteProperties:
       URLProcess.additionalUrlObjInit(ret, self.siteProperties["URLS_FIELDS_INIT"],
                                       {"site": self.site, "parent": self.urlObj})
     return ret
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ detectUrlMime()

def dc_crawler.URLProcess.URLProcess.detectUrlMime	(	self,
		contentTypeMap = `None`,
		urlObj = `None`
	)

Definition at line 339 of file URLProcess.py.

   def detectUrlMime(self, contentTypeMap=None, urlObj=None):
     del urlObj
     self.checkFieldsIsNone(["url"])
     ret = ''
     try:
       res = requests.head(self.url, timeout=self.DETECT_MIME_TIMEOUT)
       ret = res.headers.get('content-type', '').lower()
       if contentTypeMap is not None and ret in contentTypeMap:
         logger.debug(">>> Mime type replaced from %s to %s", ret, contentTypeMap[ret])
         ret = contentTypeMap[ret]
     except Exception:
       logger.warn("detect mime type for %s failed", self.url, exc_info=True)
     return ret
 
 

Here is the call graph for this function:

◆ fillRssFieldInUrlObj()

def dc_crawler.URLProcess.URLProcess.fillRssFieldInUrlObj	(	self,
		oldUrl,
		objectUrlUlr,
		batchItem,
		processorName,
		feed,
		rootFeed = `False`
	)

Definition at line 877 of file URLProcess.py.

   def fillRssFieldInUrlObj(self, oldUrl, objectUrlUlr, batchItem, processorName, feed, rootFeed=False):
     # logger.debug("oldUrl=%s\nobjectUrlUlr=%s\nbatchItem=%s\nprocessorName=%s\nfeed=%s\n",
     #             Utils.varDump(oldUrl),
     #             Utils.varDump(objectUrlUlr),
     #             Utils.varDump(batchItem),
     #             Utils.varDump(processorName),
     #             str(feed))
     self.checkFieldsIsNone(["url", "siteId", "urlObj"])
     ret = None
     status = dc.EventObjects.URL.STATUS_CRAWLED
     crawled = 1
     localType = dc.EventObjects.URL.TYPE_FETCHED
     if processorName == PCONSTS.PROCESSOR_RSS:
       status = dc.EventObjects.URL.STATUS_NEW
       crawled = 0
       localType = dc.EventObjects.URL.TYPE_SINGLE
     if rootFeed:
       ret = self.fillRssFieldOneElem(feed.feed, objectUrlUlr, batchItem, status, crawled, localType)
     else:
       for entry in feed.entries:
         if hasattr(entry, 'link'):
           logger.debug("entry.link=%s, oldUrl=%s", Utils.varDump(entry.link), Utils.varDump(oldUrl))
           if entry.link == oldUrl and ret is None:
             ret = self.fillRssFieldOneElem(entry, objectUrlUlr, batchItem, status, crawled, localType)
             if ret is None:
               logger.debug("Getting next candidate URL")
           elif ret is not None and "urlObj" in ret and ret["urlObj"] is None:
             ret = self.fillRssFieldOneElem(entry, objectUrlUlr, batchItem, status, crawled, localType)
 
     return ret
 
 

Here is the call graph for this function:

◆ fillRssFieldOneElem()

def dc_crawler.URLProcess.URLProcess.fillRssFieldOneElem	(	self,
		entry,
		urlObj,
		batchItem,
		status,
		crawled,
		localType
	)

Definition at line 918 of file URLProcess.py.

   def fillRssFieldOneElem(self, entry, urlObj, batchItem, status, crawled, localType):
     # variable for result
     ret = {}
     ret["entry"] = entry
     ret["urlObj"] = dc.EventObjects.URL(self.siteId, self.url, normalizeMask=self.normMask)
     ret["parent_rss_feed"] = urlObj
     ret["parent_rss_feed_urlMd5"] = batchItem.urlId
     # Getting pubdate from feed
     pubdate = None
     for date in CONSTS.pubdateFeedNames:
       if date in entry:
         try:
           dt = DateTimeType.parse(entry[date], True, logger, False)
           if dt is not None:
             logger.debug("Convert pubdate from: '%s' to '%s'", str(entry[date]), dt.isoformat(' '))
             pubdate = DateTimeType.toUTC(dt).strftime("%Y-%m-%d %H:%M:%S")
             logger.debug("pubdate converted to UTC: '%s'", str(pubdate))
             break
         except TypeError:
           logger.debug("Unsupported date format: '%s'", str(entry[date]))
         except Exception, err:
           logger.debug("Error: %s, data: '%s'", str(err), str(entry[date]))
 
     logger.debug("!!! Before apply 'SQLExpression' and 'STAGE_COLLECT_URLS' pubdate: " + str(pubdate))
     localFilters = Filters(None, self.dbWrapper, batchItem.siteId, 0, None, Filters.OC_SQLE, Filters.STAGE_COLLECT_URLS)
     isExistFilter = localFilters.isExist(Filters.STAGE_COLLECT_URLS, Filters.OC_SQLE)
     logger.debug("Filter is exists: " + str(bool(isExistFilter)))
     if isExistFilter and  pubdate is not None:
       collectURLs = CollectURLs()
       if collectURLs.filtersApply(None, '', batchItem.depth, self.dbWrapper, batchItem.siteId,
                                   {'PDATE':str(pubdate)}, Filters.OC_SQLE, Filters.STAGE_COLLECT_URLS, None, False):
         logger.debug("Candidate URL matched SQLExpression filter.")
       else:
         logger.debug("Candidate URL not matched SQLExpression filter, skipped.")
         # ret["urlObj"] = None
         ret = None
         return ret
 
     if len(entry.links) > 0 and hasattr(entry.links[0], 'type'):
       contentType = entry.links[0].type
     ret["urlObj"].status = status
     ret["urlObj"].crawled = crawled
     ret["urlObj"].contentType = contentType
     ret["urlObj"].pDate = pubdate
     ret["urlObj"].type = localType
     size = len(str(ret))
     ret["urlObj"].size = size
     ret["pubdate"] = pubdate
     # TODO Additional URL init
     if self.siteProperties is not None and "URLS_FIELDS_INIT" in self.siteProperties:
       URLProcess.additionalUrlObjInit(ret, self.siteProperties["URLS_FIELDS_INIT"],
                                       {"site": self.site, "parent": self.urlObj})
 
     # logger.debug(">>>> ret[\"urlObj\"].pDate = " + str(ret["urlObj"].pDate))
 
     return ret
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ getDepthFromUrl()

def dc_crawler.URLProcess.URLProcess.getDepthFromUrl	(	self,
		urlMd5
	)

Definition at line 357 of file URLProcess.py.

   def getDepthFromUrl(self, urlMd5):
     ret = 0
     if self.dbWrapper is not None:
       self.checkFieldsIsNone(["dbWrapper", "siteId"])
 
       urlStatusObj = dc.EventObjects.URLStatus(self.siteId, urlMd5)
       result = self.dbWrapper.urlStatus(urlStatusObj, True)
       if len(result) > 0 and isinstance(result[0], dc.EventObjects.URL):
         ret = result[0].depth
 
     return ret
 
 

Here is the call graph for this function:

◆ getRealUrl()

def dc_crawler.URLProcess.URLProcess.getRealUrl ( self )

Definition at line 411 of file URLProcess.py.

   def getRealUrl(self):
     self.checkFieldsIsNone(["url"])
     if self.url.startswith("http%3A") or self.url.startswith("https%3A"):
       ret = urllib.unquote(self.url.url).decode('utf-8')
     else:
       ret = self.url.decode('utf8')
     return ret
 
 

Here is the call graph for this function:

◆ isUrlExist()

def dc_crawler.URLProcess.URLProcess.isUrlExist	(	self,
		recrawlPeriod,
		urlMd5
	)

Definition at line 208 of file URLProcess.py.

   def isUrlExist(self, recrawlPeriod, urlMd5):
     # variable for result
     ret = False
 
     if self.dbWrapper is not None:
       self.checkFieldsIsNone(["url", "dbWrapper", "siteId", "urlTable"])
 
       if "RECRAWL_URL_AGE_EXPRESSION" in self.siteProperties and self.siteProperties["RECRAWL_URL_AGE_EXPRESSION"] != "":
         ageExpr = self.siteProperties["RECRAWL_URL_AGE_EXPRESSION"].replace("%RECRAWL_PERIOD%", str(recrawlPeriod))
       else:
         ageExpr = "(DATE_ADD(UDate, INTERVAL %s MINUTE)-NOW())" % (str(recrawlPeriod))
       query = "SELECT COUNT(*), %s, `Type` FROM `%s` WHERE `URLMd5` = '%s'" % (ageExpr, self.urlTable, urlMd5)
       result = self.dbWrapper.customRequest(query, CrawlerTask.DB_URLS)
       if result is not None and len(result) > 0 and len(result[0]) > 0 and result[0][0] > 0:
         if recrawlPeriod == 0 or result[0][1] > 0 or result[0][2] == dc.EventObjects.URL.TYPE_FETCHED or \
         ("RECRAWL_NO_ROOT_URLS" in self.siteProperties and self.siteProperties["RECRAWL_NO_ROOT_URLS"] == "0"):
           logger.debug("URL skipped, exists and re-crawling not active, time not reached or URL Type is " +
                        "RSS feed (not to fetch)\n %s %s", self.url, urlMd5)
           ret = True
         else:
           self.updateURLFields(urlMd5, self.dbWrapper, self.siteId)
           logger.debug("URL state updated to NEW because re-crawling\n %s %s", self.url, urlMd5)
           ret = True
       else:
         logger.debug("URL %s treated as new\n %s", self.url, urlMd5)
 
     return ret
 
 

Here is the call graph for this function:

◆ processURL()

def dc_crawler.URLProcess.URLProcess.processURL	(	self,
		realUrl,
		internalLinks,
		externalLinks,
		filtersApply = `None`,
		siteFilters = `None`,
		baseUrl = `None`
	)

Definition at line 157 of file URLProcess.py.

   def processURL(self, realUrl, internalLinks, externalLinks, filtersApply=None, siteFilters=None, baseUrl=None):
     self.checkFieldsIsNone(["urlObj", "siteId", "url"])
     retUrl = None
     retContinue = False
 
     logger.debug("URL: %s", self.url)
 
     if self.urlObj.type == dc.EventObjects.URL.TYPE_SINGLE or not self.url:
       logger.debug("URL type is TYPE_SINGLE - not collect urls. Skip url.")
       retContinue = True
 
     if not retContinue:
       retUrl = UrlNormalize.execute(siteProperties=self.siteProperties, base=baseUrl, url=self.url, supportProtocols=self.protocolsList, log=logger)
       if retUrl is not None:
         localFilters = None
         protocolAllowed = True
         if filtersApply is not None:
           # Stage 'collect urls protocols' and operation code 'regular expression'
           logger.debug(">>> Filters() (3.1) siteFilters: " + str(siteFilters))
           localFilters = Filters(siteFilters, self.dbWrapper, self.siteId, 0, None, Filters.OC_RE, \
                                  Filters.STAGE_COLLECT_URLS_PROTOCOLS)
 
           if localFilters.isExistStage(Filters.STAGE_COLLECT_URLS_PROTOCOLS):
             resFilterApply = filtersApply(siteFilters, retUrl, 0, self.dbWrapper, self.siteId,
                                           None, Filters.OC_RE, Filters.STAGE_COLLECT_URLS_PROTOCOLS)
             logger.debug("Filter apply: " + str(resFilterApply))
 
         protocolAllowed = self.checkUrlByProtocol(retUrl)
         logger.debug("checkUrlByProtocol return: " + str(protocolAllowed))
         logger.debug("retUrl: " + str(retUrl))
         logger.debug("realUrl: " + str(realUrl))
 
         if protocolAllowed:
           if Utils.parseHost(retUrl) == Utils.parseHost(realUrl):
             internalLinks.append(retUrl)
             logger.debug("URL classified as internal")
           elif Utils.parseHost(retUrl):
             externalLinks.append(retUrl)
             logger.debug("URL classified as external")
           else:  # not valid url like http://
             retContinue = True
         else:
           retContinue = True
       else:
         logger.debug(">>> Bad url normalization, url = " + str(retUrl))
         retContinue = True
     return retUrl, retContinue
 
 

Here is the call graph for this function:

◆ readCurrentCnt()

def dc_crawler.URLProcess.URLProcess.readCurrentCnt	(	self,
		maxURLs
	)

Definition at line 126 of file URLProcess.py.

   def readCurrentCnt(self, maxURLs):
     currentCnt = 0
     if self.dbWrapper is not None:
       self.checkFieldsIsNone(["dbWrapper", "urlTable"])
 
       if maxURLs > 0:
         countsql = "SELECT COUNT(*) AS cnt FROM `%s` WHERE NOT (Status=4 AND Crawled=0 AND Processed=0)" % \
                     (self.urlTable,)
         result = self.dbWrapper.customRequest(countsql, CrawlerTask.DB_URLS)
         if result is not None and len(result) > 0 and len(result[0]) > 0:
           currentCnt = result[0][0]
         else:
           currentCnt = 0
 
     logger.debug("!!! maxURLs = %s, currentCnt = %s", str(maxURLs), str(currentCnt))
 
     return currentCnt
 
 

Here is the call graph for this function:

◆ recrawlUrlUpdateHandler()

def dc_crawler.URLProcess.URLProcess.recrawlUrlUpdateHandler	(	self,
		dbWrapper,
		recrawlUrlUpdateProperty,
		urlUpdateObj
	)

Definition at line 298 of file URLProcess.py.

   def recrawlUrlUpdateHandler(self, dbWrapper, recrawlUrlUpdateProperty, urlUpdateObj):
     if dbWrapper is not None:
       propertyStruct = None
       try:
         propertyStruct = json.loads(recrawlUrlUpdateProperty)
       except Exception, err:
         logger.error("Load property 'RECRAWL_URL_UPDATE' was failed, error: %s", str(err))
 
       # If load json was successfully
       if propertyStruct is not None:
         try:
           # list elements or one element?
           for pattern, rules in propertyStruct.items():
             if re.search(pattern, self.url) is not None:
               # Update data accord to parameters
               if "new" in rules and int(rules["new"]) > 0:
                 saveAffectDB = dbWrapper.affect_db
                 dbWrapper.affect_db = True
                 dbWrapper.urlNew(self.urlObj)
                 dbWrapper.affect_db = saveAffectDB
 
               if "fields" in rules and isinstance(rules["fields"], dict):
                 for key, value in rules["fields"].items():
                   if key in DB_CONSTS.URLTableDict.values():
                     for urlUpdateObjName, DBSchemaName in DB_CONSTS.URLTableDict.items():
                       if key == DBSchemaName and hasattr(urlUpdateObj, urlUpdateObjName):
                         setattr(urlUpdateObj, urlUpdateObjName, value)
                         logger.debug("For '" + str(DBSchemaName) + "' found attribute 'UrlUpdate." + \
                                      str(urlUpdateObjName) + "' and set value: " + str(value) + \
                                      " type: " + str(type(value)))
                         break
                   else:
                     logger.debug("Wrong DB schema field name '" + str(key) + "' in property 'RECRAWL_URL_UPDATE'")
 
         except Exception, err:
           logger.error("Usage property 'RECRAWL_URL_UPDATE' was failed, error: %s", str(err))
 
 

Here is the caller graph for this function:

◆ resetErrorMask()

def dc_crawler.URLProcess.URLProcess.resetErrorMask	(	self,
		batchItem
	)

Definition at line 550 of file URLProcess.py.

   def resetErrorMask(self, batchItem):
     if self.dbWrapper is not None:
       self.checkFieldsIsNone(["dbWrapper", "siteId"])
       urlUpdateObj = dc.EventObjects.URLUpdate(self.siteId, batchItem.urlId, dc.EventObjects.URLStatus.URL_TYPE_MD5, \
                                                normalizeMask=self.normMask)
       urlUpdateObj.errorMask = batchItem.urlObj.errorMask = 0
       urlUpdateObj.tcDate = batchItem.urlObj.tcDate = SQLExpression("NOW()")
       urlUpdateObj.UDate = batchItem.urlObj.UDate = SQLExpression("NOW()")
       self.dbWrapper.urlUpdate(urlUpdateObj)
 
 

Here is the call graph for this function:

◆ resolveHTTP()

def dc_crawler.URLProcess.URLProcess.resolveHTTP	(	self,
		postForms,
		headersDict
	)

Definition at line 425 of file URLProcess.py.

   def resolveHTTP(self, postForms, headersDict):
     self.checkFieldsIsNone(["urlObj"])
     logger.debug("headersDict: %s", str(headersDict))
     postData = None
     try:
       method = self.urlObj.httpMethod.lower()
     except Exception:
       method = "get"
     if method == "post":
       postData = postForms
       logger.debug("use post, post_data:%s", postData)
 #    else:
 #      logger.debug("last modified: <<%s>>", str((self.urlObj.lastModified)))
 #      if str(self.urlObj.lastModified) != "None" and str(self.urlObj.lastModified) != "NULL":
 #        logger.debug("If-Modified-Since: <<%s>>", self.urlObj.lastModified)
 #        headersDict["If-Modified-Since"] = \
 #        Utils.convertToHttpDateFmt(datetime.datetime.strptime(str(self.urlObj.lastModified), "%Y-%m-%d %H:%M:%S"))
     return postData
 
 

Here is the call graph for this function:

◆ resolveTableName()

def dc_crawler.URLProcess.URLProcess.resolveTableName	(	self,
		localSiteId
	)

Definition at line 119 of file URLProcess.py.

   def resolveTableName(self, localSiteId):
     self.urlTable = self.DC_URLS_TABLE_PREFIX + localSiteId
     return self.urlTable
 
 

◆ setProtocols()

def dc_crawler.URLProcess.URLProcess.setProtocols	(	self,
		protocols = `None`
	)

Definition at line 74 of file URLProcess.py.

   def setProtocols(self, protocols=None):
     if protocols is not None:
       try:
         self.protocolsList = json.loads(protocols)
       except Exception:
         self.protocolsList = []
     logger.debug(">>> THAT PROTOCOLS = " + str(self.protocolsList))
 
 

◆ simpleURLCanonize()

def dc_crawler.URLProcess.URLProcess.simpleURLCanonize	(	self,
		realUrl
	)

Definition at line 145 of file URLProcess.py.

   def simpleURLCanonize(self, realUrl):
     self.checkFieldsIsNone(["url"])
     if not self.checkUrlByPath(self.url):
       self.url = urlparse.urljoin(realUrl, self.url)
       # normalization
     retUrl = Utils.UrlNormalizator.normalize(self.url, self.protocolsList, self.normMask)
     return retUrl
 
 

Here is the call graph for this function:

◆ updateAdditionProps()

def dc_crawler.URLProcess.URLProcess.updateAdditionProps	(	self,
		internalLinksCount,
		externalLinksCount,
		batchItem,
		size,
		freq,
		contentMd5
	)

Definition at line 787 of file URLProcess.py.

   def updateAdditionProps(self, internalLinksCount, externalLinksCount, batchItem, size, freq, contentMd5):
     if self.dbWrapper is not None:
       self.checkFieldsIsNone(["dbWrapper", "siteId"])
       urlUpdateObj = dc.EventObjects.URLUpdate(self.siteId, batchItem.urlId, dc.EventObjects.URLStatus.URL_TYPE_MD5, \
                                                normalizeMask=self.normMask)
       urlUpdateObj.tcDate = SQLExpression("NOW()")
       urlUpdateObj.size = size
       urlUpdateObj.linksI = internalLinksCount
       urlUpdateObj.linksE = externalLinksCount
       urlUpdateObj.freq = freq
       urlUpdateObj.rawContentMd5 = contentMd5
       self.dbWrapper.urlUpdate(urlUpdateObj)
 
 

Here is the call graph for this function:

◆ updateCollectTimeAndMime()

def dc_crawler.URLProcess.URLProcess.updateCollectTimeAndMime	(	self,
		detectedMime,
		batchItem,
		crawledTime,
		autoDetectMime,
		httpHeaders = `None`,
		strContent = `None`
	)

Definition at line 714 of file URLProcess.py.

                                strContent=None):
     if self.dbWrapper is not None:
       self.checkFieldsIsNone(["dbWrapper", "siteId"])
       if crawledTime is not None:
         collectTime = int((time.time() - crawledTime) * 1000)
       else:
         collectTime = 0
       urlUpdateObj = dc.EventObjects.URLUpdate(self.siteId, batchItem.urlId, dc.EventObjects.URLStatus.URL_TYPE_MD5, \
                                                normalizeMask=self.normMask)
       if strContent is not None:
         urlUpdateObj.rawContentMd5 = hashlib.md5(strContent).hexdigest()
       urlUpdateObj.crawlingTime = SQLExpression(("`CrawlingTime` + %s" % str(collectTime)))
       urlUpdateObj.totalTime = SQLExpression(("`TotalTime` + %s" % str(collectTime)))
       urlUpdateObj.tcDate = SQLExpression("NOW()")
       urlUpdateObj.UDate = SQLExpression("NOW()")
       logger.debug(">>> detectMime = " + str(detectedMime))
       if httpHeaders is not None:
         for header in httpHeaders:
           if header.lower() == "etag":
             # Simple get only first from several
             urlUpdateObj.eTag = httpHeaders[header].split(',')[0].strip("\"'")
       if detectedMime is not None and autoDetectMime is not None:
         urlUpdateObj.contentType = str(detectedMime)
       self.dbWrapper.urlUpdate(urlUpdateObj)
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ updateCrawledURL()

def dc_crawler.URLProcess.URLProcess.updateCrawledURL	(	self,
		crawledResource,
		batchItem,
		contentSize,
		status = `dc.EventObjects.URL.STATUS_CRAWLED`
	)

Definition at line 448 of file URLProcess.py.

   def updateCrawledURL(self, crawledResource, batchItem, contentSize, status=dc.EventObjects.URL.STATUS_CRAWLED):
 
     if self.dbWrapper is not None:
       self.checkFieldsIsNone(["urlObj", "dbWrapper", "siteId"])
       logger.debug(">>> Start urls update")
 
       updatedCount = self.urlObj.mRate * self.urlObj.mRateCounter
       if crawledResource.http_code != 304:
         updatedCount += 1
       mrate = updatedCount / (self.urlObj.mRateCounter + 1)
 
 
       urlUpdateObj = dc.EventObjects.URLUpdate(self.siteId, batchItem.urlId, dc.EventObjects.URLStatus.URL_TYPE_MD5, \
                                                normalizeMask=self.normMask)
 
       urlUpdateObj.contentType = batchItem.urlObj.contentType
       urlUpdateObj.charset = batchItem.urlObj.charset
       urlUpdateObj.errorMask = batchItem.urlObj.errorMask
       urlUpdateObj.crawlingTime = batchItem.urlObj.crawlingTime
       urlUpdateObj.totalTime = batchItem.urlObj.crawlingTime
       urlUpdateObj.httpCode = batchItem.urlObj.httpCode
 
       urlUpdateObj.status = batchItem.urlObj.status = status
       urlUpdateObj.size = batchItem.urlObj.size = contentSize
       urlUpdateObj.mRate = batchItem.urlObj.mRate = mrate
 
       batchItem.urlObj.UDate = batchItem.urlObj.tcDate = str(datetime.datetime.now())
       urlUpdateObj.UDate = urlUpdateObj.tcDate = SQLExpression("NOW()")
       batchItem.urlObj.mRateCounter += 1
       urlUpdateObj.mRateCounter = SQLExpression("`MRateCounter` + 1")
       urlUpdateObj.lastModified = batchItem.urlObj.lastModified = crawledResource.last_modified
       urlUpdateObj.urlMd5 = batchItem.urlObj.urlMd5
 
       if APP_CONSTS.SQL_EXPRESSION_FIELDS_UPDATE_CRAWLER in self.siteProperties:
         # Evaluate URL class values if neccessary
         changedFieldsDict = FieldsSQLExpressionEvaluator.execute(self.siteProperties, self.dbWrapper, None,
                                                                  batchItem.urlObj, logger,
                                                                  APP_CONSTS.SQL_EXPRESSION_FIELDS_UPDATE_CRAWLER)
         # Update URL values
         if changedFieldsDict is not None:
           for name, value in changedFieldsDict.items():
             if hasattr(urlUpdateObj, name):
               setattr(urlUpdateObj, name, value)
 
       logger.debug("!!! Before self.dbWrapper.urlUpdate(urlUpdateObj, \"`Status` = 3\")")
       affectDB = self.dbWrapper.affect_db
       self.dbWrapper.affect_db = True
       updatedRowsCount = self.dbWrapper.urlUpdate(urlUpdateObj, "`Status` = 3")
       self.dbWrapper.affect_db = affectDB
       logger.debug("!!! updatedRowsCount = " + str(updatedRowsCount))
 
 

Here is the call graph for this function:

◆ updateTypeForURLObjects()

def dc_crawler.URLProcess.URLProcess.updateTypeForURLObjects	(	self,
		urlObjects,
		typeArg = `dc.EventObjects.URL.TYPE_CHAIN`
	)

Definition at line 855 of file URLProcess.py.

   def updateTypeForURLObjects(self, urlObjects, typeArg=dc.EventObjects.URL.TYPE_CHAIN):
     if self.dbWrapper is not None:
       updateUrlObjects = []
       for urlObject in urlObjects:
         localUrlObject = dc.EventObjects.URLUpdate(urlObject.siteId, urlObject.url, normalizeMask=self.normMask)
         localUrlObject.urlMd5 = urlObject.urlMd5
         localUrlObject.type = typeArg
         updateUrlObjects.append(localUrlObject)
       if len(updateUrlObjects) > 0:
         self.dbWrapper.urlUpdate(updateUrlObjects)
 
 
 

◆ updateURL()

def dc_crawler.URLProcess.URLProcess.updateURL	(	self,
		batchItem,
		batchId,
		status = `dc.EventObjects.URL.STATUS_CRAWLING`
	)

Definition at line 505 of file URLProcess.py.

   def updateURL(self, batchItem, batchId, status=dc.EventObjects.URL.STATUS_CRAWLING):
 
     if self.dbWrapper is not None:
       self.checkFieldsIsNone(["urlObj", "dbWrapper", "siteId"])
       urlUpdateObj = dc.EventObjects.URLUpdate(self.siteId, batchItem.urlId, dc.EventObjects.URLStatus.URL_TYPE_MD5, \
                                                normalizeMask=self.normMask)
       urlUpdateObj.batchId = batchId
       if not self.urlObj.httpMethod:
         urlUpdateObj.httpMethod = batchItem.urlObj.httpMethod = "get"
       else:
         urlUpdateObj.httpMethod = batchItem.urlObj.httpMethod = self.urlObj.httpMethod
 
       urlUpdateObj.status = batchItem.urlObj.status = status
       batchItem.urlObj.crawled += 1
       urlUpdateObj.crawled = SQLExpression("`Crawled`+1")
       urlUpdateObj.tcDate = batchItem.urlObj.tcDate = SQLExpression("NOW()")
       urlUpdateObj.UDate = batchItem.urlObj.UDate = SQLExpression("NOW()")
 
       if status == dc.EventObjects.URL.STATUS_CRAWLING:
         
         urlUpdateObj.contentType = batchItem.urlObj.contentType = dc.EventObjects.URL.CONTENT_TYPE_UNDEFINED
         
         urlUpdateObj.httpCode = batchItem.urlObj.httpCode = 0
 
       updatedRowsCount = self.dbWrapper.urlUpdate(urlUpdateObj)
       logger.debug("!!! updatedRowsCount = " + str(updatedRowsCount))
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ updateURLFields()

def dc_crawler.URLProcess.URLProcess.updateURLFields	(	self,
		urlMd5,
		wrapper,
		siteId
	)

Definition at line 242 of file URLProcess.py.

   def updateURLFields(self, urlMd5, wrapper, siteId):
     urlUpdateObj = dc.EventObjects.URLUpdate(siteId, urlMd5, dc.EventObjects.URLStatus.URL_TYPE_MD5, \
                                              normalizeMask=self.normMask)
     if self.siteProperties is None:
       self.siteProperties = {}
 
     logger.debug('!!!!!! updateURLFields !!! self.siteProperties: ' + str(self.siteProperties))
     # Status field
     if "RECRAWL_URL_UPDATE_STATUS" in self.siteProperties and self.siteProperties["RECRAWL_URL_UPDATE_STATUS"] != "-1":
       urlUpdateObj.status = int(self.siteProperties["RECRAWL_URL_UPDATE_STATUS"])
     else:
       if "RECRAWL_URL_UPDATE_STATUS" in self.siteProperties and \
       self.siteProperties["RECRAWL_URL_UPDATE_STATUS"] == "-1":
         urlUpdateObj.status = None
       else:
         urlUpdateObj.status = dc.EventObjects.URL.STATUS_NEW
 
     # TcDate field
     if "RECRAWL_URL_UPDATE_TCDATE" in self.siteProperties and self.siteProperties["RECRAWL_URL_UPDATE_TCDATE"] != "":
       urlUpdateObj.tcDate = self.siteProperties["RECRAWL_URL_UPDATE_TCDATE"]
     else:
       if "RECRAWL_URL_UPDATE_TCDATE" in self.siteProperties and self.siteProperties["RECRAWL_URL_UPDATE_TCDATE"] == "":
         urlUpdateObj.tcDate = None
       else:
         urlUpdateObj.tcDate = SQLExpression("NOW()")
 
     # CDate field
     if "RECRAWL_URL_UPDATE_CDATE" in self.siteProperties and self.siteProperties["RECRAWL_URL_UPDATE_CDATE"] != "":
       urlUpdateObj.CDate = self.siteProperties["RECRAWL_URL_UPDATE_CDATE"]
 
     # UDate field
     if "RECRAWL_URL_UPDATE_UDATE" in self.siteProperties and self.siteProperties["RECRAWL_URL_UPDATE_UDATE"] != "":
       urlUpdateObj.UDate = self.siteProperties["RECRAWL_URL_UPDATE_UDATE"]
     else:
       if "RECRAWL_URL_UPDATE_UDATE" in self.siteProperties and self.siteProperties["RECRAWL_URL_UPDATE_UDATE"] == "":
         urlUpdateObj.UDate = None
       else:
         urlUpdateObj.UDate = SQLExpression("NOW()")
 
     # Recrawl url update
     if "RECRAWL_URL_UPDATE" in self.siteProperties and self.siteProperties["RECRAWL_URL_UPDATE"] != "":
       self.recrawlUrlUpdateHandler(wrapper, self.siteProperties["RECRAWL_URL_UPDATE"], urlUpdateObj)
 
     if wrapper is not None:
       saveAffectDB = wrapper.affect_db
       wrapper.affect_db = True
       wrapper.urlUpdate(urlUpdateObj, "`State`=0")
       wrapper.affect_db = saveAffectDB
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ updateURLForFailed()

def dc_crawler.URLProcess.URLProcess.updateURLForFailed	(	self,
		errorBit,
		batchItem,
		httpCode = `CONSTS.HTTP_CODE_400`,
		status = `dc.EventObjects.URL.STATUS_CRAWLED`,
		updateUdate = `True`
	)

Definition at line 375 of file URLProcess.py.

                          status=dc.EventObjects.URL.STATUS_CRAWLED, updateUdate=True):
     if self.dbWrapper is not None:
       self.checkFieldsIsNone(["dbWrapper", "siteId"])
       logger.debug("Set errorBit = " + str(errorBit) + ", httpCode = " + str(httpCode))
       urlUpdateObj = dc.EventObjects.URLUpdate(self.siteId, batchItem.urlId, dc.EventObjects.URLStatus.URL_TYPE_MD5, \
                                                normalizeMask=self.normMask)
 
       batchItem.urlObj.errorMask = batchItem.urlObj.errorMask | errorBit
       urlUpdateObj.errorMask = SQLExpression("`ErrorMask` | " + str(errorBit))
 
       urlUpdateObj.status = batchItem.urlObj.status = status
       urlUpdateObj.tcDate = batchItem.urlObj.tcDate = SQLExpression("NOW()")
       if updateUdate:
         urlUpdateObj.UDate = batchItem.urlObj.UDate = SQLExpression("NOW()")
 
       if httpCode is not None:
         urlUpdateObj.httpCode = batchItem.urlObj.httpCode = httpCode
         self.urlObj.httpCode = httpCode  # #???
 
       if self.dbWrapper is not None:
         # Evaluate URL class values if neccessary
         changedFieldsDict = FieldsSQLExpressionEvaluator.execute(self.siteProperties, self.dbWrapper, None,
                                                                  batchItem.urlObj, logger,
                                                                  APP_CONSTS.SQL_EXPRESSION_FIELDS_UPDATE_CRAWLER)
         # Update URL values
         for name, value in changedFieldsDict.items():
           if hasattr(urlUpdateObj, name):
             setattr(urlUpdateObj, name, value)
         urlUpdateObj.errorMask = SQLExpression("`ErrorMask` | " + str(errorBit))
 
         # Update URL data in DB
         self.dbWrapper.urlUpdate(urlUpdateObj)
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ updateURLStatus()

def dc_crawler.URLProcess.URLProcess.updateURLStatus	(	self,
		urlId,
		status = `dc.EventObjects.URL.STATUS_CRAWLED`
	)

Definition at line 537 of file URLProcess.py.

   def updateURLStatus(self, urlId, status=dc.EventObjects.URL.STATUS_CRAWLED):
     if status is not None and self.dbWrapper is not None:
       self.checkFieldsIsNone(["siteId"])
       urlUpdateObj = dc.EventObjects.URLUpdate(self.siteId, urlId, dc.EventObjects.URLStatus.URL_TYPE_MD5, \
                                                normalizeMask=self.normMask)
       urlUpdateObj.status = status
       updatedRowsCount = self.dbWrapper.urlUpdate(urlUpdateObj)
       logger.debug("!!! updatedRowsCount = " + str(updatedRowsCount))
 
 

Here is the call graph for this function:

◆ urlDBSync()

def dc_crawler.URLProcess.URLProcess.urlDBSync	(	self,
		batchItem,
		crawlerType,
		recrawlPeriod,
		autoRemoveProps
	)

Definition at line 746 of file URLProcess.py.

   def urlDBSync(self, batchItem, crawlerType, recrawlPeriod, autoRemoveProps):
     if self.dbWrapper is not None:
       self.checkFieldsIsNone(["dbWrapper", "siteId"])
       self.isUpdateCollection = False
       # Request for check exists url on here host
       sqlQuery = "SELECT COUNT(*) FROM `%s` WHERE `URLMd5` = '%s'" % \
                  (DB_CONSTS.DC_URLS_TABLE_NAME_TEMPLATE % self.siteId, batchItem.urlId)
       logger.debug("!!! urlDBSync sqlQuery: " + str(sqlQuery))
 
       result = self.dbWrapper.customRequest(sqlQuery, CrawlerTask.DB_URLS)
       logger.debug("!!! urlDBSync result: " + varDump(result))
 
       isExist = False
       if result is not None and len(result) > 0 and len(result[0]) > 0:
         logger.debug("!!! urlDBSync result[0][0]: " + str(result[0][0]) + " type: " + str(type(result[0][0])))
         isExist = bool(int(result[0][0]) > 0)
 
       try:
         if isExist:
           logger.debug("Url already exist in DB.")
         else:
           # When url come from another dc cluster's host it is not present in the db
           if self.addURLFromBatchToDB(batchItem, crawlerType, recrawlPeriod, autoRemoveProps):
             self.urlDBSync(batchItem, crawlerType, recrawlPeriod, autoRemoveProps)
           else:
             msg = "Can't add url from batch."
             logger.debug(msg)
             raise SyncronizeException(msg)
       except SyncronizeException, err:
         logger.debug("Can't synchronize url with db: " + str(err))
         raise err
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ urlTemplateApply()

def dc_crawler.URLProcess.URLProcess.urlTemplateApply	(	self,
		url,
		crawlerType,
		urlTempalteRegular,
		urlTempalteRealtime,
		urlTempalteRegularEncode,
		urlTempalteRealtimeEncode
	)

Definition at line 986 of file URLProcess.py.

                        urlTempalteRealtimeEncode):
     ret = url
     if crawlerType == dc.EventObjects.Batch.TYPE_REAL_TIME_CRAWLER:
       if urlTempalteRealtime is not None:
         try:
           if urlTempalteRealtimeEncode is not None and bool(int(urlTempalteRealtimeEncode)):
             encodedUrl = urllib.quote(url)
           else:
             encodedUrl = url
         except ValueError:
           encodedUrl = url
         ret = urlTempalteRealtime.replace(self.URL_TEMPLATE_CONST, encodedUrl)
     else:
       if urlTempalteRegular is not None:
         try:
           if urlTempalteRegularEncode is not None and bool(int(urlTempalteRegularEncode)):
             encodedUrl = urllib.quote(url)
           else:
             encodedUrl = url
         except ValueError:
           encodedUrl = url
         ret = urlTempalteRegular.replace(self.URL_TEMPLATE_CONST, encodedUrl)
     if ret != url:
       logger.debug(">>> url was replaced ")
       logger.debug(">>> new url = " + ret)
     return ret
 
 

Here is the caller graph for this function:

Member Data Documentation

◆ dbWrapper

dc_crawler.URLProcess.URLProcess.dbWrapper

Definition at line 62 of file URLProcess.py.

◆ DC_URLS_TABLE_PREFIX

string dc_crawler.URLProcess.URLProcess.DC_URLS_TABLE_PREFIX = "urls_"

static

Definition at line 50 of file URLProcess.py.

◆ DEFAULT_PROTOCOLS

list dc_crawler.URLProcess.URLProcess.DEFAULT_PROTOCOLS = ["http", "https"]

static

Definition at line 55 of file URLProcess.py.

◆ DETECT_MIME_TIMEOUT

int dc_crawler.URLProcess.URLProcess.DETECT_MIME_TIMEOUT = 1

static

Definition at line 51 of file URLProcess.py.

◆ isUpdateCollection

dc_crawler.URLProcess.URLProcess.isUpdateCollection

Definition at line 59 of file URLProcess.py.

◆ normMask

dc_crawler.URLProcess.URLProcess.normMask

Definition at line 69 of file URLProcess.py.

◆ PATTERN_WITH_PROTOCOL

dc_crawler.URLProcess.URLProcess.PATTERN_WITH_PROTOCOL = re.compile('[a-zA-Z]+:(//)?')

static

Definition at line 52 of file URLProcess.py.

◆ PROTOCOL_PREFIX

string dc_crawler.URLProcess.URLProcess.PROTOCOL_PREFIX = "://"

static

Definition at line 54 of file URLProcess.py.

◆ protocolsList

dc_crawler.URLProcess.URLProcess.protocolsList

Definition at line 66 of file URLProcess.py.

◆ site

dc_crawler.URLProcess.URLProcess.site

Definition at line 64 of file URLProcess.py.

◆ siteId

dc_crawler.URLProcess.URLProcess.siteId

Definition at line 63 of file URLProcess.py.

◆ siteProperties

dc_crawler.URLProcess.URLProcess.siteProperties

Definition at line 67 of file URLProcess.py.

◆ url

dc_crawler.URLProcess.URLProcess.url

Definition at line 61 of file URLProcess.py.

◆ URL_TEMPLATE_CONST

string dc_crawler.URLProcess.URLProcess.URL_TEMPLATE_CONST = "%URL%"

static

Definition at line 53 of file URLProcess.py.

◆ urlObj

dc_crawler.URLProcess.URLProcess.urlObj

Definition at line 60 of file URLProcess.py.

◆ urlTable

dc_crawler.URLProcess.URLProcess.urlTable

Definition at line 65 of file URLProcess.py.

The documentation for this class was generated from the following file:

sources/hce/dc_crawler/URLProcess.py

Public Member Functions

Static Public Member Functions

Public Attributes

Static Public Attributes

Detailed Description

Constructor & Destructor Documentation

◆ __init__()

Member Function Documentation

◆ additionalUrlObjInit()

◆ addURLFromBatchToDB()

◆ autoRemoveURL()

◆ checkDictEmptyStrings()

◆ checkFieldsIsNone()

◆ checkUrlByPath()

◆ checkUrlByProtocol()

◆ conditionEvaluate()

◆ createUrlObjForChain()

◆ createUrlObjForCollectURLs()

◆ detectUrlMime()

◆ fillRssFieldInUrlObj()

◆ fillRssFieldOneElem()

◆ getDepthFromUrl()

◆ getRealUrl()

◆ isUrlExist()

◆ processURL()

◆ readCurrentCnt()

◆ recrawlUrlUpdateHandler()

◆ resetErrorMask()

◆ resolveHTTP()

◆ resolveTableName()

◆ setProtocols()

◆ simpleURLCanonize()

◆ updateAdditionProps()

◆ updateCollectTimeAndMime()

◆ updateCrawledURL()

◆ updateTypeForURLObjects()

◆ updateURL()

◆ updateURLFields()

◆ updateURLForFailed()

◆ updateURLStatus()

◆ urlDBSync()

◆ urlTemplateApply()

Member Data Documentation

◆ dbWrapper

◆ DC_URLS_TABLE_PREFIX

◆ DEFAULT_PROTOCOLS

◆ DETECT_MIME_TIMEOUT

◆ isUpdateCollection

◆ normMask

◆ PATTERN_WITH_PROTOCOL

◆ PROTOCOL_PREFIX

◆ protocolsList

◆ site

◆ siteId

◆ siteProperties

◆ url

◆ URL_TEMPLATE_CONST

◆ urlObj

◆ urlTable

◆ init()