Inheritance diagram for dc_crawler.CollectURLs.CollectURLs:

Collaboration diagram for dc_crawler.CollectURLs.CollectURLs:

Public Member Functions
def	__init__ (self, isAbortedByTTL=None)

def	checkFieldsIsNone (self)

def	process (self, httpCode, readOnly=False, httpApplyHeaders=None, proxyName=None)

def	feedElementsProcessing (self, urlMd5, httpCode, elemUrl, localSiteId, localUrlObj, localUrl, params, maxURLsFromPage, rootFeed=False)

def	processProcessor (self, urlSet, dom, urlXpathList, urlObj)

def	evaluateDateMacro (self, localPattern, dateFromat)

def	extractFormURL (self, dom, siteProperties)

def	getFieldParams (self, formFields, postForms, siteId)

def	insertNewSiteProperties (self, params, wrapper, siteId)

def	feedparserParseDateFixes (self, aDateString)

Static Public Member Functions
def	filtersApply (inputFilters, subject, depth, wrapper, siteId, fields=None, opCode=Filters.OC_RE, stage=Filters.STAGE_COLLECT_URLS, selectSubject=None, defaultValue=False)

Public Attributes
	crawledResource

	url

	dom

	realUrl

	baseUrl

	processorName

	batchItem

	urlXpathList

	feedItems

	feed

	siteProperties

	site

	dbWrapper

	autoRemoveProps

	autoDetectMime

	processContentTypes

	postForms

	urlProcess

	robotsParser

	urlsXpathList

	isAbortedByTTL

Static Public Attributes
	BINARY_CONTENT_TYPE_PATTERN = re.compile('(text)\|(xml)', re.I)

string	COLLECT_POST_DATA_NAME = "COLLECT_POST_DATA"

string	COLLECT_POST_DATA = "1"

string	DC_URLS_TABLE_PREFIX = "urls_"

	PATTERN_WITH_PROTOCOL = re.compile('[a-zA-Z]+:(//)?')

string	DETECT_MIME_MAIN_CONTENT = "1"

string	DETECT_MIME_COLLECTED_URL = "2"

int	DETECT_MIME_TIMEOUT = 1

Private Member Functions
def	_normalize_attributes (self, kv)

Detailed Description

Definition at line 38 of file CollectURLs.py.

Constructor & Destructor Documentation

◆ init()

def dc_crawler.CollectURLs.CollectURLs.__init__	(	self,
		isAbortedByTTL = `None`
	)

Definition at line 50 of file CollectURLs.py.

   def __init__(self, isAbortedByTTL=None):
     self.crawledResource = None
     self.url = None
     self.dom = None
     self.realUrl = None
     self.baseUrl = None
     self.processorName = None
     self.batchItem = None
     self.urlXpathList = None
     self.feedItems = None
     self.feed = None
     self.siteProperties = None
     self.site = None
     self.dbWrapper = None
     self.autoRemoveProps = None
     self.autoDetectMime = None
     self.processContentTypes = []
     self.postForms = None
     self.urlProcess = None
     self.robotsParser = None
     self.urlsXpathList = []
     self.isAbortedByTTL = (lambda: False) if isAbortedByTTL is None else isAbortedByTTL
 
 

Member Function Documentation

◆ _normalize_attributes()

def dc_crawler.CollectURLs.CollectURLs._normalize_attributes	(	self,
		kv
	)

private

Definition at line 730 of file CollectURLs.py.

730 def _normalize_attributes(self, kv):

731 return (kv[0], kv[1])

732

Here is the caller graph for this function:

◆ checkFieldsIsNone()

def dc_crawler.CollectURLs.CollectURLs.checkFieldsIsNone ( self )

Definition at line 76 of file CollectURLs.py.

   def checkFieldsIsNone(self):
     excludeList = ['feedItems', 'feed', 'processorName', 'autoDetectMime', 'processContentTypes',
                    'postForms', 'robotsParser', 'dom', 'dbWrapper']
     for field in self.__dict__:
       if field not in excludeList and (not hasattr(self, field) or getattr(self, field) is None):
         msg = "Mandatory field must be initialized, field Name = " + field
         logger.error(msg)
         raise Exception(msg)
 
 

Here is the caller graph for this function:

◆ evaluateDateMacro()

def dc_crawler.CollectURLs.CollectURLs.evaluateDateMacro	(	self,
		localPattern,
		dateFromat
	)

Definition at line 565 of file CollectURLs.py.

   def evaluateDateMacro(self, localPattern, dateFromat):
     import time
     import datetime
     try:
       d = {'DATE':'', 'SHORTYEAR':'y', 'YEAR':'Y', 'MONTH':'m', 'DAY':'d', 'HOUR':'H', 'MINUTE':'M', 'SECOND':'S'}
       regex = re.compile("%@(SHORTYEAR|YEAR|MONTH|DAY|HOUR|MINUTE|SECOND|DATE)\\(([\\+|\\-]\\d{1,2})\\)%")
       matchArray = regex.findall(localPattern)
       for i in matchArray:
         if i[0] == 'DATE':
           f = dateFromat
         else:
           f = '%' + d[i[0]]
         t = time.strftime(f, time.gmtime(time.time() + datetime.timedelta(hours=(+int(i[1]))).seconds))
         localPattern = localPattern.replace("%@" + i[0] + "(" + i[1] + ")%", t)
     except Exception, err:
       logger.error(str(err))
 
     return localPattern
 
 

Here is the caller graph for this function:

◆ extractFormURL()

def dc_crawler.CollectURLs.CollectURLs.extractFormURL	(	self,
		dom,
		siteProperties
	)

Definition at line 591 of file CollectURLs.py.

   def extractFormURL(self, dom, siteProperties):
     formUrls, formMethods, formFields = [], {}, {}
     if self.COLLECT_POST_DATA_NAME in siteProperties and \
     siteProperties['COLLECT_POST_DATA'] == self.COLLECT_POST_DATA:
       for form in dom.xpath("//form"):
         formAction = None
         formMethod = 'get'
         for attr in form.keys():
           if attr.lower() == "action":
             formAction = form.get(attr)
             formUrls.append(formAction)
           elif attr.lower() == "method":
             formMethod = form.get(attr)
         if not formAction:
           continue
         formMethods[formAction] = formMethod
         for field in form.getchildren():
           tagName, tagValue = None, ""
           for fieldTag in field.keys():
             if fieldTag.lower() == "name":
               tagName = field.get(fieldTag)
             elif fieldTag.lower() == "value":
               tagValue = field.get(fieldTag)
           if tagName:
             formFields[tagName] = tagValue
       logger.info("extracted form data, formUrls:%s, formMethods:%s, formFields:%s", \
       formUrls, formMethods, formFields)
     return formUrls, formMethods, formFields
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ feedElementsProcessing()

def dc_crawler.CollectURLs.CollectURLs.feedElementsProcessing	(	self,
		urlMd5,
		httpCode,
		elemUrl,
		localSiteId,
		localUrlObj,
		localUrl,
		params,
		maxURLsFromPage,
		rootFeed = `False`
	)

Definition at line 441 of file CollectURLs.py.

                              maxURLsFromPage, rootFeed=False):
 
     if maxURLsFromPage > 0 and len(self.feedItems) >= maxURLsFromPage:
       logger.debug("Site maxURLsFromPage = %s limit reached on %s number.",
                    str(maxURLsFromPage), str(len(self.feedItems)))
     else:
       if self.feed is not None:
         self.urlProcess.url = elemUrl
         self.urlProcess.siteId = localSiteId
         self.urlProcess.urlObj = localUrlObj
         localRet = self.urlProcess.fillRssFieldInUrlObj(localUrl, self.url.url, self.batchItem, self.processorName,
                                                         self.feed, rootFeed)
         self.urlProcess.urlObj = None
         if localRet is not None:
           localRet["urlMd5"] = urlMd5
           if localRet["urlObj"] is not None:
             localRet["urlObj"].httpCode = httpCode
             localRet["urlObj"].processingDelay = 0
             localRet["urlObj"].parentMd5 = self.url.urlMd5
 
             # logger.debug("localRet = %s", str(dict(localRet)))
 
             params.append(localRet["urlObj"])
             self.feedItems.append(localRet)
       else:
         logger.debug("self.feed is None!")
 

Here is the caller graph for this function:

◆ feedparserParseDateFixes()

def dc_crawler.CollectURLs.CollectURLs.feedparserParseDateFixes	(	self,
		aDateString
	)

Definition at line 715 of file CollectURLs.py.

   def feedparserParseDateFixes(self, aDateString):
     ret = None
     ds = aDateString
 
     # Assumes that date format broken and contains the semicolon ":" in TZ like: "Wed, 19 Aug 2015 08:45:53 +01:00"
     parts = ds.split(' ')
     if ("+" in parts[len(parts) - 1] or "-" in parts[len(parts) - 1]) and ":" in parts[len(parts) - 1]:
       parts[len(parts) - 1] = parts[len(parts) - 1].replace(":", "")
       ds = " ".join(parts)
       # ret = feedparser._parse_date_rfc822(ds)
       ret = feedparser._parse_date(ds)  # pylint: disable=W0212
 
     return ret
 
 

Here is the caller graph for this function:

◆ filtersApply()

def dc_crawler.CollectURLs.CollectURLs.filtersApply	(	inputFilters,
		subject,
		depth,
		wrapper,
		siteId,
		fields = `None`,
		opCode = `Filters.OC_RE`,
		stage = `Filters.STAGE_COLLECT_URLS`,
		selectSubject = `None`,
		defaultValue = `False`
	)

static

Definition at line 636 of file CollectURLs.py.

                    stage=Filters.STAGE_COLLECT_URLS, selectSubject=None, defaultValue=False):
     ret = defaultValue
     fValue = Utils.generateReplacementDict()
     fValue.update({"MAX_DEPTH": str(depth)})
 
     if inputFilters is not None:
       for inputFilter in inputFilters:
         if inputFilter.stage == Filters.STAGE_ALL or inputFilter.stage == Filters.STAGE_REDIRECT_URL:
           inputFilter.stage = Filters.STAGE_COLLECT_URLS
 
 #     logger.debug(">>> Filters() (2.1) fields: " + varDump(fields) + " inputFilters: " + varDump(inputFilters))
     localFilters = Filters(filters=inputFilters, dbTaskWrapper=wrapper, siteId=siteId, readMode=0, fields=fields,
                            opCode=opCode, stage=stage, selectSubject=selectSubject)
 
     # logger.debug(">>> before filter include = " + subject[:255] + ' . . . ')
     fResult = localFilters.filterAll(stage, fValue, Filters.LOGIC_OR, subject, 1)
     logger.debug(">>> filter result include - " + str(fResult))
     for elem in fResult:
 #       logger.debug('elem = ' + str(elem) + ' type: ' + str(type(elem)))
       if elem > 0:
         ret = True
         break
 
     if ret is True:
       # logger.debug(">>> before filter exclude = " + subject[:255] + ' . . . ')
       fResult = localFilters.filterAll(stage, fValue, Filters.LOGIC_OR, subject, -1)
       logger.debug(">>> filter result exclude - " + str(fResult))
       for elem in fResult:
 #         logger.debug('elem = ' + str(elem) + ' type: ' + str(type(elem)))
         if elem > 0:
           ret = False
           break
 
     logger.debug("Verdict: " + str(ret))
     return ret
 
 

Here is the caller graph for this function:

◆ getFieldParams()

def dc_crawler.CollectURLs.CollectURLs.getFieldParams	(	self,
		formFields,
		postForms,
		siteId
	)

Definition at line 678 of file CollectURLs.py.

   def getFieldParams(self, formFields, postForms, siteId):
     ret = []
     for fieldName, fieldValue in formFields.iteritems():
       if fieldName in postForms:
         continue
       logger.debug("field_name: %s", fieldName)
       logger.debug("field_value: %s", fieldValue)
       ret.append((siteId, "HTTP_POST_FORM_" + fieldName, fieldValue))
     return ret
 
 

Here is the caller graph for this function:

◆ insertNewSiteProperties()

def dc_crawler.CollectURLs.CollectURLs.insertNewSiteProperties	(	self,
		params,
		wrapper,
		siteId
	)

Definition at line 694 of file CollectURLs.py.

   def insertNewSiteProperties(self, params, wrapper, siteId):
     if siteId is not None and hasattr(params, '__iter__') and len(params) > 0:
       localSiteUpdate = dc.EventObjects.SiteUpdate(siteId)
       for attr in localSiteUpdate.__dict__:
         if hasattr(localSiteUpdate, attr):
           setattr(localSiteUpdate, attr, None)
       localSiteUpdate.updateType = dc.EventObjects.SiteUpdate.UPDATE_TYPE_APPEND
       localSiteUpdate.id = siteId
       localSiteUpdate.properties = []
       for param in params:
         newPropElem = {}
         newPropElem["siteId"] = param[0]
         newPropElem["name"] = param[1]
         newPropElem["value"] = param[2]
         localSiteUpdate.properties.append(newPropElem)
       wrapper.siteNewOrUpdate(localSiteUpdate, stype=dc.EventObjects.SiteUpdate)
 
 

Here is the caller graph for this function:

◆ process()

def dc_crawler.CollectURLs.CollectURLs.process	(	self,
		httpCode,
		readOnly = `False`,
		httpApplyHeaders = `None`,
		proxyName = `None`
	)

Definition at line 92 of file CollectURLs.py.

   def process(self, httpCode, readOnly=False, httpApplyHeaders=None, proxyName=None):
 
     self.checkFieldsIsNone()
     if self.siteProperties is None:
       self.siteProperties = {}
     if self.processContentTypes is None:
       self.processContentTypes = []
     localSiteId = self.batchItem.siteId if self.batchItem.siteId else "0"
     nextStep = True
     useChains = False
     internalLinks, externalLinks = [], []
     maxURLsFromPage = 0
     params = []
     chainUrls = []
     formUrls = None
     formMethods = None
     formFields = None
     urlSet = set()
 
     logger.debug("!!! self.site.maxURLsFromPage = " + str(self.site.maxURLsFromPage))
     logger.debug("!!! self.url.maxURLsFromPage = " + str(self.url.maxURLsFromPage))
 
     if self.site is not None and self.site.maxURLsFromPage is not None:
       maxURLsFromPage = self.site.maxURLsFromPage
 
     if self.url is not None and self.url.maxURLsFromPage is not None and self.url.maxURLsFromPage > 0:
       maxURLsFromPage = self.url.maxURLsFromPage
 
     if nextStep and self.crawledResource is not None and \
     not self.BINARY_CONTENT_TYPE_PATTERN.search(self.crawledResource.content_type):
       nextStep = False
 
     # don't parse url for 4XX or 5XX response
     if nextStep and self.crawledResource is not None:
       code_type = int(self.crawledResource.http_code) / 100
       if code_type == 4 or code_type == 5:
         nextStep = False
 
     if nextStep and self.crawledResource is not None and not self.crawledResource.html_content:
       nextStep = False
 
     # if nextStep and self.dom is None:
     #  logger.debug("DOM is None")
     #  nextStep = False
 
     if nextStep:
       useChains = True
       if self.dom is not None:
         self.processProcessor(urlSet, self.dom, self.urlXpathList, self.batchItem.urlObj)
         formUrls, formMethods, formFields = self.extractFormURL(self.dom, self.siteProperties)
         urlSet.update(formUrls)
       else:
         logger.debug("DOM is None")
 
       if self.url.type == dc.EventObjects.URL.TYPE_SINGLE:
         logger.debug("URL type: single")
         nextStep = False
 
       if nextStep and self.crawledResource.dynamic_fetcher_result_type == \
         SeleniumFetcher.MACRO_RESULT_TYPE_URLS_LIST:
         ul = None
         try:
           ul = json.loads(self.crawledResource.html_content)
         except Exception, err:
           logger.error("Error deserialize macro data from result string: %s\n%s", str(err),
                        self.crawledResource.html_content)
         if ul is not None:
           logger.debug("Fill urlSet from macro results: %s items", str(len(ul)))
           if isinstance(ul, list):
             urlSet.update([u for u in ul if isinstance(u, basestring) and u != ''])
 
     if nextStep:
       if self.url.type == dc.EventObjects.URL.TYPE_CHAIN:
         logger.debug("URL type: chain")
         nextStep = False
 
     if nextStep:
       # (3) END
       urlTable = self.DC_URLS_TABLE_PREFIX + localSiteId
       self.urlProcess.urlTable = urlTable
       try:
         if self.siteProperties is not None and "RSS_FEED_ZERO_ITEM" in self.siteProperties and \
         int(self.siteProperties["RSS_FEED_ZERO_ITEM"]) == 1:
           if self.processorName == PCONSTS.PROCESSOR_FEED_PARSER or self.processorName == PCONSTS.PROCESSOR_RSS:
             self.feedElementsProcessing(self.url.urlMd5, httpCode, self.url.url, localSiteId, self.url, self.url.url,
                                         params, maxURLsFromPage, True)
       except ValueError:
         logger.debug(">>> Wrong \"RSS_FEED_ZERO_ITEM\" property's value")
 
       logger.debug("URLs candidates collected %s items:\n%s", str(len(urlSet)), str(urlSet))
 
       if self.site.maxURLs > 0 and len(urlSet) >= self.site.maxURLs:
         urlSet = set(list(urlSet)[:self.site.maxURLs])
         logger.debug("Site maxURLs = %s limit reached.", str(self.site.maxURLs))
 
       if self.site.maxResources > 0 and len(urlSet) >= self.site.maxResources:
         urlSet = set(list(urlSet)[:self.site.maxResources])
         logger.debug("Site maxResources = %s limit reached.", str(self.site.maxResources))
 
       countCnt = 0
       countErrors = 0
       for elemUrl in urlSet:
 
         if self.isAbortedByTTL():
           logger.debug("Aborted by TTL. All elements skipped.")
           break
 
         if elemUrl is None:
           logger.debug("Some url from urlSet is None, skipped.")
           continue
 
         elemUrl = elemUrl.strip()
         if elemUrl == '':
           logger.debug("Some url from urlSet is empty, skipped!")
           continue
 
         localUrl = elemUrl
         self.urlProcess.urlObj = self.url
         self.urlProcess.url = elemUrl
         self.urlProcess.dbWrapper = self.dbWrapper
         self.urlProcess.siteId = localSiteId
         retUrl, retContinue = self.urlProcess.processURL(self.realUrl, internalLinks, externalLinks, self.filtersApply,
                                                          None, self.baseUrl)
         if retUrl is not None:
           elemUrl = retUrl
           elemUrl = UrlNormalize.execute(siteProperties=self.siteProperties, base=self.baseUrl, url=elemUrl, supportProtocols=None, log=logger)
         else:
           retContinue = True
 
         if retContinue:
           logger.debug("Candidate URL is not passed general checks, skipped: %s", str(elemUrl))
           continue
 
         # Apply filter to Url
         if not self.filtersApply(inputFilters=self.site.filters,
                                  subject=elemUrl,
                                  depth=self.url.depth,
                                  wrapper=self.dbWrapper,
                                  siteId=localSiteId):
           logger.debug("Candidate URL not matched filters, skipped.")
           continue
         else:
           logger.debug("Candidate URL matched filters.")
 
         # Check exist of the Url
         urlMd5 = hashlib.md5(elemUrl).hexdigest()
         self.urlProcess.url = elemUrl
         self.urlProcess.siteId = localSiteId
         self.urlProcess.urlTable = urlTable
         if self.urlProcess.isUrlExist(self.site.recrawlPeriod, urlMd5):
           logger.debug("Candidate URL %s already exist, skipped.", str(urlMd5))
           continue
 
         if self.site.maxURLs > 0:
           if httpCode == CRAWLER_CONSTS.HTTP_CODE_200:
             countCnt += 1
           else:
             countErrors += 1
 
           if self.dbWrapper is not None:
             currentCnt = self.urlProcess.readCurrentCnt(self.site.maxURLs)
             if currentCnt >= self.site.maxURLs or countCnt >= self.site.maxURLs or \
               (countCnt + countErrors) >= self.site.maxURLs:
               logger.debug("Site MaxURLs: %s limit is reached. countCnt = %s, currentCnt = %s",
                            str(self.site.maxURLs), str(countCnt), str(currentCnt))
               autoremovedURLs = self.urlProcess.autoRemoveURL(self.autoRemoveProps, self.site.recrawlPeriod, urlTable,
                                                               self.dbWrapper)
               if autoremovedURLs == 0:
                 logger.debug("No one URL auto removed, candidate URL skipped!")
                 continue
               else:
                 logger.debug("%s URLs auto removed.", str(autoremovedURLs))
 
             if currentCnt >= self.site.maxResources or countCnt >= self.site.maxResources or \
               (countCnt + countErrors) >= self.site.maxResources:
               logger.debug("Site maxResources = %s limit is reached. countCnt = %s, currentCnt = %s",
                            str(self.site.maxResources), str(countCnt), str(currentCnt))
               autoremovedURLs = self.urlProcess.autoRemoveURL(self.autoRemoveProps, self.site.recrawlPeriod, urlTable,
                                                               self.dbWrapper)
               if autoremovedURLs == 0:
                 logger.debug("No one URL auto removed, candidate URL skipped!")
                 continue
               else:
                 logger.debug("%s URLs auto removed.", str(autoremovedURLs))
 
         # detect collected url mime type and ignore non-match URL
         # (7) Detect collected url mime type and ignore non-match URL
         detectedMime = ''
         if self.autoDetectMime == self.DETECT_MIME_COLLECTED_URL and self.processContentTypes is not None:
           self.urlProcess.url = elemUrl
           detectedMime = self.urlProcess.detectUrlMime(self.siteProperties["CONTENT_TYPE_MAP"] if \
                                                        "CONTENT_TYPE_MAP" in self.siteProperties else None)
           if detectedMime not in self.processContentTypes:
             logger.debug("Candidate URL MIME type is not matched, skipped!")
             continue
         # (7) END
 
         if "ROBOTS_COLLECT" not in self.siteProperties or int(self.siteProperties["ROBOTS_COLLECT"]) > 0:
           logger.debug("Robots.txt obey mode is ON")
           if self.robotsParser and self.robotsParser.loadRobots(elemUrl, self.batchItem.siteId, httpApplyHeaders,
                                                                 proxyName):
             isAllowed, retUserAgent = self.robotsParser.checkUrlByRobots(elemUrl, self.batchItem.siteId,
                                                                          httpApplyHeaders)
             if not isAllowed:
               logger.debug("URL " + elemUrl + " is NOT Allowed by user-agent:" + str(retUserAgent))
               self.urlProcess.updateURLForFailed(APP_CONSTS.ERROR_ROBOTS_NOT_ALLOW, self.batchItem)
               continue
 
         self.urlProcess.siteId = localSiteId
         depth = self.urlProcess.getDepthFromUrl(self.batchItem.urlId)
 
         # per project redirects resolving
         if "HTTP_REDIRECT_RESOLVER" in self.siteProperties and self.siteProperties["HTTP_REDIRECT_RESOLVER"] != "":
           logger.debug('!!!!!! HTTP_REDIRECT_RESOLVER !!!!! ')
 
 
           if "CONNECTION_TIMEOUT" in self.siteProperties:
             connectionTimeout = float(self.siteProperties["CONNECTION_TIMEOUT"])
           else:
             connectionTimeout = CRAWLER_CONSTS.CONNECTION_TIMEOUT
 
           tm = int(self.url.httpTimeout) / 1000.0
           if isinstance(self.url.httpTimeout, float):
             tm += float('0' + str(self.url.httpTimeout).strip()[str(self.url.httpTimeout).strip().find('.'):])
 
           proxies = {"http": "http://" + proxyName} if proxyName is not None else None
 
           auth = None
           if 'HTTP_AUTH_NAME' in self.siteProperties and 'HTTP_AUTH_PWD' in self.siteProperties:
             authName = self.siteProperties['HTTP_AUTH_NAME']
             authPwd = self.siteProperties['HTTP_AUTH_PWD']
             if authName is not None and authPwd is not None:
               auth = (authName, authPwd)
 
           postForms = {}
           for key in self.siteProperties.keys():
             if key.startswith('HTTP_POST_FORM_'):
               postForms[key[len('HTTP_POST_FORM_'):]] = self.siteProperties[key]
           postData = self.urlProcess.resolveHTTP(postForms, httpApplyHeaders)
 
           maxRedirects = HTTPRedirectResolver.RedirectProperty.DEFAULT_VALUE_MAX_REDIRECTS
           if 'HTTP_REDIRECTS_MAX' in self.siteProperties:
             maxRedirects = int(self.siteProperties['HTTP_REDIRECTS_MAX'])
 
           redirectResolver = HTTPRedirectResolver(propertyString=self.siteProperties["HTTP_REDIRECT_RESOLVER"],
                                                   fetchType=self.site.fetchType,
                                                   dbWrapper=self.dbWrapper,
                                                   siteId=localSiteId,
                                                   connectionTimeout=connectionTimeout)
 
           resUrl = redirectResolver.resolveRedirectUrl(url=elemUrl,
                                                        headers=httpApplyHeaders,
                                                        timeout=tm,
                                                        allowRedirects=True,
                                                        proxies=proxies,
                                                        auth=auth,
                                                        postData=postData,
                                                        maxRedirects=maxRedirects,
                                                        filters=self.site.filters)
 
           logger.debug("Resolved url: %s", str(resUrl))
           elemUrl = resUrl
 
         if elemUrl is not None:
           self.urlProcess.url = elemUrl
           self.urlProcess.siteId = localSiteId
           self.urlProcess.urlObj = self.url
           localUrlObj = self.urlProcess.createUrlObjForCollectURLs(urlMd5, formMethods, self.batchItem.urlId, depth,
                                                                    detectedMime, self.site.maxURLsFromPage)
           # update counters of external and internal links
           localUrlObj.linksI = len(internalLinks)
           localUrlObj.linksE = len(externalLinks)
 
           if self.processorName == PCONSTS.PROCESSOR_FEED_PARSER or self.processorName == PCONSTS.PROCESSOR_RSS:
             self.feedElementsProcessing(urlMd5, httpCode, elemUrl, localSiteId, localUrlObj, localUrl, params,
                                         maxURLsFromPage)
           else:
             params.append(localUrlObj)
 
     if useChains and "URL_CHAIN" in self.siteProperties and self.siteProperties["URL_CHAIN"] is not None:
       localChainDict = json.loads(self.siteProperties["URL_CHAIN"])
       depth = self.urlProcess.getDepthFromUrl(self.batchItem.urlId)
       if "url_pattern" in localChainDict:
         for elemUrl in urlSet:
           if elemUrl is None:
             logger.debug("Some url from urlSet is None")
             continue
           self.urlProcess.url = elemUrl
 #           retUrl = self.urlProcess.simpleURLCanonize(self.realUrl)
 #           if retUrl is None or not UrlNormalizator.isNormalUrl(retUrl):
 
           retUrl = urlNormalization(self.baseUrl, elemUrl)
           if retUrl is None:
             logger.debug("Bad url normalization, url: %s", retUrl)
             continue
           else:
             elemUrl = retUrl
           detectedMime = ''
           if self.autoDetectMime == self.DETECT_MIME_COLLECTED_URL and self.processContentTypes is not None:
             self.urlProcess.url = elemUrl
             detectedMime = self.urlProcess.detectUrlMime(self.siteProperties["CONTENT_TYPE_MAP"] if \
                                                          "CONTENT_TYPE_MAP" in self.siteProperties else None, \
                                                          self.batchItem.urlObj)
           urlMd5 = hashlib.md5(elemUrl).hexdigest()
           self.urlProcess.url = elemUrl
           self.urlProcess.siteId = localSiteId
           self.urlProcess.urlObj = self.url
           try:
             localUrlObj = self.urlProcess.\
                               createUrlObjForChain(localChainDict["url_pattern"], urlMd5, formMethods,
                                                    self.batchItem.urlId, depth, detectedMime, self.site.maxURLsFromPage)
             if localUrlObj is not None:
               chainUrls.append(copy.deepcopy(localUrlObj))
           except Exception as excp:
             logger.error("Error in URL_CHAIN deserialize, excp = " + str(excp))
     if len(urlSet) > 0 and len(params) == 0:
       logger.debug("Zero urls are collected for len(urlSet): %s", str(len(urlSet)))
     elif len(params) > 0:
       logger.debug("Collected and send to insert as new: %s", str(len(urlSet)))
     if not readOnly:
 
       if self.dbWrapper is not None:
         self.dbWrapper.urlNew(params)
         self.dbWrapper.urlNew(chainUrls)
         self.urlProcess.updateTypeForURLObjects(chainUrls)
         self.dbWrapper.collectedURLsRecalculating(localSiteId)
 
     if formFields is not None and self.postForms is not None and self.dbWrapper is not None:
       fieldParams = self.getFieldParams(formFields, self.postForms, localSiteId)
       self.insertNewSiteProperties(fieldParams, self.dbWrapper, localSiteId)
 
     # logger.debug("Return from collectURLs:\n%s\n%s\n%s\n%s\n%s\n%s", str(nextStep), str(internalLinks),
     #             str(externalLinks), str(params), str(self.feedItems), str(chainUrls))
 
     return nextStep, internalLinks, externalLinks, params, self.feedItems, chainUrls
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ processProcessor()

def dc_crawler.CollectURLs.CollectURLs.processProcessor	(	self,
		urlSet,
		dom,
		urlXpathList,
		urlObj
	)

Definition at line 474 of file CollectURLs.py.

   def processProcessor(self, urlSet, dom, urlXpathList, urlObj):
     if (self.processorName == PCONSTS.PROCESSOR_FEED_PARSER or self.processorName == PCONSTS.PROCESSOR_RSS) \
     and urlObj.type != dc.EventObjects.URL.TYPE_FETCHED:
       if feedparser is not None:
         try:
           self.feedItems = []
           # Add one more date parsing handler function to fix some wrong datetime format cases; added by bgv
           feedparser.registerDateHandler(self.feedparserParseDateFixes)
 
           # Remove handlers to process all tags as unknown to save their names unchanged
           if not (self.siteProperties is not None and "RSS_FEEDPARSER_MODE" in self.siteProperties and \
                   int(self.siteProperties["RSS_FEEDPARSER_MODE"]) > 0):
             import inspect
             # , "_start_guid"
             excludes = ["_start_rss", "_start_channel", "_start_feed", "_start_item", "_start_link",
                         "_start_admin_errorreportsto", "_start_admin_generatoragent", "_start_guid", "_start_id",
                         "_start_entry", "_start_enclosure"]
             for methodName, functionObject in inspect.getmembers(feedparser._FeedParserMixin, predicate=inspect.ismethod): # pylint: disable=W0612,W0212,C0301
               if methodName.startswith("_start_") and methodName not in excludes:
                 delattr(feedparser._FeedParserMixin, methodName)  # pylint: disable=W0212
                 endMethodName = methodName.replace("_start_", "_end_")
                 if hasattr(feedparser._FeedParserMixin, endMethodName):  # pylint: disable=W0212
                   delattr(feedparser._FeedParserMixin, endMethodName)  # pylint: disable=W0212
             setattr(feedparser._FeedParserMixin, "_normalize_attributes", self._normalize_attributes)  # pylint: disable=W0212
             feedparser.FeedParserDict.keymap["guid"] = "guid"
             logger.debug("Feedparser in modified mode")
           else:
             logger.debug("Feedparser in native mode")
 
           self.feed = feedparser.parse(self.crawledResource.html_content)
           urlSet.update(entry.link for entry in self.feed.entries)
           # logger.debug("feed.entries: %s for url: %s\nfeed=\n%s\nurlSet:\n%s", str(len(self.feed.entries)),
           #             str(urlObj.url), str(dict(self.feed)), str(urlSet))
           if len(self.feed.entries) == 0:
             logger.debug("Zero entries in feed, self.crawledResource:\n%s", varDump(self.crawledResource))
             # logger.debug("self.processContentTypes: %s", str(self.processContentTypes))
             logger.debug("self.crawledResource.content_type = %s", str(self.crawledResource.content_type))
             if self.crawledResource.content_type == dc.EventObjects.URL.CONTENT_TYPE_TEXT_HTML:
               urlObj.errorMask |= APP_CONSTS.ERROR_MASK_SITE_UNSUPPORTED_CONTENT_TYPE
             
         except TypeError as err:
           logger.debug("WRONG CONTENT FOR URL <" + str(urlObj.url) + "> not rss feed. " + str(err.message))
         except Exception as err:
           logger.debug("SOME ERROR WITH rss feed parse " + str(err.message))
       else:
         logger.debug("feedparser module not found")
     # won't collect urls from rss feed resources
     elif self.processorName != PCONSTS.PROCESSOR_RSS:
       # Added support of urlXpathList as site's properties
       if len(urlXpathList) > 0:
         logger.debug("Site has COLLECT_URLS_XPATH_LIST property: %s", str(urlXpathList))
       else:
         # Set urls xpath list
         urlXpathList = {'sets': {'': self.urlsXpathList}}
         logger.debug("Site has no COLLECT_URLS_XPATH_LIST property, default xpath list used: %s", str(urlXpathList))
       if 'sets' in urlXpathList and isinstance(urlXpathList['sets'], dict):
         matchedSets = 0
         if 'date_format' in urlXpathList:
           dformat = str(urlXpathList['date_format'])
         else:
           dformat = '%Y-%m-%d %H:%M:%S'
         for rexpr in urlXpathList['sets']:
           if rexpr == '' or re.search(rexpr, urlObj.url) is not None:
             if 'mode' in urlXpathList and int(urlXpathList['mode']) == 1:
               xpathl = self.urlsXpathList + urlXpathList['sets'][rexpr]
             else:
               xpathl = urlXpathList['sets'][rexpr]
             matchedSets += 1
             for xpath in xpathl:
               xpath = self.evaluateDateMacro(xpath, dformat)
               elem = dom.xpath(xpath)
               elem_type = type(elem)
               if elem_type == list and len(elem) > 0 and hasattr(elem[0], "tail"):
                 urlSet.update([el.tail for el in elem])
               elif elem_type == list and len(elem) > 0 and isinstance(elem[0], lxml.html.HtmlElement):
                 urlSet.update([el.text for el in elem])
               else:
                 urlSet.update(elem)
         if matchedSets == 0:
           logger.debug("Warning! No one xpath set matched URL %s, URLs not collected!", urlObj.url)
       else:
         logger.debug('Wrong COLLECT_URLS_XPATH_LIST property, `sets` key with dict() of re->xpath_list[] expected!' + \
                      ' Collect URLs aborted!')
     # logger.debug("urlSet: %s", str(urlSet))
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

◆ autoDetectMime

dc_crawler.CollectURLs.CollectURLs.autoDetectMime

Definition at line 65 of file CollectURLs.py.

◆ autoRemoveProps

dc_crawler.CollectURLs.CollectURLs.autoRemoveProps

Definition at line 64 of file CollectURLs.py.

◆ baseUrl

dc_crawler.CollectURLs.CollectURLs.baseUrl

Definition at line 55 of file CollectURLs.py.

◆ batchItem

dc_crawler.CollectURLs.CollectURLs.batchItem

Definition at line 57 of file CollectURLs.py.

◆ BINARY_CONTENT_TYPE_PATTERN

dc_crawler.CollectURLs.CollectURLs.BINARY_CONTENT_TYPE_PATTERN = re.compile('(text)|(xml)', re.I)

static

Definition at line 40 of file CollectURLs.py.

◆ COLLECT_POST_DATA

string dc_crawler.CollectURLs.CollectURLs.COLLECT_POST_DATA = "1"

static

Definition at line 42 of file CollectURLs.py.

◆ COLLECT_POST_DATA_NAME

string dc_crawler.CollectURLs.CollectURLs.COLLECT_POST_DATA_NAME = "COLLECT_POST_DATA"

static

Definition at line 41 of file CollectURLs.py.

◆ crawledResource

dc_crawler.CollectURLs.CollectURLs.crawledResource

Definition at line 51 of file CollectURLs.py.

◆ dbWrapper

dc_crawler.CollectURLs.CollectURLs.dbWrapper

Definition at line 63 of file CollectURLs.py.

◆ DC_URLS_TABLE_PREFIX

string dc_crawler.CollectURLs.CollectURLs.DC_URLS_TABLE_PREFIX = "urls_"

static

Definition at line 43 of file CollectURLs.py.

◆ DETECT_MIME_COLLECTED_URL

string dc_crawler.CollectURLs.CollectURLs.DETECT_MIME_COLLECTED_URL = "2"

static

Definition at line 46 of file CollectURLs.py.

◆ DETECT_MIME_MAIN_CONTENT

string dc_crawler.CollectURLs.CollectURLs.DETECT_MIME_MAIN_CONTENT = "1"

static

Definition at line 45 of file CollectURLs.py.

◆ DETECT_MIME_TIMEOUT

int dc_crawler.CollectURLs.CollectURLs.DETECT_MIME_TIMEOUT = 1

static

Definition at line 47 of file CollectURLs.py.

◆ dom

dc_crawler.CollectURLs.CollectURLs.dom

Definition at line 53 of file CollectURLs.py.

◆ feed

dc_crawler.CollectURLs.CollectURLs.feed

Definition at line 60 of file CollectURLs.py.

◆ feedItems

dc_crawler.CollectURLs.CollectURLs.feedItems

Definition at line 59 of file CollectURLs.py.

◆ isAbortedByTTL

dc_crawler.CollectURLs.CollectURLs.isAbortedByTTL

Definition at line 71 of file CollectURLs.py.

◆ PATTERN_WITH_PROTOCOL

dc_crawler.CollectURLs.CollectURLs.PATTERN_WITH_PROTOCOL = re.compile('[a-zA-Z]+:(//)?')

static

Definition at line 44 of file CollectURLs.py.

◆ postForms

dc_crawler.CollectURLs.CollectURLs.postForms

Definition at line 67 of file CollectURLs.py.

◆ processContentTypes

dc_crawler.CollectURLs.CollectURLs.processContentTypes

Definition at line 66 of file CollectURLs.py.

◆ processorName

dc_crawler.CollectURLs.CollectURLs.processorName

Definition at line 56 of file CollectURLs.py.

◆ realUrl

dc_crawler.CollectURLs.CollectURLs.realUrl

Definition at line 54 of file CollectURLs.py.

◆ robotsParser

dc_crawler.CollectURLs.CollectURLs.robotsParser

Definition at line 69 of file CollectURLs.py.

◆ site

dc_crawler.CollectURLs.CollectURLs.site

Definition at line 62 of file CollectURLs.py.

◆ siteProperties

dc_crawler.CollectURLs.CollectURLs.siteProperties

Definition at line 61 of file CollectURLs.py.

◆ url

dc_crawler.CollectURLs.CollectURLs.url

Definition at line 52 of file CollectURLs.py.

◆ urlProcess

dc_crawler.CollectURLs.CollectURLs.urlProcess

Definition at line 68 of file CollectURLs.py.

◆ urlsXpathList

dc_crawler.CollectURLs.CollectURLs.urlsXpathList

Definition at line 70 of file CollectURLs.py.

◆ urlXpathList

dc_crawler.CollectURLs.CollectURLs.urlXpathList

Definition at line 58 of file CollectURLs.py.

The documentation for this class was generated from the following file:

sources/hce/dc_crawler/CollectURLs.py

Public Member Functions

Static Public Member Functions

Public Attributes

Static Public Attributes

Private Member Functions

Detailed Description

Constructor & Destructor Documentation

◆ __init__()

Member Function Documentation

◆ _normalize_attributes()

◆ checkFieldsIsNone()

◆ evaluateDateMacro()

◆ extractFormURL()

◆ feedElementsProcessing()

◆ feedparserParseDateFixes()

◆ filtersApply()

◆ getFieldParams()

◆ insertNewSiteProperties()

◆ process()

◆ processProcessor()

Member Data Documentation

◆ autoDetectMime

◆ autoRemoveProps

◆ baseUrl

◆ batchItem

◆ BINARY_CONTENT_TYPE_PATTERN

◆ COLLECT_POST_DATA

◆ COLLECT_POST_DATA_NAME

◆ crawledResource

◆ dbWrapper

◆ DC_URLS_TABLE_PREFIX

◆ DETECT_MIME_COLLECTED_URL

◆ DETECT_MIME_MAIN_CONTENT

◆ DETECT_MIME_TIMEOUT

◆ dom

◆ feed

◆ feedItems

◆ isAbortedByTTL

◆ PATTERN_WITH_PROTOCOL

◆ postForms

◆ processContentTypes

◆ processorName

◆ realUrl

◆ robotsParser

◆ site

◆ siteProperties

◆ url

◆ urlProcess

◆ urlsXpathList

◆ urlXpathList

◆ init()