Inheritance diagram for dc_db.URLContentTask.URLContentTask:

Collaboration diagram for dc_db.URLContentTask.URLContentTask:

Public Member Functions
def	__init__ (self, keyValueStorageDir, rawDataDir, dBDataTask, dcSiteTemplate, keyValueDefaultFile, dcStatTemplate, dcLogTemplate)

def	clearContents (self)

def	process (self, urlContentRequests, queryCallback)

def	calcEmptyFields (self, urlContentRequest)

def	genDBFields (self, dbFieldsList, dbFieldsListDefaultValues, row)

def	contentProcessed (self, dataDir, urlContentRequest, contentMask, queryCallback)

def	selectURLFromMySQL (self, siteId, urlMD5, queryCallback)

def	fillLists (self, filePath, elemList, typeId=dc.EventObjects.Content.CONTENT_RAW_CONTENT)

def	contentRaw (self, fList, isBreak, contentTypeId, parseAdditionType)

def	contentRawCommon (self, dataDir, localReverse=False, allFiles=False, rawDataSuff=DC_CONSTANTS.RAW_DATA_SUFF, contentTypeId=dc.EventObjects.Content.CONTENT_RAW_CONTENT, parseAdditionType=True)

def	fillAdditionContentTypes (self, typeMask, typeId, suff, dataDir)

def	getURLContent (self, urlContentRequest, queryCallback)

Public Attributes
	keyValueStorageDir

	rawDataDir

	processedContents

	rawContents

	headers

	requests

	meta

	cookies

	contentMask

	dbDataTask

	urlFetchTask

Detailed Description

Definition at line 28 of file URLContentTask.py.

Constructor & Destructor Documentation

◆ init()

def dc_db.URLContentTask.URLContentTask.__init__	(	self,
		keyValueStorageDir,
		rawDataDir,
		dBDataTask,
		dcSiteTemplate,
		keyValueDefaultFile,
		dcStatTemplate,
		dcLogTemplate
	)

Definition at line 37 of file URLContentTask.py.

                dcLogTemplate):
     self.keyValueStorageDir = keyValueStorageDir
     self.rawDataDir = rawDataDir
     self.processedContents = []
     self.rawContents = []
     self.headers = []
     self.requests = []
     self.meta = []
     self.cookies = []
     self.contentMask = None
     self.dbDataTask = dBDataTask
     self.urlFetchTask = URLFetchTask(keyValueStorageDir, rawDataDir, dBDataTask, dcSiteTemplate,
                                      keyValueDefaultFile, dcStatTemplate, dcLogTemplate,
                                      Constants.DEFAULT_LOCK_TTL)
 
 

Member Function Documentation

◆ calcEmptyFields()

def dc_db.URLContentTask.URLContentTask.calcEmptyFields	(	self,
		urlContentRequest
	)

Definition at line 100 of file URLContentTask.py.

   def calcEmptyFields(self, urlContentRequest):
     if urlContentRequest.siteId == "":
       urlContentRequest.siteId = "0"
     if urlContentRequest.urlMd5 is None or urlContentRequest.urlMd5 == "":
       urlContentRequest.urlMd5 = urlContentRequest.fillMD5(urlContentRequest.url)
 
 

Here is the caller graph for this function:

◆ clearContents()

def dc_db.URLContentTask.URLContentTask.clearContents ( self )

Definition at line 54 of file URLContentTask.py.

   def clearContents(self):
     self.processedContents = []
     self.rawContents = []
     self.headers = []
     self.requests = []
     self.meta = []
     self.cookies = []
 
 

Here is the caller graph for this function:

◆ contentProcessed()

def dc_db.URLContentTask.URLContentTask.contentProcessed	(	self,
		dataDir,
		urlContentRequest,
		contentMask,
		queryCallback
	)

Definition at line 136 of file URLContentTask.py.

   def contentProcessed(self, dataDir, urlContentRequest, contentMask, queryCallback):  # pylint: disable=W0613
     ret = []
     dataFetchRequest = dc.EventObjects.DataFetchRequest(urlContentRequest.siteId, urlContentRequest.urlMd5)
     dataFetchResponse = self.dbDataTask.process(dataFetchRequest, queryCallback)
     if dataFetchResponse is not None and len(dataFetchResponse.resultDict) > 0:
       if ProcessedContentInternalStruct.DATA_FIELD in dataFetchResponse.resultDict and \
       dataFetchResponse.resultDict[ProcessedContentInternalStruct.DATA_FIELD] is not None and \
       ProcessedContentInternalStruct.CDATE_FIELD in dataFetchResponse.resultDict and \
       dataFetchResponse.resultDict[ProcessedContentInternalStruct.CDATE_FIELD] is not None:
         ret = ProcessedContentInternalStruct.parseProcessedBuf(\
               dataFetchResponse.resultDict[ProcessedContentInternalStruct.DATA_FIELD], \
               dataFetchResponse.resultDict[ProcessedContentInternalStruct.CDATE_FIELD], contentMask)
         logger.debug(">>> ret_content == " + str(ret))
     logger.debug(">>> UrlContent result = " + str(dataFetchResponse.__dict__))
     return ret
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ contentRaw()

def dc_db.URLContentTask.URLContentTask.contentRaw	(	self,
		fList,
		isBreak,
		contentTypeId,
		parseAdditionType
	)

Definition at line 193 of file URLContentTask.py.

   def contentRaw(self, fList, isBreak, contentTypeId, parseAdditionType):
     fd = None
     wasOpen = False
     for filePath in fList:
       if os.path.isfile(filePath):
         try:
           fd = open(filePath)
           raw_content = fd.read()
           localDate = datetime.fromtimestamp(os.path.getctime(filePath))
           self.rawContents.append(dc.EventObjects.Content(base64.b64encode(raw_content), localDate.isoformat(' '),
                                                           contentTypeId))
           wasOpen = True
           fd.close()
         except IOError as err:
           logger.debug(">>> IOError with file = %s MSG = %s", str(filePath), str(err.message))
 
         if wasOpen and parseAdditionType:
           filePath = filePath[0: len(DC_CONSTANTS.RAW_DATA_SUFF) * -1]
           filePath += DC_CONSTANTS.RAW_DATA_HEADERS_SUFF
           if self.contentMask & dc.EventObjects.URLContentRequest.CONTENT_TYPE_HEADERS:
             self.fillLists(filePath, self.headers, dc.EventObjects.Content.CONTENT_HEADERS_CONTENT)
           filePath = filePath[0: len(DC_CONSTANTS.RAW_DATA_HEADERS_SUFF) * -1]
           filePath += DC_CONSTANTS.RAW_DATA_REQESTS_SUFF
           if self.contentMask & dc.EventObjects.URLContentRequest.CONTENT_TYPE_REQUESTS:
             self.fillLists(filePath, self.requests, dc.EventObjects.Content.CONTENT_REQUESTS_CONTENT)
           filePath = filePath[0: len(DC_CONSTANTS.RAW_DATA_REQESTS_SUFF) * -1]
           filePath += DC_CONSTANTS.RAW_DATA_META_SUFF
           if self.contentMask & dc.EventObjects.URLContentRequest.CONTENT_TYPE_META:
             self.fillLists(filePath, self.meta, dc.EventObjects.Content.CONTENT_META_CONTENT)
           filePath = filePath[0: len(DC_CONSTANTS.RAW_DATA_META_SUFF) * -1]
           filePath += DC_CONSTANTS.RAW_DATA_COOKIES_SUFF
           if self.contentMask & dc.EventObjects.URLContentRequest.CONTENT_TYPE_COOKIES:
             self.fillLists(filePath, self.cookies, dc.EventObjects.Content.CONTENT_COOKIES_CONTENT)
       if isBreak:
         break
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ contentRawCommon()

def dc_db.URLContentTask.URLContentTask.contentRawCommon	(	self,
		dataDir,
		localReverse = `False`,
		allFiles = `False`,
		rawDataSuff = `DC_CONSTANTS.RAW_DATA_SUFF`,
		contentTypeId = `dc.EventObjects.Content.CONTENT_RAW_CONTENT`,
		parseAdditionType = `True`
	)

Definition at line 236 of file URLContentTask.py.

                        contentTypeId=dc.EventObjects.Content.CONTENT_RAW_CONTENT, parseAdditionType=True):
     fileMask = (dataDir + "/*" + rawDataSuff)
     logger.debug(">>> contentRaw fList = " + str(fileMask))
     fList = sorted(glob.glob(fileMask), key=os.path.getctime, reverse=localReverse)
     self.contentRaw(fList, (not allFiles), contentTypeId, parseAdditionType)
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fillAdditionContentTypes()

def dc_db.URLContentTask.URLContentTask.fillAdditionContentTypes	(	self,
		typeMask,
		typeId,
		suff,
		dataDir
	)

Definition at line 249 of file URLContentTask.py.

   def fillAdditionContentTypes(self, typeMask, typeId, suff, dataDir):
     if self.contentMask & typeMask:
       if self.contentMask & dc.EventObjects.URLContentRequest.CONTENT_TYPE_RAW_LAST:
         self.contentRawCommon(dataDir, True, False, suff, typeId, False)
       if self.contentMask & dc.EventObjects.URLContentRequest.CONTENT_TYPE_RAW_FIRST:
         self.contentRawCommon(dataDir, False, False, suff, typeId, False)
       if self.contentMask & dc.EventObjects.URLContentRequest.CONTENT_TYPE_RAW_ALL:
         self.contentRawCommon(dataDir, False, True, suff, typeId, False)
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fillLists()

def dc_db.URLContentTask.URLContentTask.fillLists	(	self,
		filePath,
		elemList,
		typeId = `dc.EventObjects.Content.CONTENT_RAW_CONTENT`
	)

Definition at line 173 of file URLContentTask.py.

   def fillLists(self, filePath, elemList, typeId=dc.EventObjects.Content.CONTENT_RAW_CONTENT):
     if os.path.isfile(filePath):
       try:
         fd = open(filePath)
         raw_content = fd.read()
         localDate = datetime.fromtimestamp(os.path.getctime(filePath))
         elemList.append(dc.EventObjects.Content(base64.b64encode(raw_content.decode('utf-8')), localDate.isoformat(' '), typeId))
         fd.close()
       except IOError as err:
         elemList.append(None)
         logger.debug(">>> IOError with file = %s MSG = %s", str(filePath), str(err.message))
     else:
       elemList.append(None)
       logger.debug(">>> No file = %s", str(filePath))
 
 

Here is the caller graph for this function:

◆ genDBFields()

def dc_db.URLContentTask.URLContentTask.genDBFields	(	self,
		dbFieldsList,
		dbFieldsListDefaultValues,
		row
	)

Definition at line 113 of file URLContentTask.py.

   def genDBFields(self, dbFieldsList, dbFieldsListDefaultValues, row):
     ret = {}
     for fName in dbFieldsList:
       if fName in dbFieldsListDefaultValues:
         ret[fName] = dbFieldsListDefaultValues[fName]
 
     for fName in dbFieldsList:
       if fName is not None:
         if fName in row:
           if fName in ["UDate", "CDate", "LastModified", "TcDate", "PDate"]:
             ret[str(fName)] = str(row[fName])
           else:
             ret[str(fName)] = row[fName]
         else:
           ret[str(fName)] = None
     return ret
 
 

Here is the caller graph for this function:

◆ getURLContent()

def dc_db.URLContentTask.URLContentTask.getURLContent	(	self,
		urlContentRequest,
		queryCallback
	)

Definition at line 264 of file URLContentTask.py.

   def getURLContent(self, urlContentRequest, queryCallback):
     dataDir = self.rawDataDir + "/" + urlContentRequest.siteId + "/" + PathMaker(urlContentRequest.urlMd5).getDir()
     self.clearContents()
     self.contentMask = urlContentRequest.contentTypeMask
 
     if self.contentMask & (dc.EventObjects.URLContentRequest.CONTENT_TYPE_PROCESSED | \
                            dc.EventObjects.URLContentRequest.CONTENT_TYPE_PROCESSED_INTERNAL | \
                            dc.EventObjects.URLContentRequest.CONTENT_TYPE_PROCESSED_CUSTOM):
       self.processedContents.extend(self.contentProcessed(dataDir, urlContentRequest, self.contentMask, queryCallback))
 
     if self.contentMask & dc.EventObjects.URLContentRequest.CONTENT_TYPE_RAW:
       if self.contentMask & dc.EventObjects.URLContentRequest.CONTENT_TYPE_RAW_LAST:
         self.contentRawCommon(dataDir, True, False)
       if self.contentMask & dc.EventObjects.URLContentRequest.CONTENT_TYPE_RAW_FIRST:
         self.contentRawCommon(dataDir, False, False)
       if self.contentMask & dc.EventObjects.URLContentRequest.CONTENT_TYPE_RAW_ALL:
         self.contentRawCommon(dataDir, False, True)
 
     self.fillAdditionContentTypes(dc.EventObjects.URLContentRequest.CONTENT_TYPE_TIDY,
                                   dc.EventObjects.Content.CONTENT_TIDY_CONTENT, DC_CONSTANTS.RAW_DATA_TIDY_SUFF,
                                   dataDir)
 
     self.fillAdditionContentTypes(dc.EventObjects.URLContentRequest.CONTENT_TYPE_DYNAMIC,
                                   dc.EventObjects.Content.CONTENT_DYNAMIC_CONTENT, DC_CONSTANTS.RAW_DATA_DYNAMIC_SUFF,
                                   dataDir)
 
     self.fillAdditionContentTypes(dc.EventObjects.URLContentRequest.CONTENT_TYPE_CHAIN,
                                   dc.EventObjects.Content.CONTENT_CHAIN_PARTS, DC_CONSTANTS.RAW_DATA_CHAIN_SUFF,
                                   dataDir)
 
     logger.debug("!!!!! self.processedContents: %s", Utils.varDump(self.processedContents, stringifyType=0, ensure_ascii=False, strTypeMaxLen=5000))
 
     ret = dc.EventObjects.URLContentResponse(urlContentRequest.url, self.rawContents, self.processedContents)
     ret.headers = self.headers
     ret.requests = self.requests
     ret.meta = self.meta
     ret.cookies = self.cookies
     row = self.selectURLFromMySQL(urlContentRequest.siteId, urlContentRequest.urlMd5, queryCallback)
 
     if row is not None:
       if "Status" in row:
         ret.status = row["Status"]
       if "URL" in row:
         ret.url = row["URL"]
       if "URLMd5" in row:
         ret.urlMd5 = row["URLMd5"]
       if "RawContentMd5" in row:
         ret.rawContentMd5 = row["RawContentMd5"]
       if "ContentURLMd5" in row:
         ret.contentURLMd5 = row["ContentURLMd5"]
       if "Site_Id" in row:
         ret.siteId = row["Site_Id"]
       if hasattr(urlContentRequest.dbFieldsList, '__iter__') and len(urlContentRequest.dbFieldsList) > 0:
         ret.dbFields = self.genDBFields(urlContentRequest.dbFieldsList, \
                                         urlContentRequest.dbFieldsListDefaultValues, \
                                         row)
 
     if self.contentMask & dc.EventObjects.URLContentRequest.CONTENT_TYPE_ATTRIBUTES:
       if ret.urlMd5 is not None and ret.urlMd5 != "" and ret.siteId is not None:
         ret.attributes = AttrFetchTask.fetchUrlsAttributesByNames(ret.siteId,
                                                                   ret.urlMd5,
                                                                   queryCallback,
                                                                   urlContentRequest.attributeNames)
 
     return ret

Here is the call graph for this function:






















Here is the caller graph for this function:

◆ process()

def dc_db.URLContentTask.URLContentTask.process	(	self,
		urlContentRequests,
		queryCallback
	)

Definition at line 68 of file URLContentTask.py.

   def process(self, urlContentRequests, queryCallback):  # pylint: disable=W0613
     urlContentResponses = []
     # @todo add more complex case, implemented only from rawDataDir
     for urlContentRequest in urlContentRequests:
       if urlContentRequest is None:
         urlContentResponses.append(None)
       elif hasattr(urlContentRequest, "urlFetch") and urlContentRequest.urlFetch is not None:
         urlFetches = []
         urlFetches.append(urlContentRequest.urlFetch)
         urls = self.urlFetchTask.process(urlFetches, queryCallback)
         for url in urls:
           urlContentRequest.urlMd5 = ""
           urlContentRequest.siteId = url.siteId
           urlContentRequest.url = url.url
           self.calcEmptyFields(urlContentRequest)
           StatisticLogManager.logUpdate(queryCallback, "LOG_URL_CONTENT", urlContentRequest, urlContentRequest.siteId,
                                         urlContentRequest.urlMd5)
           urlContentResponses.append(self.getURLContent(urlContentRequest, queryCallback))
       else:
         self.calcEmptyFields(urlContentRequest)
         StatisticLogManager.logUpdate(queryCallback, "LOG_URL_CONTENT", urlContentRequest, urlContentRequest.siteId,
                                       urlContentRequest.urlMd5)
         urlContentResponses.append(self.getURLContent(urlContentRequest, queryCallback))
       logger.debug(">>> urlContentResponses len = %s", str(len(urlContentResponses)))
       # logger.debug("!!! urlContentResponses: %s", Utils.varDump(urlContentResponses, stringifyType=0, strTypeMaxLen=5000))
 
     return urlContentResponses
 
 

Here is the call graph for this function:

Here is the caller graph for this function:

◆ selectURLFromMySQL()

def dc_db.URLContentTask.URLContentTask.selectURLFromMySQL	(	self,
		siteId,
		urlMD5,
		queryCallback
	)

Definition at line 158 of file URLContentTask.py.

   def selectURLFromMySQL(self, siteId, urlMD5, queryCallback):
     row = None
     tableName = Constants.DC_URLS_TABLE_NAME_TEMPLATE % siteId
     SELECT_URL_QUERY = "SELECT * FROM %s WHERE `URLMd5` = '%s'"
     query = SELECT_URL_QUERY % (tableName, urlMD5)
     res = queryCallback(query, Constants.SECONDARY_DB_ID, Constants.EXEC_NAME)
     if hasattr(res, '__iter__') and len(res) >= 1:
       row = res[0]
     return row
 
 

Here is the caller graph for this function:

Member Data Documentation

◆ contentMask

dc_db.URLContentTask.URLContentTask.contentMask

Definition at line 46 of file URLContentTask.py.

◆ cookies

dc_db.URLContentTask.URLContentTask.cookies

Definition at line 45 of file URLContentTask.py.

◆ dbDataTask

dc_db.URLContentTask.URLContentTask.dbDataTask

Definition at line 47 of file URLContentTask.py.

◆ headers

dc_db.URLContentTask.URLContentTask.headers

Definition at line 42 of file URLContentTask.py.

◆ keyValueStorageDir

dc_db.URLContentTask.URLContentTask.keyValueStorageDir

Definition at line 38 of file URLContentTask.py.

◆ meta

dc_db.URLContentTask.URLContentTask.meta

Definition at line 44 of file URLContentTask.py.

◆ processedContents

dc_db.URLContentTask.URLContentTask.processedContents

Definition at line 40 of file URLContentTask.py.

◆ rawContents

dc_db.URLContentTask.URLContentTask.rawContents

Definition at line 41 of file URLContentTask.py.

◆ rawDataDir

dc_db.URLContentTask.URLContentTask.rawDataDir

Definition at line 39 of file URLContentTask.py.

◆ requests

dc_db.URLContentTask.URLContentTask.requests

Definition at line 43 of file URLContentTask.py.

◆ urlFetchTask

dc_db.URLContentTask.URLContentTask.urlFetchTask

Definition at line 48 of file URLContentTask.py.

The documentation for this class was generated from the following file:

sources/hce/dc_db/URLContentTask.py

Public Member Functions

Public Attributes

Detailed Description

Constructor & Destructor Documentation

◆ __init__()

Member Function Documentation

◆ calcEmptyFields()

◆ clearContents()

◆ contentProcessed()

◆ contentRaw()

◆ contentRawCommon()

◆ fillAdditionContentTypes()

◆ fillLists()

◆ genDBFields()

◆ getURLContent()

◆ process()

◆ selectURLFromMySQL()

Member Data Documentation

◆ contentMask

◆ cookies

◆ dbDataTask

◆ headers

◆ keyValueStorageDir

◆ meta

◆ processedContents

◆ rawContents

◆ rawDataDir

◆ requests

◆ urlFetchTask

◆ init()