dc_crawler.CollectProperties.CollectProperties Class Reference

Inheritance diagram for dc_crawler.CollectProperties.CollectProperties:

Collaboration diagram for dc_crawler.CollectProperties.CollectProperties:

Public Member Functions
def	__init__ (self)
def	checkFieldsIsNone (self, checkList)
def	process (self, dom, internalLinks, externalLinks)
def	collectProperties (self, dom, internalLinks, externalLinks, siteId, kvDbDir, res, urlId)
def	collectAddtionalProp (self, kvCursor, internalLinksCount, externalLinksCount, batchItem, realUrl)
def	prepareKvDbConnector (self, siteId, kvDbDir)
def	checkKVTable (self, kvConnector, kvCursor)

Public Attributes
	siteId
	kvDbDir
	res
	batchItem
	realUrl
	urlProcess

Static Public Attributes
dictionary	KV_TABLE_TEMPLATES
tuple	KV_DB_TABLE_NAMES = ("titles", "redirects", "internal_links", "external_links")

Detailed Description

Definition at line 22 of file CollectProperties.py.

Constructor & Destructor Documentation

__init__()

def dc_crawler.CollectProperties.CollectProperties.__init__

(

self

)

Definition at line 43 of file CollectProperties.py.

  def __init__(self):
    self.siteId = None
    self.kvDbDir = None
    self.res = None
    self.batchItem = None
    self.realUrl = None
    self.urlProcess = None

Member Function Documentation

checkFieldsIsNone()

def dc_crawler.CollectProperties.CollectProperties.checkFieldsIsNone	(		self,
			checkList
	)

Definition at line 54 of file CollectProperties.py.

  def checkFieldsIsNone(self, checkList):
    # for field in self.__dict__:
    #  if field in checkList and (not hasattr(self, field) or getattr(self, field) is None):
    #    raise Exception(">>> [CollectProperties] Mandatory field must be initialized, field Name = " + field)
    for name in checkList:
      if not hasattr(self, name) or getattr(self, name) is None:
        raise Exception("Some mandatory field `%s` must be initialized!", name)

Here is the caller graph for this function:

checkKVTable()

def dc_crawler.CollectProperties.CollectProperties.checkKVTable	(		self,
			kvConnector,
			kvCursor
	)

Definition at line 186 of file CollectProperties.py.

  def checkKVTable(self, kvConnector, kvCursor):
    for table in self.KV_DB_TABLE_NAMES:
      sql = "SELECT COUNT(*) as cnt FROM sqlite_master WHERE type='table' AND name='%s'" % table
      kvCursor.execute(sql)
      if kvCursor.fetchone()["cnt"] == 0:
        logger.info("kv table %s dose not exist, createing...", table)
        kvCursor.execute(self.KV_TABLE_TEMPLATES[table])
        kvConnector.commit()

collectAddtionalProp()

def dc_crawler.CollectProperties.CollectProperties.collectAddtionalProp	(		self,
			kvCursor,
			internalLinksCount,
			externalLinksCount,
			batchItem,
			realUrl
	)

Definition at line 150 of file CollectProperties.py.

  def collectAddtionalProp(self, kvCursor, internalLinksCount, externalLinksCount, batchItem, realUrl):
    self.checkFieldsIsNone(["res", "urlProcess"])
    # logger.debug("Response: %s", str(self.res))
    size = len(self.res.str_content)
    contentMd5 = hashlib.md5(self.res.str_content).hexdigest()
    kvSql = "SELECT data FROM internal_links WHERE url_id <> '%s'" % (batchItem.urlId,)
    kvCursor.execute(kvSql)
    freq = 0
    for row in kvCursor.fetchall():
      urlInternalLists = row["data"]
      if realUrl in urlInternalLists:
        freq += 1

    self.urlProcess.siteId = batchItem.siteId
    self.urlProcess.updateAdditionProps(internalLinksCount, externalLinksCount, batchItem, size, freq, contentMd5)

Here is the call graph for this function:

Here is the caller graph for this function:

collectProperties()

def dc_crawler.CollectProperties.CollectProperties.collectProperties	(		self,
			dom,
			internalLinks,
			externalLinks,
			siteId,
			kvDbDir,
			res,
			urlId
	)

Definition at line 102 of file CollectProperties.py.

  def collectProperties(self, dom, internalLinks, externalLinks, siteId, kvDbDir, res, urlId):
    kvConnector, kvCursor = self.prepareKvDbConnector(siteId, kvDbDir)
    self.checkKVTable(kvConnector, kvCursor)
    title = None
    domTitle = None
    tmp = dom.find(".//title")
    if tmp is not None:
      domTitle = tmp.text
    if isinstance(domTitle, lxml.etree._Element):  # pylint: disable=E1101,W0212
      title = domTitle.text
    if isinstance(title, str):
      title = title.decode('utf-8')

    histories = []
    for history in res.redirects:
      textHeaders = '\r\n'.join(['%s: %s' % (k, v) for k, v in history.headers.iteritems()])
      historyItem = {"status_code": history.status_code, "headers": textHeaders}
      histories.append(historyItem)
    historiesData = json.dumps(histories)
    internalLinksData = json.dumps(internalLinks)
    externalLinksData = json.dumps(externalLinks)
    # save title
    kvCursor.execute('''INSERT OR REPLACE INTO titles(url_id, data) VALUES(?, ?)''', (urlId, title))

    # save redirects
    kvCursor.execute('''INSERT OR REPLACE INTO redirects(url_id, data) VALUES(?, ?)''', (urlId, historiesData))

    # save internal links
    kvCursor.execute('''INSERT OR REPLACE INTO internal_links(url_id, data) VALUES(?, ?)''', \
      (urlId, internalLinksData))

    # save external links
    kvCursor.execute('''INSERT OR REPLACE INTO external_links(url_id, data) VALUES(?, ?)''', \
      (urlId, externalLinksData))

    kvConnector.commit()
    return kvConnector, kvCursor

Here is the call graph for this function:

Here is the caller graph for this function:

prepareKvDbConnector()

def dc_crawler.CollectProperties.CollectProperties.prepareKvDbConnector	(		self,
			siteId,
			kvDbDir
	)

Definition at line 172 of file CollectProperties.py.

  def prepareKvDbConnector(self, siteId, kvDbDir):
    dbFile = os.path.join(kvDbDir, "%s_fields.db" % (siteId,))
    kvConnector = sqlite3.connect(dbFile)
    kvConnector.row_factory = sqlite3.Row
    kvConnector.text_factory = unicode
    kvCursor = kvConnector.cursor()
    return kvConnector, kvCursor

Here is the caller graph for this function:

process()

def dc_crawler.CollectProperties.CollectProperties.process	(		self,
			dom,
			internalLinks,
			externalLinks
	)

Definition at line 69 of file CollectProperties.py.

  def process(self, dom, internalLinks, externalLinks):
    if dom is None:
      raise Exception(">>> [CollectProperties.process] dom param must be not None")
    if internalLinks is None:
      raise Exception(">>> [CollectProperties.process] internalLinks param must be not None")
    if externalLinks is None:
      raise Exception(">>> [CollectProperties.process] externalLinks param must be not None")
    self.checkFieldsIsNone(["siteId", "kvDbDir", "res", "batchItem", "realUrl"])
    kvCursor = None
    try:
      kvConnector, kvCursor = self.collectProperties(dom, internalLinks, externalLinks, self.siteId, self.kvDbDir,
                                                     self.res, self.batchItem.urlId)
    except Exception, err:
      ExceptionLog.handler(logger, err, "collect base properties to key-value db failed", \
                      (self.siteId, self.kvDbDir, self.res, self.batchItem.urlId))
    if kvCursor is not None and kvConnector is not None:
      try:
        self.collectAddtionalProp(kvCursor, len(internalLinks), len(externalLinks), self.batchItem, self.realUrl)
      except Exception, err:
        ExceptionLog.handler(logger, err, "collect addtional propeties to main db failed", \
                             (self.realUrl))
      kvConnector.close()

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

batchItem

dc_crawler.CollectProperties.CollectProperties.batchItem

Definition at line 47 of file CollectProperties.py.

KV_DB_TABLE_NAMES

tuple dc_crawler.CollectProperties.CollectProperties.KV_DB_TABLE_NAMES = ("titles", "redirects", "internal_links", "external_links")

static

Definition at line 41 of file CollectProperties.py.

KV_TABLE_TEMPLATES

dictionary dc_crawler.CollectProperties.CollectProperties.KV_TABLE_TEMPLATES

static

Initial value:

=  {
      "titles": ,

      "redirects": ,

      "internal_links": ,

      "external_links": 
  }

Definition at line 24 of file CollectProperties.py.

kvDbDir

dc_crawler.CollectProperties.CollectProperties.kvDbDir

Definition at line 45 of file CollectProperties.py.

realUrl

dc_crawler.CollectProperties.CollectProperties.realUrl

Definition at line 48 of file CollectProperties.py.

res

dc_crawler.CollectProperties.CollectProperties.res

Definition at line 46 of file CollectProperties.py.

siteId

dc_crawler.CollectProperties.CollectProperties.siteId

Definition at line 44 of file CollectProperties.py.

urlProcess

dc_crawler.CollectProperties.CollectProperties.urlProcess

Definition at line 49 of file CollectProperties.py.

---

The documentation for this class was generated from the following file:

• sources/hce/dc_crawler/CollectProperties.py