dc_db.SiteFindTask.SiteFindTask Class Reference

Inheritance diagram for dc_db.SiteFindTask.SiteFindTask:

Collaboration diagram for dc_db.SiteFindTask.SiteFindTask:

Public Member Functions
def	__init__ (self, dcSiteTemplate, keyValueDefaultFile, keyValueStorageDir, dBDataTask, dcStatTemplates, dcLogTemplate, dcAttrTemplate)
def	process (self, siteFind, queryCallback)
def	loadSiteFromDB (self, siteFind, site_id, site, queryCallback)
def	loadListOfSitesFromDB (self, siteFind, queryCallback)
def	siteFind (self, siteFind, queryCallback)
Public Member Functions inherited from dc_db.SiteTask.SiteTask
def	__init__ (self, dcSiteTemplate, keyValueDefaultFile, keyValueStorageDir, dBDataTask, dcStatTemplates, dcLogTemplate, dcAttrTemplate, tasksManager=None)
def	process (self, site, queryCallback)
def	addSite (self, site, queryCallback)
def	addSitesFilter (self, site, queryCallback)
def	updateSitesFilter (self, site, queryCallback)
def	createPropDict (self, field, site)
def	isIsoFormatDate (self, dateValue)
def	addSiteProperties (self, site, queryCallback)
def	updateSiteProperties (self, site, queryCallback)
def	addSiteURLSites (self, site, queryCallback)
def	updateSiteURLSites (self, site, queryCallback)
def	addSiteURLURLs (self, site, queryCallback)
def	createTableFromTemplate (self, site, template, dbId, queryCallback, replaceDic=None)
def	addSiteInKVDB (self, site, queryCallback)
def	siteDelete (self, site, queryCallback)
Public Member Functions inherited from dc_db.BaseTask.BaseTask
def	isSiteExist (self, siteId, queryCallback, userId=None)
def	generateCriterionSQL (self, criterions, additionWhere=None, siteId=None)
def	fetchByCriterions (self, criterions, queryCallback)
def	dbLock (self, mutexName, queryCallback, sleepTime=1, mutexLockTTL=Constants.DEFAULT_LOCK_TTL)
def	dbUnlock (self, mutexName, queryCallback)
def	createUrlsInsertQuery (self, siteId, localKeys, localValues)
def	copyUrlsToDcUrls (self, siteId, queryCallback)
def	statisticLogUpdate (self, localObj, urlMd5, siteId, status, queryCallback, isInsert=False)
def	calculateMd5FormUrl (self, url, urlType, useNormilize=False)

Public Attributes
	siteStatusTask
Public Attributes inherited from dc_db.SiteTask.SiteTask
	dcSiteTemplate
	keyValueDefaultFile
	keyValueStorageDir
	dBDataTask
	dcStatTemplates
	dcLogTemplate
	dcAttrTemplate
	siteDeleteTask
	tasksManager

Additional Inherited Members
Static Public Member Functions inherited from dc_db.SiteTask.SiteTask
def	execSiteCriterions (criterions, queryCallback)
Static Public Member Functions inherited from dc_db.BaseTask.BaseTask
def	readValueFromSiteProp (siteId, propName, queryCallback, urlMd5=None)
Static Public Attributes inherited from dc_db.SiteTask.SiteTask
string	FIELD_NAME_URLS = "urls"
string	FIELD_NAME_PROPERTIES = "properties"
string	FIELD_NAME_FILTERS = "filters"

Detailed Description

Definition at line 32 of file SiteFindTask.py.

Constructor & Destructor Documentation

__init__()

def dc_db.SiteFindTask.SiteFindTask.__init__	(		self,
			dcSiteTemplate,
			keyValueDefaultFile,
			keyValueStorageDir,
			dBDataTask,
			dcStatTemplates,
			dcLogTemplate,
			dcAttrTemplate
	)

Definition at line 39 of file SiteFindTask.py.

               dcLogTemplate, dcAttrTemplate):
    super(SiteFindTask, self).__init__(dcSiteTemplate, keyValueDefaultFile, keyValueStorageDir, dBDataTask,
                                       dcStatTemplates, dcLogTemplate, dcAttrTemplate)
    self.siteStatusTask = None

Member Function Documentation

loadListOfSitesFromDB()

def dc_db.SiteFindTask.SiteFindTask.loadListOfSitesFromDB	(		self,
			siteFind,
			queryCallback
	)

Definition at line 96 of file SiteFindTask.py.

  def loadListOfSitesFromDB(self, siteFind, queryCallback):
    if siteFind.url is not None and (not siteFind.criterions):
      query = "SELECT `Site_Id` FROM sites_urls WHERE `URL` LIKE '" + MySQLdb.escape_string(siteFind.url) + "%' GROUP BY `Site_Id`"  # pylint: disable=E1101,C0301
    elif (siteFind.url is not None) and (siteFind.criterions is not None):
      additionCriterion = " `URL` LIKE '" + MySQLdb.escape_string(siteFind.url) + "%' "  # pylint: disable=E1101,C0301
      query = "SELECT `Site_Id` FROM sites_urls " + self.generateCriterionSQL(siteFind.criterions, additionCriterion)
    else:
      # Fix for tables list to use both "sites" and "sites_urls" tables
      tableName = "sites_urls"
      if ("WHERE" in siteFind.criterions) and (siteFind.criterions["WHERE"] is not None) and \
      (tableName in siteFind.criterions["WHERE"]):
        addTable = ", " + tableName
      else:
        addTable = ""

      sitesTableName = 'sites'
      if dc.EventObjects.SiteFind.CRITERION_TABLES in siteFind.criterions and \
        siteFind.criterions[dc.EventObjects.SiteFind.CRITERION_TABLES] is not None and \
        siteFind.criterions[dc.EventObjects.SiteFind.CRITERION_TABLES] != "":
        if sitesTableName not in siteFind.criterions[dc.EventObjects.SiteFind.CRITERION_TABLES]:
          addTable = ", " + siteFind.criterions[dc.EventObjects.SiteFind.CRITERION_TABLES]

      query = "SELECT `Id` AS Site_Id FROM " + sitesTableName + addTable + self.generateCriterionSQL(siteFind.criterions)

    # logger.debug("query: %s", str(query))
    site_ids = queryCallback(query, Constants.PRIMARY_DB_ID, Constants.EXEC_NAME)
    logger.debug("List of Site_Id: %s", str(site_ids))

    return site_ids

Here is the call graph for this function:

Here is the caller graph for this function:

loadSiteFromDB()

def dc_db.SiteFindTask.SiteFindTask.loadSiteFromDB	(		self,
			siteFind,
			site_id,
			site,
			queryCallback
	)

Definition at line 58 of file SiteFindTask.py.

  def loadSiteFromDB(self, siteFind, site_id, site, queryCallback):
    tableName = 'sites'
    tables = tableName
    if dc.EventObjects.SiteFind.CRITERION_TABLES in siteFind.criterions and \
      siteFind.criterions[dc.EventObjects.SiteFind.CRITERION_TABLES] is not None and \
      siteFind.criterions[dc.EventObjects.SiteFind.CRITERION_TABLES] != "":
      if tableName not in siteFind.criterions[dc.EventObjects.SiteFind.CRITERION_TABLES]:
        tables = tableName + ", " + siteFind.criterions[dc.EventObjects.SiteFind.CRITERION_TABLES]
      else:
        tables = siteFind.criterions[dc.EventObjects.SiteFind.CRITERION_TABLES]

    query = GET_SITE_SQL_TEMPLATE % (tables, site_id["Site_Id"])
    # logger.debug("query: %s", str(query))
    site_row = queryCallback(query, Constants.PRIMARY_DB_ID, Constants.EXEC_NAME)
    logger.debug("Get site from sites: %s", str(site_row))
    for (key, value) in Constants.siteDict.items():
      if str(value)[:1] == "`":
        value = str(value)[1:-1]
      # logger.debug("key: %s; value: %s", str(key), str(value))
      logger.debug("site field: %s; table field: %s", str(site.__dict__[key]), str(site_row[0].get(value, None)))
      if key == "uDate":
        site.__dict__[key] = str(site_row[0].get(value, None))
      elif key == "tcDate":
        site.__dict__[key] = str(site_row[0].get(value, None))
      elif key == "tcDateProcess":
        site.__dict__[key] = str(site_row[0].get(value, None))
      elif key == "cDate":
        site.__dict__[key] = str(site_row[0].get(value, None))
      elif key == "recrawlDate":
        site.__dict__[key] = str(site_row[0].get(value, None))
      else:
        site.__dict__[key] = site_row[0].get(value, "a")
      # site.cDate = str(site_row[0]["CDate"])
    return site

Here is the caller graph for this function:

process()

def dc_db.SiteFindTask.SiteFindTask.process	(		self,
			siteFind,
			queryCallback
	)

Definition at line 50 of file SiteFindTask.py.

  def process(self, siteFind, queryCallback):
    self.siteStatusTask = SiteStatusTask()
    sites = self.siteFind(siteFind, queryCallback)
    return sites

Here is the call graph for this function:

Here is the caller graph for this function:

siteFind()

def dc_db.SiteFindTask.SiteFindTask.siteFind	(		self,
			siteFind,
			queryCallback
	)

Definition at line 132 of file SiteFindTask.py.

  def siteFind(self, siteFind, queryCallback):
    sites = []
    # get all UNIQ site id's with urls for given url
    site_ids = self.loadListOfSitesFromDB(siteFind, queryCallback)
    if hasattr(site_ids, "__iter__"):
      # for each site fill it fields
      for site_id in site_ids:
        site = dc.EventObjects.Site("")
        # load site from sites table
        self.loadSiteFromDB(siteFind, site_id, site, queryCallback)
        if SiteTask.FIELD_NAME_URLS not in siteFind.excludeList:
          site.urls = self.siteStatusTask.fillUrls(site, queryCallback)
        else:
          site.urls = None
        if SiteTask.FIELD_NAME_PROPERTIES not in siteFind.excludeList:
          site.properties = self.siteStatusTask.fillProperties(site, queryCallback)
        else:
          site.properties = None
        if SiteTask.FIELD_NAME_FILTERS not in siteFind.excludeList:
          site.filters = self.siteStatusTask.fillFilters(site, queryCallback)
        else:
          site.filters = None
        sites.append(site)

    return sites

Member Data Documentation

siteStatusTask

dc_db.SiteFindTask.SiteFindTask.siteStatusTask

Definition at line 42 of file SiteFindTask.py.

---

The documentation for this class was generated from the following file:

• sources/hce/dc_db/SiteFindTask.py