HCE Project Python language Distributed Tasks Manager Application, Distributed Crawler Application and client API bindings.
2.0.0-chaika
Hierarchical Cluster Engine Python language binding
|
Public Member Functions | |
def | __init__ (self, dcSiteTemplate, keyValueDefaultFile, keyValueStorageDir, dBDataTask, dcStatTemplates, dcLogTemplate, dcAttrTemplate, tasksManager=None) |
def | process (self, site, queryCallback) |
def | addSite (self, site, queryCallback) |
def | addSitesFilter (self, site, queryCallback) |
def | updateSitesFilter (self, site, queryCallback) |
def | createPropDict (self, field, site) |
def | isIsoFormatDate (self, dateValue) |
def | addSiteProperties (self, site, queryCallback) |
def | updateSiteProperties (self, site, queryCallback) |
def | addSiteURLSites (self, site, queryCallback) |
def | updateSiteURLSites (self, site, queryCallback) |
def | addSiteURLURLs (self, site, queryCallback) |
def | createTableFromTemplate (self, site, template, dbId, queryCallback, replaceDic=None) |
def | addSiteInKVDB (self, site, queryCallback) |
def | siteDelete (self, site, queryCallback) |
Public Member Functions inherited from dc_db.BaseTask.BaseTask | |
def | isSiteExist (self, siteId, queryCallback, userId=None) |
def | generateCriterionSQL (self, criterions, additionWhere=None, siteId=None) |
def | fetchByCriterions (self, criterions, queryCallback) |
def | dbLock (self, mutexName, queryCallback, sleepTime=1, mutexLockTTL=Constants.DEFAULT_LOCK_TTL) |
def | dbUnlock (self, mutexName, queryCallback) |
def | createUrlsInsertQuery (self, siteId, localKeys, localValues) |
def | copyUrlsToDcUrls (self, siteId, queryCallback) |
def | statisticLogUpdate (self, localObj, urlMd5, siteId, status, queryCallback, isInsert=False) |
def | calculateMd5FormUrl (self, url, urlType, useNormilize=False) |
Static Public Member Functions | |
def | execSiteCriterions (criterions, queryCallback) |
Static Public Member Functions inherited from dc_db.BaseTask.BaseTask | |
def | readValueFromSiteProp (siteId, propName, queryCallback, urlMd5=None) |
Public Attributes | |
dcSiteTemplate | |
keyValueDefaultFile | |
keyValueStorageDir | |
dBDataTask | |
dcStatTemplates | |
dcLogTemplate | |
dcAttrTemplate | |
siteDeleteTask | |
tasksManager | |
Static Public Attributes | |
string | FIELD_NAME_URLS = "urls" |
string | FIELD_NAME_PROPERTIES = "properties" |
string | FIELD_NAME_FILTERS = "filters" |
Definition at line 29 of file SiteTask.py.
def dc_db.SiteTask.SiteTask.__init__ | ( | self, | |
dcSiteTemplate, | |||
keyValueDefaultFile, | |||
keyValueStorageDir, | |||
dBDataTask, | |||
dcStatTemplates, | |||
dcLogTemplate, | |||
dcAttrTemplate, | |||
tasksManager = None |
|||
) |
Definition at line 40 of file SiteTask.py.
def dc_db.SiteTask.SiteTask.addSite | ( | self, | |
site, | |||
queryCallback | |||
) |
def dc_db.SiteTask.SiteTask.addSiteInKVDB | ( | self, | |
site, | |||
queryCallback | |||
) |
Definition at line 383 of file SiteTask.py.
def dc_db.SiteTask.SiteTask.addSiteProperties | ( | self, | |
site, | |||
queryCallback | |||
) |
Definition at line 222 of file SiteTask.py.
def dc_db.SiteTask.SiteTask.addSitesFilter | ( | self, | |
site, | |||
queryCallback | |||
) |
def dc_db.SiteTask.SiteTask.addSiteURLSites | ( | self, | |
site, | |||
queryCallback | |||
) |
def dc_db.SiteTask.SiteTask.addSiteURLURLs | ( | self, | |
site, | |||
queryCallback | |||
) |
Definition at line 334 of file SiteTask.py.
def dc_db.SiteTask.SiteTask.createPropDict | ( | self, | |
field, | |||
site | |||
) |
Definition at line 162 of file SiteTask.py.
def dc_db.SiteTask.SiteTask.createTableFromTemplate | ( | self, | |
site, | |||
template, | |||
dbId, | |||
queryCallback, | |||
replaceDic = None |
|||
) |
|
static |
Definition at line 407 of file SiteTask.py.
def dc_db.SiteTask.SiteTask.isIsoFormatDate | ( | self, | |
dateValue | |||
) |
def dc_db.SiteTask.SiteTask.process | ( | self, | |
site, | |||
queryCallback | |||
) |
Definition at line 61 of file SiteTask.py.
def dc_db.SiteTask.SiteTask.siteDelete | ( | self, | |
site, | |||
queryCallback | |||
) |
Definition at line 395 of file SiteTask.py.
def dc_db.SiteTask.SiteTask.updateSiteProperties | ( | self, | |
site, | |||
queryCallback | |||
) |
Definition at line 241 of file SiteTask.py.
def dc_db.SiteTask.SiteTask.updateSitesFilter | ( | self, | |
site, | |||
queryCallback | |||
) |
def dc_db.SiteTask.SiteTask.updateSiteURLSites | ( | self, | |
site, | |||
queryCallback | |||
) |
Definition at line 277 of file SiteTask.py.
dc_db.SiteTask.SiteTask.dBDataTask |
Definition at line 45 of file SiteTask.py.
dc_db.SiteTask.SiteTask.dcAttrTemplate |
Definition at line 48 of file SiteTask.py.
dc_db.SiteTask.SiteTask.dcLogTemplate |
Definition at line 47 of file SiteTask.py.
dc_db.SiteTask.SiteTask.dcSiteTemplate |
Definition at line 42 of file SiteTask.py.
dc_db.SiteTask.SiteTask.dcStatTemplates |
Definition at line 46 of file SiteTask.py.
|
static |
Definition at line 33 of file SiteTask.py.
|
static |
Definition at line 32 of file SiteTask.py.
|
static |
Definition at line 31 of file SiteTask.py.
dc_db.SiteTask.SiteTask.keyValueDefaultFile |
Definition at line 43 of file SiteTask.py.
dc_db.SiteTask.SiteTask.keyValueStorageDir |
Definition at line 44 of file SiteTask.py.
dc_db.SiteTask.SiteTask.siteDeleteTask |
Definition at line 50 of file SiteTask.py.
dc_db.SiteTask.SiteTask.tasksManager |
Definition at line 53 of file SiteTask.py.