HCE Project Python language Distributed Tasks Manager Application, Distributed Crawler Application and client API bindings.
2.0.0-chaika
Hierarchical Cluster Engine Python language binding
|
Public Member Functions | |
def | __init__ (self, dbDataType=Constants.DB_DATA_KVDB, keyValueDefaultFile=None, dcContentTemplate=None, keyValueStorageDir=None, rawDataDir=None) |
def | process (self, dbDataRequest, queryCallback) |
def | fetchKVDBSpecificFunct (self, dbConnect, query, response) |
def | insertKVDBSpecificFunct (self, dbConnect, query, response) |
def | KVDBCommonExecute (self, request, response, specificFunc, query) |
def | KVDBFetch (self, request) |
def | MySQLFetch (self, request) |
def | createInsertQueryKVDB (self, insertDict, supportFields, tableName) |
def | createInsertQueryMySQL (self, insertDict, supportFields, tableName) |
def | KVDBInsert (self, request) |
def | MySQLInsert (self, request) |
def | deleteSurroundStorageFiles (self, rawStoragePath, urlMd5) |
def | insertFSFunc (self, dbDataRequest) |
def | KVDBDelRecord (self, request) |
def | KVDBDelTable (self, request) |
def | MySQLDelRecord (self, request) |
def | MySQLDelTable (self, request) |
def | KVDBCreateTable (self, request) |
def | MySQLCreateTable (self, request) |
def | resolveFileSuffix (self, contentType) |
Public Member Functions inherited from dc_db.BaseTask.BaseTask | |
def | isSiteExist (self, siteId, queryCallback, userId=None) |
def | generateCriterionSQL (self, criterions, additionWhere=None, siteId=None) |
def | fetchByCriterions (self, criterions, queryCallback) |
def | dbLock (self, mutexName, queryCallback, sleepTime=1, mutexLockTTL=Constants.DEFAULT_LOCK_TTL) |
def | dbUnlock (self, mutexName, queryCallback) |
def | createUrlsInsertQuery (self, siteId, localKeys, localValues) |
def | copyUrlsToDcUrls (self, siteId, queryCallback) |
def | statisticLogUpdate (self, localObj, urlMd5, siteId, status, queryCallback, isInsert=False) |
def | calculateMd5FormUrl (self, url, urlType, useNormilize=False) |
Public Attributes | |
rawDataDir | |
keyValueStorageDir | |
dbDataType | |
queryCallback | |
fetchFunc | |
insertFunc | |
createTableFunc | |
delRecordFunc | |
delTableFunc | |
keyValueDefaultFile | |
dcContentTemplate | |
Static Public Attributes | |
string | NO_DIR_STR = "No such file or directory:" |
Additional Inherited Members | |
Static Public Member Functions inherited from dc_db.BaseTask.BaseTask | |
def | readValueFromSiteProp (siteId, propName, queryCallback, urlMd5=None) |
Definition at line 32 of file DBDataTask.py.
def dc_db.DBDataTask.DBDataTask.__init__ | ( | self, | |
dbDataType = Constants.DB_DATA_KVDB , |
|||
keyValueDefaultFile = None , |
|||
dcContentTemplate = None , |
|||
keyValueStorageDir = None , |
|||
rawDataDir = None |
|||
) |
Definition at line 41 of file DBDataTask.py.
def dc_db.DBDataTask.DBDataTask.createInsertQueryKVDB | ( | self, | |
insertDict, | |||
supportFields, | |||
tableName | |||
) |
def dc_db.DBDataTask.DBDataTask.createInsertQueryMySQL | ( | self, | |
insertDict, | |||
supportFields, | |||
tableName | |||
) |
def dc_db.DBDataTask.DBDataTask.deleteSurroundStorageFiles | ( | self, | |
rawStoragePath, | |||
urlMd5 | |||
) |
Definition at line 252 of file DBDataTask.py.
def dc_db.DBDataTask.DBDataTask.fetchKVDBSpecificFunct | ( | self, | |
dbConnect, | |||
query, | |||
response | |||
) |
def dc_db.DBDataTask.DBDataTask.insertFSFunc | ( | self, | |
dbDataRequest | |||
) |
Definition at line 267 of file DBDataTask.py.
def dc_db.DBDataTask.DBDataTask.insertKVDBSpecificFunct | ( | self, | |
dbConnect, | |||
query, | |||
response | |||
) |
def dc_db.DBDataTask.DBDataTask.KVDBCommonExecute | ( | self, | |
request, | |||
response, | |||
specificFunc, | |||
query | |||
) |
def dc_db.DBDataTask.DBDataTask.KVDBCreateTable | ( | self, | |
request | |||
) |
Definition at line 379 of file DBDataTask.py.
def dc_db.DBDataTask.DBDataTask.KVDBDelRecord | ( | self, | |
request | |||
) |
Definition at line 314 of file DBDataTask.py.
def dc_db.DBDataTask.DBDataTask.KVDBDelTable | ( | self, | |
request | |||
) |
Definition at line 337 of file DBDataTask.py.
def dc_db.DBDataTask.DBDataTask.KVDBFetch | ( | self, | |
request | |||
) |
def dc_db.DBDataTask.DBDataTask.KVDBInsert | ( | self, | |
request | |||
) |
def dc_db.DBDataTask.DBDataTask.MySQLCreateTable | ( | self, | |
request | |||
) |
def dc_db.DBDataTask.DBDataTask.MySQLDelRecord | ( | self, | |
request | |||
) |
Definition at line 357 of file DBDataTask.py.
def dc_db.DBDataTask.DBDataTask.MySQLDelTable | ( | self, | |
request | |||
) |
Definition at line 368 of file DBDataTask.py.
def dc_db.DBDataTask.DBDataTask.MySQLFetch | ( | self, | |
request | |||
) |
Definition at line 153 of file DBDataTask.py.
def dc_db.DBDataTask.DBDataTask.MySQLInsert | ( | self, | |
request | |||
) |
Definition at line 235 of file DBDataTask.py.
def dc_db.DBDataTask.DBDataTask.process | ( | self, | |
dbDataRequest, | |||
queryCallback | |||
) |
Definition at line 73 of file DBDataTask.py.
def dc_db.DBDataTask.DBDataTask.resolveFileSuffix | ( | self, | |
contentType | |||
) |
Definition at line 415 of file DBDataTask.py.
dc_db.DBDataTask.DBDataTask.createTableFunc |
Definition at line 49 of file DBDataTask.py.
dc_db.DBDataTask.DBDataTask.dbDataType |
Definition at line 45 of file DBDataTask.py.
dc_db.DBDataTask.DBDataTask.dcContentTemplate |
Definition at line 65 of file DBDataTask.py.
dc_db.DBDataTask.DBDataTask.delRecordFunc |
Definition at line 54 of file DBDataTask.py.
dc_db.DBDataTask.DBDataTask.delTableFunc |
Definition at line 55 of file DBDataTask.py.
dc_db.DBDataTask.DBDataTask.fetchFunc |
Definition at line 47 of file DBDataTask.py.
dc_db.DBDataTask.DBDataTask.insertFunc |
Definition at line 48 of file DBDataTask.py.
dc_db.DBDataTask.DBDataTask.keyValueDefaultFile |
Definition at line 64 of file DBDataTask.py.
dc_db.DBDataTask.DBDataTask.keyValueStorageDir |
Definition at line 44 of file DBDataTask.py.
|
static |
Definition at line 34 of file DBDataTask.py.
dc_db.DBDataTask.DBDataTask.queryCallback |
Definition at line 46 of file DBDataTask.py.
dc_db.DBDataTask.DBDataTask.rawDataDir |
Definition at line 43 of file DBDataTask.py.