4 @link: http://hierarchical-cluster-engine.com/ 5 @copyright: Copyright © 2013-2014 IOIX Ukraine 6 @license: http://hierarchical-cluster-engine.com/license/ 26 from BaseTask
import BaseTask
34 NO_DIR_STR =
"No such file or directory:" 40 def __init__(self, dbDataType=Constants.DB_DATA_KVDB, keyValueDefaultFile=None, dcContentTemplate=None,
41 keyValueStorageDir=None, rawDataDir=None):
51 logger.debug(
">>> set KVDB")
57 elif self.
dbDataType == Constants.DB_DATA_MYSQL:
58 logger.debug(
">>> set MySQL")
73 def process(self, dbDataRequest, queryCallback):
76 logger.debug(
">>> DBDataTask process")
83 if (
"id" not in dbDataRequest.putDict
or dbDataRequest.putDict[
"id"]
is None)
and \
84 dbDataRequest.urlMd5
is not None:
85 dbDataRequest.putDict[
"id"] = dbDataRequest.urlMd5
86 if "CDate" not in dbDataRequest.putDict
or dbDataRequest.putDict[
"CDate"]
is None:
87 dbDataRequest.putDict[
"CDate"] =
SQLExpression(
'CURRENT_TIMESTAMP')
88 if dbDataRequest.contentType == dc.EventObjects.Content.CONTENT_PROCESSOR_CONTENT:
96 ret.contentType = dbDataRequest.contentType
98 if dbDataRequest.urlMd5
is None:
103 if dbDataRequest.urlMd5
is None:
108 logger.error(
">>> Unsupport request object type = " + str(
type(dbDataRequest)))
113 dbConnect.row_factory = sqlite3.Row
114 cursor = dbConnect.cursor()
115 for row
in cursor.execute(query):
116 response.resultDict = dict(itertools.izip(row.keys(), row))
120 cursor = dbConnect.cursor()
121 cursor.execute(query)
125 dbName = Constants.KEY_VALUE_FILE_NAME_TEMPLATE % request.siteId
127 logger.debug(
">>> KVDB SQL = " + query)
128 logger.debug(
">>> KVDB Path = " + str(dbFileName))
130 if os.path.exists(dbFileName):
131 dbConnect = sqlite3.connect(dbFileName)
132 dbConnect.text_factory = str
133 specificFunc(dbConnect, query, response)
136 logger.error(
">>> path not exist")
138 response.errMessage =
">>> path not exist" 140 logger.error(
">>> some kvdb exception")
142 response.errMessage =
">>> some kvdb exception" 146 logger.debug(
">>> KVDBFetch start")
149 (Constants.SELECT_DB_STORAGE % (
"articles", request.urlMd5)))
154 logger.debug(
">>> MySQLFetch start")
157 tableName = Constants.DC_CONTENTS_TABLE_NAME_TEMPLATE % request.siteId
158 query = Constants.SELECT_DB_STORAGE % (tableName, request.urlMd5)
159 res = self.
queryCallback(query, Constants.FIFTH_DB_ID, Constants.EXEC_NAME)
160 if hasattr(res,
'__iter__')
and len(res) >= 1:
161 ret.resultDict = res[0]
162 for key
in ret.resultDict:
163 if type(ret.resultDict[key]) == datetime.datetime:
164 ret.resultDict[key] = str(ret.resultDict[key])
168 ret.errMessage =
">>> empty mysql response" 173 mainSQL =
"REPLACE INTO `%s`" 174 valusesStr =
"VALUES(" 176 for key
in insertDict:
177 if key
in supportFields
and insertDict[key]
is not None:
182 if type(insertDict[key])
in types.StringTypes
or type(insertDict[key]) == datetime.datetime:
184 valusesStr += str(insertDict[key])
185 if type(insertDict[key])
in types.StringTypes
or type(insertDict[key]) == datetime.datetime:
189 logger.debug(
">>> field not in Constants fields list field or None value = " + key)
190 if valusesStr[-1] ==
",":
191 valusesStr = valusesStr[0:-1]
192 if fieldsStr[-1] ==
",":
193 fieldsStr = fieldsStr[0:-1]
197 query = mainSQL % tableName
208 SQL_INSERT_TEMPLATE =
"INSERT INTO %s SET %s ON DUPLICATE KEY UPDATE %s" 209 for key
in insertDict:
210 if key
in supportFields
and insertDict[key]
is not None:
212 if type(insertDict[key])
in types.StringTypes:
213 escapingStr = MySQLdb.escape_string(str(insertDict[key]))
214 valueList.append((
"'" + escapingStr +
"'"))
216 valueList.append(str(insertDict[key]))
217 setString = Constants.createFieldsValuesString(keyList, valueList)
219 if len(setString) > 0:
220 query = SQL_INSERT_TEMPLATE % (tableName, setString, setString)
222 logger.debug(
">>> createInsertQueryMySQL empty SET list")
227 logger.debug(
">>> KVDBInsert start")
230 Constants.DB_STORAGE_TABLE_NAME)
236 logger.debug(
">>> MySQLInsert start")
239 tableName = Constants.DC_CONTENTS_TABLE_NAME_TEMPLATE % request.siteId
240 SQL_CHECK_TABLE =
"SELECT count(*) FROM `%s`" 241 query = SQL_CHECK_TABLE % tableName
243 if res
is None or len(res) == 0:
253 delContentTypes = [dc.EventObjects.Content.CONTENT_TIDY_CONTENT, dc.EventObjects.Content.CONTENT_HEADERS_CONTENT,
254 dc.EventObjects.Content.CONTENT_REQUESTS_CONTENT, dc.EventObjects.Content.CONTENT_META_CONTENT,
255 dc.EventObjects.Content.CONTENT_COOKIES_CONTENT, dc.EventObjects.Content.CONTENT_DYNAMIC_CONTENT]
257 for cType
in delContentTypes:
259 rawStorageFileName = Utils.accumulateSubstrings([urlMd5,
None, localFileSuffix], [
"",
"_",
""])
260 localPath = rawStoragePath + rawStorageFileName
268 logger.debug(
">>> insertFSFunc start")
270 if "data" in dbDataRequest.putDict
and dbDataRequest.putDict[
"data"]
is not None:
274 if rawStoragePath[-1] !=
'/':
275 rawStoragePath +=
'/' 276 rawStoragePath += str(dbDataRequest.siteId)
277 rawStoragePath +=
'/' 278 pathMaker =
PathMaker(dbDataRequest.urlMd5)
279 rawStoragePath += pathMaker.getDir()
280 rawStoragePath +=
'/' 281 rawStorageFileName = Utils.accumulateSubstrings(
282 [dbDataRequest.urlMd5, dbDataRequest.fileStorageSuffix, localFileSuffix], [
"",
"_",
""])
283 localPath = rawStoragePath + rawStorageFileName
284 if not os.path.exists(rawStoragePath):
285 os.makedirs(rawStoragePath)
286 logger.debug(
">>> MAKE PATH - " + rawStoragePath)
287 if os.path.exists(localPath):
288 logger.debug(
">>> file is exists - " + localPath)
289 if dbDataRequest.contentType == dc.EventObjects.Content.CONTENT_RAW_CONTENT:
293 fd = open(localPath,
"w")
294 except IOError
as ex:
296 logger.error(
">>> Wrong path, file - " + str(localPath))
298 logger.error(
">>> Some file open error, file - " + localPath)
300 fd.write(dbDataRequest.putDict[
"data"])
304 logger.error(
">>> rawDataDir is None")
306 ret.errMessage =
">>> rawDataDir is None" 308 logger.error(
">>> 'data' not in dbDataRequest.putDict or dbDataRequest.putDict['data'] is None")
310 ret.errMessage =
">>> 'data' not in dbDataRequest.putDict or dbDataRequest.putDict['data'] is None" 315 logger.debug(
">>> KVDBDelRecord start")
317 dbName = Constants.KEY_VALUE_FILE_NAME_TEMPLATE % request.siteId
320 if os.path.exists(dbFileName):
321 dbConnect = sqlite3.connect(dbFileName)
322 dbConnect.text_factory = str
323 cursor = dbConnect.cursor()
324 cursor.execute(
"DELETE from `articles` where `id` = '%s'" % request.urlMd5)
327 logger.error(
">>> path not exist")
329 ret.errMessage =
">>> path not exist" 331 logger.error(
">>> some kvdb exception")
333 ret.errMessage =
">>> some kvdb exception" 338 logger.debug(
">>> KVDBDelTable start")
341 if request.filesSuffix
is not None:
343 logger.debug(
">>> Del file = %s", str(delFile))
346 logger.error(
">>> request.filesSuffix is None")
348 ret.errMessage = (
">>> request.filesSuffix is None")
349 except OSError
as err:
350 logger.error((str(err.filename) +
" " + str(err.strerror)))
352 ret.errMessage = (str(err.filename) +
" " + str(err.strerror))
353 logger.debug(
">>> [KVDBDelTable] CURRENT DIR " + str(os.getcwd()))
358 logger.debug(
">>> MySQLDelRecord start")
361 DELETE_SQL_REQUEST =
"DELETE FROM `%s` WHERE `id`='%s'" 362 tableName = Constants.DC_CONTENTS_TABLE_NAME_TEMPLATE % request.siteId
363 query = DELETE_SQL_REQUEST % (tableName, request.urlMd5)
369 logger.debug(
">>> MySQLDelTable start")
372 DROP_SQL_REQUEST =
"DROP TABLE IF EXISTS `%s`" 373 tableName = Constants.DC_CONTENTS_TABLE_NAME_TEMPLATE % request.siteId
374 query = DROP_SQL_REQUEST % tableName
380 logger.debug(
">>> KVDBCreateTable start")
384 if keyValueFileName[-1] !=
'/':
385 keyValueFileName +=
'/' 386 keyValueFileName += (Constants.KEY_VALUE_FILE_NAME_TEMPLATE % request.siteId)
389 logger.error(
">>> keyValueDefaultFile is None")
391 ret.errMessage =
">>> keyValueDefaultFile is None" 396 logger.debug(
">>> MySQLCreateTable start")
401 query = template.replace(
"%SITE_ID%", str(request.siteId))
402 logger.debug(
">>> create content_tables id = " + str(request.siteId))
405 logger.error(
">>> queryCallback is None")
407 ret.errMessage =
">>> queryCallback is None" 409 logger.error(
">>> keyValueDefaultFile is None")
411 ret.errMessage =
">>> keyValueDefaultFile is None" 417 if contentType == dc.EventObjects.Content.CONTENT_RAW_CONTENT:
418 ret = dc.Constants.RAW_DATA_SUFF
419 elif contentType == dc.EventObjects.Content.CONTENT_TIDY_CONTENT:
420 ret = dc.Constants.RAW_DATA_TIDY_SUFF
421 elif contentType == dc.EventObjects.Content.CONTENT_HEADERS_CONTENT:
422 ret = dc.Constants.RAW_DATA_HEADERS_SUFF
423 elif contentType == dc.EventObjects.Content.CONTENT_REQUESTS_CONTENT:
424 ret = dc.Constants.RAW_DATA_REQESTS_SUFF
425 elif contentType == dc.EventObjects.Content.CONTENT_META_CONTENT:
426 ret = dc.Constants.RAW_DATA_META_SUFF
427 elif contentType == dc.EventObjects.Content.CONTENT_COOKIES_CONTENT:
428 ret = dc.Constants.RAW_DATA_COOKIES_SUFF
429 elif contentType == dc.EventObjects.Content.CONTENT_DYNAMIC_CONTENT:
430 ret = dc.Constants.RAW_DATA_DYNAMIC_SUFF
431 elif contentType == dc.EventObjects.Content.CONTENT_CHAIN_PARTS:
432 ret = dc.Constants.RAW_DATA_CHAIN_SUFF
def MySQLCreateTable(self, request)
def KVDBDelRecord(self, request)
def deleteSurroundStorageFiles(self, rawStoragePath, urlMd5)
def MySQLDelRecord(self, request)
def process(self, dbDataRequest, queryCallback)
def KVDBFetch(self, request)
def MySQLInsert(self, request)
def insertKVDBSpecificFunct(self, dbConnect, query, response)
def insertFSFunc(self, dbDataRequest)
def MySQLDelTable(self, request)
def createInsertQueryKVDB(self, insertDict, supportFields, tableName)
def KVDBCreateTable(self, request)
def resolveFileSuffix(self, contentType)
def MySQLFetch(self, request)
def KVDBInsert(self, request)
def fetchKVDBSpecificFunct(self, dbConnect, query, response)
def __init__(self, dbDataType=Constants.DB_DATA_KVDB, keyValueDefaultFile=None, dcContentTemplate=None, keyValueStorageDir=None, rawDataDir=None)
def KVDBCommonExecute(self, request, response, specificFunc, query)
def KVDBDelTable(self, request)
def createInsertQueryMySQL(self, insertDict, supportFields, tableName)