2 Created on Apr 10, 2014 6 @link: http://hierarchical-cluster-engine.com/ 7 @copyright: Copyright © 2013-2014 IOIX Ukraine 8 @license: http://hierarchical-cluster-engine.com/license/ 17 from cement.core
import foundation
35 class DCC(foundation.CementApp):
38 MSG_ERROR_EMPTY_CONFIG_FILE_NAME =
"Config file name is empty." 39 MSG_ERROR_WRONG_CONFIG_FILE_NAME =
"Config file name is wrong" 40 MSG_ERROR_LOAD_APP_CONFIG =
"Error loading application config file." 41 MSG_ERROR_READ_LOG_CONFIG =
"Error read log config file." 43 ERROR_FILE_ERROR_MESSAGE_FIELD_NAME =
'errorMessage' 44 ERROR_FILE_ERROR_CODE_FIELD_NAME =
'errorCode' 47 label = CONSTANTS.APP_NAME
52 foundation.CementApp.__init__(self)
71 timeout = CONSTANTS.TCP_TIMEOUT
78 config = ConfigParser.ConfigParser()
79 config.optionxform = str
81 readOk = config.read(configName)
86 if config.has_section(APP_CONSTS.CONFIG_APPLICATION_SECTION_NAME):
87 confLogFileName = str(config.get(CONSTANTS.LOG_CONFIG_SECTION_NAME, CONSTANTS.LOG_CONFIG_OPTION_NAME))
89 if config.has_section(CONSTANTS.APP_NAME):
90 timeout = config.get(CONSTANTS.APP_NAME, CONSTANTS.TCP_TIMEOUT_CONFIG_NAME)
91 host = str(config.get(CONSTANTS.APP_NAME, CONSTANTS.DTM_HOST))
92 port = str(config.get(CONSTANTS.APP_NAME, CONSTANTS.DTM_PORT))
94 if hasattr(self.pargs,
'dcc_timeout')
and self.pargs.dcc_timeout !=
None and \
95 self.pargs.dcc_timeout.isdigit():
96 timeout = int(self.pargs.dcc_timeout)
98 if hasattr(self.pargs,
'dcc_clientHost')
and self.pargs.dcc_clientHost !=
None:
99 host = self.pargs.dcc_clientHost
101 if hasattr(self.pargs,
'dcc_clientPort')
and self.pargs.dcc_clientPort !=
None:
102 port = self.pargs.dcc_clientPort
104 if hasattr(self.pargs,
'output_file')
and self.pargs.output_file !=
None:
105 outputFileName = self.pargs.output_file
107 if hasattr(self.pargs,
'error_file')
and self.pargs.error_file !=
None:
108 errorFileName = self.pargs.error_file
110 except Exception, err:
113 return confLogFileName, timeout, host, port, outputFileName, errorFileName
122 if isinstance(configName, str)
and len(configName) == 0:
125 logging.config.fileConfig(configName)
129 except Exception, err:
137 def fillError(self, errorStr, errorCode, isLogging=True):
142 self.
logger.debug(Utils.getTracebackInfo())
154 if len(host) > 1
and host[0] ==
"\"":
159 addr = host +
":" + port
161 transport.Consts.TCP_TYPE)
184 eventTypesList = [EVENT_TYPES.SITE_NEW, EVENT_TYPES.SITE_UPDATE, EVENT_TYPES.SITE_STATUS, EVENT_TYPES.SITE_DELETE,
185 EVENT_TYPES.SITE_CLEANUP, EVENT_TYPES.URL_NEW, EVENT_TYPES.URL_STATUS, EVENT_TYPES.URL_UPDATE,
186 EVENT_TYPES.URL_FETCH, EVENT_TYPES.URL_DELETE, EVENT_TYPES.URL_CLEANUP, EVENT_TYPES.URL_CONTENT,
187 EVENT_TYPES.SITE_FIND, EVENT_TYPES.SQL_CUSTOM, EVENT_TYPES.BATCH, EVENT_TYPES.URL_PURGE,
188 EVENT_TYPES.FIELD_RECALCULATE, EVENT_TYPES.URL_VERIFY, EVENT_TYPES.URL_AGE, EVENT_TYPES.URL_PUT,
189 EVENT_TYPES.URL_HISTORY, EVENT_TYPES.URL_STATS, EVENT_TYPES.PROXY_NEW, EVENT_TYPES.PROXY_UPDATE,
190 EVENT_TYPES.PROXY_DELETE, EVENT_TYPES.PROXY_STATUS, EVENT_TYPES.PROXY_FIND, EVENT_TYPES.ATTR_SET,
191 EVENT_TYPES.ATTR_UPDATE, EVENT_TYPES.ATTR_DELETE, EVENT_TYPES.ATTR_FETCH]
194 eventType = eventTypesList[CONSTANTS.TASKS.index(task)]
196 self.
logger.
error(
">>> Task name not support = " + str(task))
202 if self.pargs.merge !=
None:
203 mergeVal = bool((self.pargs.merge) > 0)
205 except ValueError
as err:
212 event.cookie[CONSTANTS.COOKIE_MERGE_NAME] = 1
214 event.cookie[CONSTANTS.COOKIE_MERGE_NAME] = 0
218 self.
logger.debug(
"!!! Call 'pool' with 'timeout' = %s", str(timeout))
220 self.
fillError(CONSTANTS.ERROR_STR7.format(str(timeout)), CONSTANTS.ERROR_NETWORK)
224 if hasattr(retEvent,
'eventObj'):
225 ret = retEvent.eventObj
228 except Exception
as err:
229 self.
fillError(
"Error: " + str(err), CONSTANTS.ERROR_EXCEPTION)
241 ffile = open(fileName,
"r") 245 jsonData = json.loads(data) 246 except ValueError, err:
247 self.
logger.
error(
"Deserialize error: %s, json: %s, fileName: '%s'", str(err), str(data), str(fileName))
253 logger.debug(
"!!! command = " + str(command) +
'\njson: ' + str(jsonData))
254 if command == CONSTANTS.TASKS[0]:
255 eventObj = dCCObjectsSerializator.siteNewDeserialize(jsonData)
256 elif command == CONSTANTS.TASKS[1]:
257 eventObj = dCCObjectsSerializator.siteUpdateDeserialize(jsonData)
258 elif command == CONSTANTS.TASKS[2]:
259 eventObj = dCCObjectsSerializator.siteStatusDeserialize(jsonData)
260 elif command == CONSTANTS.TASKS[3]:
261 eventObj = dCCObjectsSerializator.siteDeleteDeserialize(jsonData)
262 elif command == CONSTANTS.TASKS[4]:
263 eventObj = dCCObjectsSerializator.siteCleanupDeserialize(jsonData)
264 elif command == CONSTANTS.TASKS[5]:
265 eventObj = dCCObjectsSerializator.URLNewDeserialize(jsonData)
266 elif command == CONSTANTS.TASKS[6]:
267 eventObj = dCCObjectsSerializator.URLStatusDeserialize(jsonData)
268 elif command == CONSTANTS.TASKS[7]:
269 eventObj = dCCObjectsSerializator.URLUpdateDeserialize(jsonData)
270 elif command == CONSTANTS.TASKS[8]:
271 eventObj = dCCObjectsSerializator.URLFetchDeserialize(jsonData)
272 elif command == CONSTANTS.TASKS[9]:
273 eventObj = dCCObjectsSerializator.URLDeleteDeserialize(jsonData)
274 elif command == CONSTANTS.TASKS[10]:
275 eventObj = dCCObjectsSerializator.URLCleanupDeserialize(jsonData)
276 elif command == CONSTANTS.TASKS[11]:
277 eventObj = dCCObjectsSerializator.URLContentDeserialize(jsonData)
279 elif command == CONSTANTS.TASKS[12]:
281 eventObj = dCCObjectsSerializator.siteFindDeserialize(jsonData)
282 elif command == CONSTANTS.TASKS[13]:
283 eventObj = dCCObjectsSerializator.SQLCustomDeserialize(jsonData)
284 elif command == CONSTANTS.TASKS[14]:
285 eventObj = dCCObjectsSerializator.BatchDeserialize(jsonData)
286 elif command == CONSTANTS.TASKS[15]:
287 eventObj = dCCObjectsSerializator.URLPurgeDeserialize(jsonData)
288 elif command == CONSTANTS.TASKS[16]:
289 eventObj = dCCObjectsSerializator.FieldRecalculatorDeserialize(jsonData)
290 elif command == CONSTANTS.TASKS[17]:
291 eventObj = dCCObjectsSerializator.URLVerifyDeserialize(jsonData)
292 elif command == CONSTANTS.TASKS[18]:
293 eventObj = dCCObjectsSerializator.URLAgeDeserialize(jsonData)
294 elif command == CONSTANTS.TASKS[19]:
295 eventObj = dCCObjectsSerializator.URLPutDeserialize(jsonData)
296 elif command == CONSTANTS.TASKS[20]:
297 eventObj = dCCObjectsSerializator.URLHistoryDeserialize(jsonData)
298 elif command == CONSTANTS.TASKS[21]:
299 eventObj = dCCObjectsSerializator.URLStatsDeserialize(jsonData)
300 elif command == CONSTANTS.TASKS[22]:
301 eventObj = dCCObjectsSerializator.ProxyNewDeserialize(jsonData)
302 elif command == CONSTANTS.TASKS[23]:
303 eventObj = dCCObjectsSerializator.ProxyUpdateDeserialize(jsonData)
304 elif command == CONSTANTS.TASKS[24]:
305 eventObj = dCCObjectsSerializator.ProxyDeleteDeserialize(jsonData)
306 elif command == CONSTANTS.TASKS[25]:
307 eventObj = dCCObjectsSerializator.ProxyStatusDeserialize(jsonData)
308 elif command == CONSTANTS.TASKS[26]:
309 eventObj = dCCObjectsSerializator.ProxyFindDeserialize(jsonData)
310 elif command == CONSTANTS.TASKS[27]:
311 eventObj = dCCObjectsSerializator.AttrSetDeserialize(jsonData)
312 elif command == CONSTANTS.TASKS[28]:
313 eventObj = dCCObjectsSerializator.AttrUpdateDeserialize(jsonData)
314 elif command == CONSTANTS.TASKS[29]:
315 eventObj = dCCObjectsSerializator.AttrDeleteDeserialize(jsonData)
316 elif command == CONSTANTS.TASKS[30]:
317 eventObj = dCCObjectsSerializator.AttrFetchDeserialize(jsonData)
329 if eventObjClassName != [].__class__.__name__:
330 raise Exceptions.WrongEventObjectTypeException(CONSTANTS.ERROR_STR8.format(command,
"list[]"))
332 for listElement
in eventObj:
334 eventObjClassName = listElement.__class__.__name__
335 if eventObjClassName != listElementClassName:
336 raise Exceptions.WrongEventObjectTypeException(CONSTANTS.ERROR_STR8.format(command, excpString + str(i) +
"]"))
345 eventObjClassName = eventObj.__class__.__name__
347 raise Exceptions.WrongEventObjectTypeException(CONSTANTS.ERROR_STR8.format(command,
"ClientResponse"))
349 ret = eventObj.toJSON()
350 except Exception, err:
351 self.
logger.
error(
">>> Serialize error %s %s", Utils.varDump(eventObj), str(err))
352 self.
fillError(
"Error: " + CONSTANTS.ERROR_STR13, CONSTANTS.ERROR_JSON)
359 foundation.CementApp.setup(self)
369 for args
in range(0, CONSTANTS.OBJECT_MAX_INIT_ARGUMENTS):
373 argsList =
",".
join([
'None' for argsList
in xrange(args)])
377 self.
logger.debug(
"Try to instantiate as: %s",
"dc.EventObjects." + objectName +
"(" + argsList +
")")
378 objectInst = eval(
"dc.EventObjects." + objectName +
"(" + argsList +
")")
380 except TypeError
as exp:
382 except Exception, err:
383 self.
logger.
error(
"Create instance of `%s`: %s : type is `%s`",
384 str(objectName), str(err), str(
type(objectInst)))
385 raise Exceptions.DeserilizeException(
"Create instance of object error: " + str(err))
389 for name
in dir(objectInst):
390 if not name.startswith(
'__'):
391 setattr(objectInst, name,
None)
392 except Exception
as exp:
393 self.
logger.
error(
"Build object `%s`: %s", str(objectName), str(exp.message))
394 raise Exceptions.DeserilizeException(
"Build object error: " + str(exp.message))
407 fieldsDict = json.loads(fieldsJson)
411 self.
logger.debug(
"fieldsDict: %s", str(fieldsDict))
412 self.
logger.debug(
"objectInst: %s", Utils.varDump(objectInst))
413 self.
logger.debug(
"mode: %s", self.pargs.mode)
415 if isinstance(objectInst, list):
418 for itemObj
in objectInst:
420 for name
in dir(itemObj):
421 if not name.startswith(
'__')
and name
in fieldsDict:
422 self.
setAttr(itemObj, name, fieldsDict[name], self.pargs.mode)
423 newList.append(itemObj)
427 for name
in dir(objectInst):
428 if not name.startswith(
'__')
and name
in fieldsDict:
430 self.
setAttr(objectInst, name, fieldsDict[name], self.pargs.mode)
432 except Exception
as err:
433 ExceptionLog.handler(self.
logger, err,
"Fill object error:")
434 self.
logger.
error(
"type: `%s`,\nfields: `%s`", str(
type(objectInst)), str(fieldsJson))
435 raise Exceptions.DeserilizeException(
"Fill object error: " + str(err))
448 if mode
is None or mode ==
"" or mode ==
"0" or (
not hasattr(obj, name)):
450 setattr(obj, name, valsDict)
453 objAttr = getattr(obj, name)
455 if str(
type(objAttr)).startswith(
'<class'):
457 for name2
in dir(objAttr):
458 if not name2.startswith(
'__'):
460 if isinstance(valsDict, dict)
and name2
in valsDict:
461 objAttr2 = getattr(objAttr, name2)
463 if str(
type(objAttr2)).startswith(
'<class'):
464 self.
setAttr(objAttr, name2, valsDict[name2], mode)
466 setattr(objAttr, name2, self.
mergeDictList(objAttr, valsDict[name2], mode))
467 setattr(obj, name, objAttr)
470 setattr(obj, name, self.
mergeDictList(objAttr, valsDict, mode))
485 if isinstance(iter1, dict)
and isinstance(iter2, dict):
487 if mode
is None or mode ==
"" or mode ==
"0":
489 retIter = dict(iter2)
492 retIter = dict(iter1)
494 for key, value
in iter2.iteritems():
496 if key
in retIter
or mode ==
"2":
499 if isinstance(iter1, list)
and isinstance(iter2, list):
501 if mode
is None or mode ==
"" or mode ==
"0" or mode ==
"1":
503 retIter = list(iter2)
506 retIter = list(iter1)
512 if isinstance(iter2, dict):
514 retIter = dict(iter2)
516 if isinstance(iter2, list):
518 retIter = list(iter2)
532 for k, v
in d.iteritems():
533 if isinstance(v, dict):
536 if (isinstance(v, str)
or isinstance(v, unicode))
and v.startswith(CONSTANTS.FILE_PROTOCOL_SIGNATURE):
537 self.
logger.debug(
"File:%s", str(v[len(CONSTANTS.FILE_PROTOCOL_SIGNATURE):]))
538 d[k] = open(v[len(CONSTANTS.FILE_PROTOCOL_SIGNATURE):],
'r').read() 546 foundation.CementApp.run(self)
548 if '-h' in self.argv
or '--help' in self.argv:
553 startTime = time.time()
556 confLogFileName, timeout, host, port, outputFileName, errorFileName = self.
__loadAppConfig(self.pargs.config)
561 if '-v' in self.argv
or '--verbose' in self.argv:
562 self.
logger.info(
"Started, args: %s", str(self.argv))
564 if self.
errorCode == CONSTANTS.ERROR_NOERROR:
567 except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
568 self.
fillError(CONSTANTS.ERROR_STR10, CONSTANTS.ERROR_CONFIG_SECTION)
569 if isHelpArg
is False and self.pargs.command
is None:
570 self.
fillError(CONSTANTS.ERROR_STR1, CONSTANTS.ERROR_ARGS1)
572 self.
fillError(CONSTANTS.ERROR_STR12, CONSTANTS.ERROR_CONNECTION)
573 elif self.pargs.command !=
None:
574 if self.pargs.command
in CONSTANTS.TASKS:
575 if self.pargs.file !=
None or self.pargs.fields !=
None:
577 if self.pargs.file !=
None:
583 CONSTANTS.TASKS_OBJECTS[CONSTANTS.TASKS.index(self.pargs.command)])
585 if self.pargs.fields !=
None:
589 self.
logger.debug(
"resulted eventObj: %s", Utils.varDump(eventObj))
591 self.
fillError(CONSTANTS.ERROR_STR4, CONSTANTS.ERROR_BAD_FILE_NAME)
593 self.
fillError(CONSTANTS.ERROR_STR5, CONSTANTS.ERROR_BAD_JSON)
594 except Exceptions.DeserilizeException
as excp:
595 self.
fillError(CONSTANTS.ERROR_STR6.format(excp.message), CONSTANTS.ERROR_DCC)
598 if retEventObj !=
None:
601 except Exceptions.WrongEventObjectTypeException
as excp:
602 self.
fillError(excp.message, CONSTANTS.ERROR_WRONG_RESPONSE)
604 elif self.
errorCode == CONSTANTS.ERROR_NOERROR:
605 self.
fillError(CONSTANTS.ERROR_STR14, CONSTANTS.ERROR_OBJECT_CREATE)
607 self.
fillError(CONSTANTS.ERROR_STR2, CONSTANTS.ERROR_ARGS2)
609 self.
fillError(CONSTANTS.ERROR_STR3, CONSTANTS.ERROR_BAD_TASK)
610 except Exception, err:
611 self.
fillError(str(err), CONSTANTS.ERROR_INITIALIZATION,
False)
616 if outputFileName
is None:
618 sys.stdout.write(jsonBuf)
624 if errorFileName
is not None:
628 if '-v' in self.argv
or '--verbose' in self.argv:
629 self.
logger.info(
"Finished, time: %s sec", str(time.time() - startTime))
632 self.
logger.info(APP_CONSTS.LOGGER_DELIMITER_LINE)
639 foundation.CementApp.close(self)
649 if fileName
is not None and isinstance(fileName, basestring)
and fileName !=
"":
650 f = open(fileName,
'w')
654 raise Exception(
"Bad parameter 'fileName' = '%s'" % str(fileName))
656 except Exception, err:
657 self.
logger.
error(
"Write to file '%s' failed, error: '%s'", str(fileName), str(err))
669 return json.dumps(dataDict)
def transportCommunications(self, task, eventObj, timeout)
def fillError(self, errorStr, errorCode, isLogging=True)
string ERROR_FILE_ERROR_CODE_FIELD_NAME
def __loadAppConfig(self, configName)
string MSG_ERROR_WRONG_CONFIG_FILE_NAME
string MSG_ERROR_EMPTY_CONFIG_FILE_NAME
def generateEmptyResponse(self)
def commandListSerialize(self, command, eventObj, eventObjClassName, listElementClassName, excpString)
def setAttr(self, obj, name, valsDict, mode)
def makeErrorFileContent(self)
string MSG_ERROR_READ_LOG_CONFIG
Class hides routines of bulding connection objects.
string MSG_ERROR_LOAD_APP_CONFIG
def connectionInit(self, host, port)
def writeToFile(self, fileName, data)
def mergeDictList(self, iter1, iter2, mode)
def commandProcessingBuild(self, objectName)
def varDump(obj, stringify=True, strTypeMaxLen=256, strTypeCutSuffix='...', stringifyType=1, ignoreErrors=False, objectsHash=None, depth=0, indent=2, ensure_ascii=False, maxDepth=10)
def commandProcessingFill(self, objectInst, fieldsJson)
def __init__(self, connectionBuilder=None)
string ERROR_FILE_ERROR_MESSAGE_FIELD_NAME
def commandProcessingSerialize(self, command, eventObj)
def dictFillFromFile(self, d)
def commandProcessingDeserialize(self, command, fileName)
def __readLoggerConfig(self, configName)