HCE Project Python language Distributed Tasks Manager Application, Distributed Crawler Application and client API bindings.  2.0.0-chaika
Hierarchical Cluster Engine Python language binding
DCC.py
Go to the documentation of this file.
1 '''
2 Created on Apr 10, 2014
3 
4 @package: dc
5 @author: scorp
6 @link: http://hierarchical-cluster-engine.com/
7 @copyright: Copyright © 2013-2014 IOIX Ukraine
8 @license: http://hierarchical-cluster-engine.com/license/
9 @since: 0.1
10 '''
11 import ConfigParser
12 import logging
13 import logging.config
14 import json
15 import sys
16 import time
17 from cement.core import foundation
18 import transport.Consts
19 from transport.ConnectionBuilderLight import ConnectionBuilderLight
20 from transport.Event import EventBuilder
21 import dc.EventObjects
22 from dc.Constants import EVENT_TYPES as EVENT_TYPES
23 import app.Consts as APP_CONSTS
24 import app.Utils as Utils # pylint: disable=F0401
25 import app.Exceptions as Exceptions
26 from app.Utils import ExceptionLog
27 from app.Utils import varDump
28 import dcc.Constants as CONSTANTS
29 from dcc.DCCObjectsSerializator import DCCObjectsSerializator
30 from algorithms.BaseMetric import logger
31 
32 
33 # #DCC Class contents main functional of DCC application, class inherits from foundation.CementApp
34 #
35 class DCC(foundation.CementApp):
36 
37  # # Constants error messages used in class
38  MSG_ERROR_EMPTY_CONFIG_FILE_NAME = "Config file name is empty."
39  MSG_ERROR_WRONG_CONFIG_FILE_NAME = "Config file name is wrong"
40  MSG_ERROR_LOAD_APP_CONFIG = "Error loading application config file."
41  MSG_ERROR_READ_LOG_CONFIG = "Error read log config file."
42 
43  ERROR_FILE_ERROR_MESSAGE_FIELD_NAME = 'errorMessage'
44  ERROR_FILE_ERROR_CODE_FIELD_NAME = 'errorCode'
45 
46  class Meta: # pylint: disable=W0232, C1001
47  label = CONSTANTS.APP_NAME
48 
49  # #constructor
50  # initialise all class variable and recieve connectionBuilder as param(not mandatory)
51  def __init__(self, connectionBuilder=None):
52  foundation.CementApp.__init__(self)
53  # self.config = ConfigParser.ConfigParser()
54  self.connectionBuilder = connectionBuilder
55  self.localConnection = None
57  self.errorCode = CONSTANTS.ERROR_NOERROR
58  self.errorStr = ""
59  self.sendEventType = None
60  self.sendEventObject = None
61  self.logger = None
62 
63 
64  # #load application config file
65  #
66  # @param configName - name of application config file
67  # @return - profiler config file name
68  def __loadAppConfig(self, configName):
69  # variable for result
70  confLogFileName = ""
71  timeout = CONSTANTS.TCP_TIMEOUT
72  host = ''
73  port = ''
74  outputFileName = None
75  errorFileName = None
76 
77  try:
78  config = ConfigParser.ConfigParser()
79  config.optionxform = str
80 
81  readOk = config.read(configName)
82 
83  if len(readOk) == 0:
84  raise Exception(self.MSG_ERROR_WRONG_CONFIG_FILE_NAME + ": " + configName)
85 
86  if config.has_section(APP_CONSTS.CONFIG_APPLICATION_SECTION_NAME):
87  confLogFileName = str(config.get(CONSTANTS.LOG_CONFIG_SECTION_NAME, CONSTANTS.LOG_CONFIG_OPTION_NAME))
88 
89  if config.has_section(CONSTANTS.APP_NAME):
90  timeout = config.get(CONSTANTS.APP_NAME, CONSTANTS.TCP_TIMEOUT_CONFIG_NAME)
91  host = str(config.get(CONSTANTS.APP_NAME, CONSTANTS.DTM_HOST))
92  port = str(config.get(CONSTANTS.APP_NAME, CONSTANTS.DTM_PORT))
93 
94  if hasattr(self.pargs, 'dcc_timeout') and self.pargs.dcc_timeout != None and \
95  self.pargs.dcc_timeout.isdigit():
96  timeout = int(self.pargs.dcc_timeout)
97 
98  if hasattr(self.pargs, 'dcc_clientHost') and self.pargs.dcc_clientHost != None:
99  host = self.pargs.dcc_clientHost
100 
101  if hasattr(self.pargs, 'dcc_clientPort') and self.pargs.dcc_clientPort != None:
102  port = self.pargs.dcc_clientPort
103 
104  if hasattr(self.pargs, 'output_file') and self.pargs.output_file != None:
105  outputFileName = self.pargs.output_file
106 
107  if hasattr(self.pargs, 'error_file') and self.pargs.error_file != None:
108  errorFileName = self.pargs.error_file
109 
110  except Exception, err:
111  raise Exception(self.MSG_ERROR_LOAD_APP_CONFIG + ' ' + str(err))
112 
113  return confLogFileName, timeout, host, port, outputFileName, errorFileName
114 
115 
116  # #load logger config file
117  #
118  # @param configName - name of logger config file
119  # @return - None
120  def __readLoggerConfig(self, configName):
121  try:
122  if isinstance(configName, str) and len(configName) == 0:
123  raise Exception(self.MSG_ERROR_EMPTY_CONFIG_FILE_NAME)
124 
125  logging.config.fileConfig(configName)
126  # call rotation log files and initialization logger
127  self.logger = Utils.MPLogger().getLogger()
128 
129  except Exception, err:
130  raise Exception(self.MSG_ERROR_READ_LOG_CONFIG + ' ' + str(err))
131 
132 
133  # #fillError method
134  # calls from error-code point from main processing (...from event handlers)
135  # errorStr - external errorMessage
136  # errorCode - external errorCode
137  def fillError(self, errorStr, errorCode, isLogging=True):
138  self.errorCode = errorCode
139  self.errorStr = errorStr
140  if isLogging:
141  self.logger.error("Error msg: %s, error code: %s", str(self.errorStr), str(self.errorCode))
142  self.logger.debug(Utils.getTracebackInfo())
143 
144 
145  # #connectionInit method
146  # initializes internal variables that containts network connections/communications
147  #
148  # @param host - host name value
149  # @param port - port value
150  def connectionInit(self, host, port):
151  if self.connectionBuilder is None:
153 
154  if len(host) > 1 and host[0] == "\"":
155  host = host[1:]
156  if host[-1] == "\"":
157  host = host[0:-1]
158 
159  addr = host + ":" + port
160  self.localConnection = self.connectionBuilder.build(transport.Consts.CLIENT_CONNECT, addr,
161  transport.Consts.TCP_TYPE)
162 
163 
164  # #generateEmptyResponse method
165  # If here was some critical error, we generate empty response here , instead real response
166  # task - task arg
169  obj.error_code = self.errorCode
170  obj.error_message = self.errorStr
171  return obj.toJSON()
172 
173 
174  # #taskProcessingDeserialize method
175  # Method serializes incoming task to the JSON string
176  # @param task - task arg
177  # @param eventObj - response event object
178  # @param timeout - timeout value
179  def transportCommunications(self, task, eventObj, timeout):
180  retEvent = None
181  ret = None
182  eventType = None
183 
184  eventTypesList = [EVENT_TYPES.SITE_NEW, EVENT_TYPES.SITE_UPDATE, EVENT_TYPES.SITE_STATUS, EVENT_TYPES.SITE_DELETE,
185  EVENT_TYPES.SITE_CLEANUP, EVENT_TYPES.URL_NEW, EVENT_TYPES.URL_STATUS, EVENT_TYPES.URL_UPDATE,
186  EVENT_TYPES.URL_FETCH, EVENT_TYPES.URL_DELETE, EVENT_TYPES.URL_CLEANUP, EVENT_TYPES.URL_CONTENT,
187  EVENT_TYPES.SITE_FIND, EVENT_TYPES.SQL_CUSTOM, EVENT_TYPES.BATCH, EVENT_TYPES.URL_PURGE,
188  EVENT_TYPES.FIELD_RECALCULATE, EVENT_TYPES.URL_VERIFY, EVENT_TYPES.URL_AGE, EVENT_TYPES.URL_PUT,
189  EVENT_TYPES.URL_HISTORY, EVENT_TYPES.URL_STATS, EVENT_TYPES.PROXY_NEW, EVENT_TYPES.PROXY_UPDATE,
190  EVENT_TYPES.PROXY_DELETE, EVENT_TYPES.PROXY_STATUS, EVENT_TYPES.PROXY_FIND, EVENT_TYPES.ATTR_SET,
191  EVENT_TYPES.ATTR_UPDATE, EVENT_TYPES.ATTR_DELETE, EVENT_TYPES.ATTR_FETCH]
192 
193  try:
194  eventType = eventTypesList[CONSTANTS.TASKS.index(task)]
195  except ValueError:
196  self.logger.error(">>> Task name not support = " + str(task))
197 
198  self.sendEventType = eventType
199  self.sendEventObject = eventObj
200  mergeVal = None
201  try:
202  if self.pargs.merge != None:
203  mergeVal = bool((self.pargs.merge) > 0)
204 
205  except ValueError as err:
206  pass
207 
208  event = self.eventBuilder.build(eventType, eventObj)
209  if mergeVal != None:
210  event.cookie = {}
211  if mergeVal:
212  event.cookie[CONSTANTS.COOKIE_MERGE_NAME] = 1
213  else:
214  event.cookie[CONSTANTS.COOKIE_MERGE_NAME] = 0
215  self.localConnection.send(event)
216 
217  try:
218  self.logger.debug("!!! Call 'pool' with 'timeout' = %s", str(timeout))
219  if self.localConnection.poll(timeout) == 0:
220  self.fillError(CONSTANTS.ERROR_STR7.format(str(timeout)), CONSTANTS.ERROR_NETWORK)
221  else:
222  retEvent = self.localConnection.recv()
223  if retEvent != None:
224  if hasattr(retEvent, 'eventObj'):
225  ret = retEvent.eventObj
226  else:
227  self.fillError("retEvent hasn't 'eventObj', dump: %s", varDump(retEvent))
228  except Exception as err:
229  self.fillError("Error: " + str(err), CONSTANTS.ERROR_EXCEPTION)
230 
231  return ret
232 
233 
234  # #commandProcessingDeserialize method
235  # Reads task from file and deserializes it.
236  # command - command arg
237  # fileName - json filename
238  def commandProcessingDeserialize(self, command, fileName):
239 
240  try:
241  ffile = open(fileName, "r")
242  data = ffile.read()
243  ffile.close()
244 
245  jsonData = json.loads(data)
246  except ValueError, err:
247  self.logger.error("Deserialize error: %s, json: %s, fileName: '%s'", str(err), str(data), str(fileName))
248  raise err
249 
250  dCCObjectsSerializator = DCCObjectsSerializator()
251  eventObj = None
252 
253  logger.debug("!!! command = " + str(command) + '\njson: ' + str(jsonData))
254  if command == CONSTANTS.TASKS[0]:
255  eventObj = dCCObjectsSerializator.siteNewDeserialize(jsonData)
256  elif command == CONSTANTS.TASKS[1]:
257  eventObj = dCCObjectsSerializator.siteUpdateDeserialize(jsonData) # pylint: disable=R0204
258  elif command == CONSTANTS.TASKS[2]:
259  eventObj = dCCObjectsSerializator.siteStatusDeserialize(jsonData)
260  elif command == CONSTANTS.TASKS[3]:
261  eventObj = dCCObjectsSerializator.siteDeleteDeserialize(jsonData)
262  elif command == CONSTANTS.TASKS[4]:
263  eventObj = dCCObjectsSerializator.siteCleanupDeserialize(jsonData)
264  elif command == CONSTANTS.TASKS[5]:
265  eventObj = dCCObjectsSerializator.URLNewDeserialize(jsonData)
266  elif command == CONSTANTS.TASKS[6]:
267  eventObj = dCCObjectsSerializator.URLStatusDeserialize(jsonData)
268  elif command == CONSTANTS.TASKS[7]:
269  eventObj = dCCObjectsSerializator.URLUpdateDeserialize(jsonData)
270  elif command == CONSTANTS.TASKS[8]:
271  eventObj = dCCObjectsSerializator.URLFetchDeserialize(jsonData)
272  elif command == CONSTANTS.TASKS[9]:
273  eventObj = dCCObjectsSerializator.URLDeleteDeserialize(jsonData)
274  elif command == CONSTANTS.TASKS[10]:
275  eventObj = dCCObjectsSerializator.URLCleanupDeserialize(jsonData)
276  elif command == CONSTANTS.TASKS[11]:
277  eventObj = dCCObjectsSerializator.URLContentDeserialize(jsonData)
278  # find site by root url
279  elif command == CONSTANTS.TASKS[12]:
280  # logger.debug("jsonData: %s", str(jsonData))
281  eventObj = dCCObjectsSerializator.siteFindDeserialize(jsonData)
282  elif command == CONSTANTS.TASKS[13]:
283  eventObj = dCCObjectsSerializator.SQLCustomDeserialize(jsonData)
284  elif command == CONSTANTS.TASKS[14]:
285  eventObj = dCCObjectsSerializator.BatchDeserialize(jsonData)
286  elif command == CONSTANTS.TASKS[15]:
287  eventObj = dCCObjectsSerializator.URLPurgeDeserialize(jsonData)
288  elif command == CONSTANTS.TASKS[16]:
289  eventObj = dCCObjectsSerializator.FieldRecalculatorDeserialize(jsonData)
290  elif command == CONSTANTS.TASKS[17]:
291  eventObj = dCCObjectsSerializator.URLVerifyDeserialize(jsonData)
292  elif command == CONSTANTS.TASKS[18]:
293  eventObj = dCCObjectsSerializator.URLAgeDeserialize(jsonData)
294  elif command == CONSTANTS.TASKS[19]:
295  eventObj = dCCObjectsSerializator.URLPutDeserialize(jsonData)
296  elif command == CONSTANTS.TASKS[20]:
297  eventObj = dCCObjectsSerializator.URLHistoryDeserialize(jsonData)
298  elif command == CONSTANTS.TASKS[21]:
299  eventObj = dCCObjectsSerializator.URLStatsDeserialize(jsonData)
300  elif command == CONSTANTS.TASKS[22]:
301  eventObj = dCCObjectsSerializator.ProxyNewDeserialize(jsonData)
302  elif command == CONSTANTS.TASKS[23]:
303  eventObj = dCCObjectsSerializator.ProxyUpdateDeserialize(jsonData)
304  elif command == CONSTANTS.TASKS[24]:
305  eventObj = dCCObjectsSerializator.ProxyDeleteDeserialize(jsonData)
306  elif command == CONSTANTS.TASKS[25]:
307  eventObj = dCCObjectsSerializator.ProxyStatusDeserialize(jsonData)
308  elif command == CONSTANTS.TASKS[26]:
309  eventObj = dCCObjectsSerializator.ProxyFindDeserialize(jsonData)
310  elif command == CONSTANTS.TASKS[27]:
311  eventObj = dCCObjectsSerializator.AttrSetDeserialize(jsonData)
312  elif command == CONSTANTS.TASKS[28]:
313  eventObj = dCCObjectsSerializator.AttrUpdateDeserialize(jsonData)
314  elif command == CONSTANTS.TASKS[29]:
315  eventObj = dCCObjectsSerializator.AttrDeleteDeserialize(jsonData)
316  elif command == CONSTANTS.TASKS[30]:
317  eventObj = dCCObjectsSerializator.AttrFetchDeserialize(jsonData)
318  return eventObj
319 
320 
321  # #commandListSerialize method
322  # Method implements common serialize processing for list of elements
323  # command - command arg
324  # eventObj - response event object
325  # eventObjClassName - list class name
326  # listElementClassName - list element class name
327  # excpString - exception substring
328  def commandListSerialize(self, command, eventObj, eventObjClassName, listElementClassName, excpString):
329  if eventObjClassName != [].__class__.__name__:
330  raise Exceptions.WrongEventObjectTypeException(CONSTANTS.ERROR_STR8.format(command, "list[]"))
331  i = 0
332  for listElement in eventObj:
333  i += 1
334  eventObjClassName = listElement.__class__.__name__
335  if eventObjClassName != listElementClassName:
336  raise Exceptions.WrongEventObjectTypeException(CONSTANTS.ERROR_STR8.format(command, excpString + str(i) + "]"))
337 
338 
339  # #commandProcessingSerialize method
340  # Method serializes incoming task to the JSON string
341  # command - command arg
342  # eventObj - response event object
343  def commandProcessingSerialize(self, command, eventObj):
344  ret = None
345  eventObjClassName = eventObj.__class__.__name__
346  if eventObjClassName != dc.EventObjects.ClientResponse([]).__class__.__name__:
347  raise Exceptions.WrongEventObjectTypeException(CONSTANTS.ERROR_STR8.format(command, "ClientResponse"))
348  try:
349  ret = eventObj.toJSON()
350  except Exception, err:
351  self.logger.error(">>> Serialize error %s %s", Utils.varDump(eventObj), str(err))
352  self.fillError("Error: " + CONSTANTS.ERROR_STR13, CONSTANTS.ERROR_JSON)
353  return ret
354 
355 
356  # #setup method
357  # Method calls before run application
358  def setup(self):
359  foundation.CementApp.setup(self)
360 
361 
362  # #Command processing build object instance by command name
363  # initialize fields with None value
364  # @param objectName name of an object class
365  #
366  def commandProcessingBuild(self, objectName):
367  objectInst = None
368 
369  for args in range(0, CONSTANTS.OBJECT_MAX_INIT_ARGUMENTS):
370  try:
371  # Iteratively try to create instance with mandatory constructor arguments
372  if args > 0:
373  argsList = ",".join(['None' for argsList in xrange(args)])
374  else:
375  argsList = ""
376  # Try to create object instance by name
377  self.logger.debug("Try to instantiate as: %s", "dc.EventObjects." + objectName + "(" + argsList + ")")
378  objectInst = eval("dc.EventObjects." + objectName + "(" + argsList + ")") # pylint: disable=W0123
379  break
380  except TypeError as exp:
381  continue
382  except Exception, err:
383  self.logger.error("Create instance of `%s`: %s : type is `%s`",
384  str(objectName), str(err), str(type(objectInst)))
385  raise Exceptions.DeserilizeException("Create instance of object error: " + str(err))
386 
387  # Fill with None value all fields
388  try:
389  for name in dir(objectInst):
390  if not name.startswith('__'):
391  setattr(objectInst, name, None)
392  except Exception as exp:
393  self.logger.error("Build object `%s`: %s", str(objectName), str(exp.message))
394  raise Exceptions.DeserilizeException("Build object error: " + str(exp.message))
395 
396  return objectInst
397 
398 
399  # #Command processing fill object instance attributes from fields set json
400  # initialize fields with None value
401  # @param objectInst object instance
402  # @param fieldsJson fields json
403  #
404  def commandProcessingFill(self, objectInst, fieldsJson):
405  try:
406  # De-serialize fields set json
407  fieldsDict = json.loads(fieldsJson)
408 
409  # Process load from file reference
410  fieldsDict = self.dictFillFromFile(fieldsDict)
411  self.logger.debug("fieldsDict: %s", str(fieldsDict))
412  self.logger.debug("objectInst: %s", Utils.varDump(objectInst))
413  self.logger.debug("mode: %s", self.pargs.mode)
414 
415  if isinstance(objectInst, list):
416  newList = []
417  # Set fields in object if it is list of objects
418  for itemObj in objectInst:
419  # logger.debug("Type of item is: %s", str(type(itemObj)))
420  for name in dir(itemObj):
421  if not name.startswith('__') and name in fieldsDict:
422  self.setAttr(itemObj, name, fieldsDict[name], self.pargs.mode)
423  newList.append(itemObj)
424  objectInst = newList
425  else:
426  # Set fields in object if it is single
427  for name in dir(objectInst):
428  if not name.startswith('__') and name in fieldsDict:
429  # setattr(objectInst, name, fieldsDict[name])
430  self.setAttr(objectInst, name, fieldsDict[name], self.pargs.mode)
431 
432  except Exception as err:
433  ExceptionLog.handler(self.logger, err, "Fill object error:")
434  self.logger.error("type: `%s`,\nfields: `%s`", str(type(objectInst)), str(fieldsJson))
435  raise Exceptions.DeserilizeException("Fill object error: " + str(err))
436 
437  return objectInst
438 
439 
440  # #Sets object attributes without inheritance overwrite or not
441  #
442  # @param obj object instance
443  # @param name attribute name of object
444  # @param valsDict dict of attribute values
445  # @param mode 0 - not nested overwrite mode, 1 - nested overwrite, 2 - nested overwrite with append
446  #
447  def setAttr(self, obj, name, valsDict, mode):
448  if mode is None or mode == "" or mode == "0" or (not hasattr(obj, name)):
449  # Without nesting, only one first level of fields overwrite
450  setattr(obj, name, valsDict)
451  else:
452  # With nesting support, overwrite and/or append
453  objAttr = getattr(obj, name)
454  # if hasattr(objAttr, '__class__'):
455  if str(type(objAttr)).startswith('<class'):
456  # If attribute is a class
457  for name2 in dir(objAttr):
458  if not name2.startswith('__'):
459  # print str(name2) + " | " + str(valsDict)
460  if isinstance(valsDict, dict) and name2 in valsDict:
461  objAttr2 = getattr(objAttr, name2)
462  # if hasattr(objAttr2, "__class__"):
463  if str(type(objAttr2)).startswith('<class'):
464  self.setAttr(objAttr, name2, valsDict[name2], mode)
465  else:
466  setattr(objAttr, name2, self.mergeDictList(objAttr, valsDict[name2], mode))
467  setattr(obj, name, objAttr)
468  else:
469  # If attribute is not a class
470  setattr(obj, name, self.mergeDictList(objAttr, valsDict, mode))
471 
472 
473  # #Merges two dictionaries with nesting of levels
474  #
475  # @param iter1 dictionary or list to merge
476  # @param iter2 dictionary or list to merge with
477  # @param mode 0 - not nested overwrite mode, 1 - nested for dicts and overwrite for lists,
478  # 2 - nested overwrite with append if not present for dicts and append for lists
479  # If iter1 and iter2 is not the same types or not dict and list type - iter2 overwrites iter1 not nested way (mode 0),
480  # and nested way (mode 1 and 2)
481  #
482  def mergeDictList(self, iter1, iter2, mode):
483  retIter = None
484 
485  if isinstance(iter1, dict) and isinstance(iter2, dict):
486  # Both iterable are dicts
487  if mode is None or mode == "" or mode == "0":
488  # Just overwrite with copy
489  retIter = dict(iter2)
490  else:
491  # Init with source copy
492  retIter = dict(iter1)
493  # Process source items
494  for key, value in iter2.iteritems():
495  # If key exists in source and destination or mode 2 (need to be appended)
496  if key in retIter or mode == "2":
497  retIter[key] = self.mergeDictList(retIter[key], value, mode)
498  else:
499  if isinstance(iter1, list) and isinstance(iter2, list):
500  # Both iterable are lists
501  if mode is None or mode == "" or mode == "0" or mode == "1":
502  # Just overwrite with copy
503  retIter = list(iter2) # pylint: disable=R0204
504  else:
505  # Init with source copy
506  retIter = list(iter1)
507  # Process source items
508  for item in iter2:
509  retIter.append(item)
510  else:
511  # Types of iter1 and iter2 are not the same
512  if isinstance(iter2, dict):
513  # Overwrite with dict copy
514  retIter = dict(iter2)
515  else:
516  if isinstance(iter2, list):
517  # Overwrite with list copy
518  retIter = list(iter2)
519  else:
520  # Overwrite with copy of none iterable type or reference of the object
521  retIter = iter2
522 
523  return retIter
524 
525 
526  # #Command processing fill object instance attributes from fields set json
527  # initialize fields with None value
528  # @param objectInst object instance
529  # @param fieldsJson fields json
530  #
531  def dictFillFromFile(self, d):
532  for k, v in d.iteritems():
533  if isinstance(v, dict):
534  d[k] = self.dictFillFromFile(v)
535  else:
536  if (isinstance(v, str) or isinstance(v, unicode)) and v.startswith(CONSTANTS.FILE_PROTOCOL_SIGNATURE):
537  self.logger.debug("File:%s", str(v[len(CONSTANTS.FILE_PROTOCOL_SIGNATURE):]))
538  d[k] = open(v[len(CONSTANTS.FILE_PROTOCOL_SIGNATURE):], 'r').read()
539 
540  return d
541 
542 
543  # #run method
544  # Method contains main application functionality
545  def run(self):
546  foundation.CementApp.run(self)
547  isHelpArg = False
548  if '-h' in self.argv or '--help' in self.argv:
549  isHelpArg = True
550  eventObj = None
551  retEventObj = None
552  jsonBuf = None
553  startTime = time.time()
554 
555  try:
556  confLogFileName, timeout, host, port, outputFileName, errorFileName = self.__loadAppConfig(self.pargs.config)
557  self.__readLoggerConfig(confLogFileName)
558 
559  self.logger.debug("!!! Dump pargs: %s", varDump(self.pargs))
560 
561  if '-v' in self.argv or '--verbose' in self.argv:
562  self.logger.info("Started, args: %s", str(self.argv))
563 
564  if self.errorCode == CONSTANTS.ERROR_NOERROR:
565  try:
566  self.connectionInit(host, port)
567  except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
568  self.fillError(CONSTANTS.ERROR_STR10, CONSTANTS.ERROR_CONFIG_SECTION)
569  if isHelpArg is False and self.pargs.command is None:
570  self.fillError(CONSTANTS.ERROR_STR1, CONSTANTS.ERROR_ARGS1)
571  elif self.localConnection is None:
572  self.fillError(CONSTANTS.ERROR_STR12, CONSTANTS.ERROR_CONNECTION)
573  elif self.pargs.command != None:
574  if self.pargs.command in CONSTANTS.TASKS:
575  if self.pargs.file != None or self.pargs.fields != None:
576  try:
577  if self.pargs.file != None:
578  # De-serialize object from json file
579  eventObj = self.commandProcessingDeserialize(self.pargs.command, self.pargs.file)
580  else:
581  # Dynamic object creation and initialize all fields with None
582  eventObj = self.commandProcessingBuild(
583  CONSTANTS.TASKS_OBJECTS[CONSTANTS.TASKS.index(self.pargs.command)])
584  # Cover some fields with values from command line argument
585  if self.pargs.fields != None:
586  # Fill some object's fields with values
587  eventObj = self.commandProcessingFill(eventObj, self.pargs.fields)
588  # logger.debug("resulted eventObj: %s", str(eventObj))
589  self.logger.debug("resulted eventObj: %s", Utils.varDump(eventObj))
590  except IOError:
591  self.fillError(CONSTANTS.ERROR_STR4, CONSTANTS.ERROR_BAD_FILE_NAME)
592  except ValueError:
593  self.fillError(CONSTANTS.ERROR_STR5, CONSTANTS.ERROR_BAD_JSON)
594  except Exceptions.DeserilizeException as excp:
595  self.fillError(CONSTANTS.ERROR_STR6.format(excp.message), CONSTANTS.ERROR_DCC)
596  if eventObj != None:
597  retEventObj = self.transportCommunications(self.pargs.command, eventObj, timeout)
598  if retEventObj != None:
599  try:
600  jsonBuf = self.commandProcessingSerialize(self.pargs.command, retEventObj)
601  except Exceptions.WrongEventObjectTypeException as excp:
602  self.fillError(excp.message, CONSTANTS.ERROR_WRONG_RESPONSE)
603 
604  elif self.errorCode == CONSTANTS.ERROR_NOERROR:
605  self.fillError(CONSTANTS.ERROR_STR14, CONSTANTS.ERROR_OBJECT_CREATE)
606  else:
607  self.fillError(CONSTANTS.ERROR_STR2, CONSTANTS.ERROR_ARGS2)
608  else:
609  self.fillError(CONSTANTS.ERROR_STR3, CONSTANTS.ERROR_BAD_TASK)
610  except Exception, err:
611  self.fillError(str(err), CONSTANTS.ERROR_INITIALIZATION, False)
612 
613  if jsonBuf is None:
614  jsonBuf = self.generateEmptyResponse()
615 
616  if outputFileName is None:
617  # output to stdout
618  sys.stdout.write(jsonBuf)
619  sys.stdout.flush()
620  else:
621  # output to output file
622  self.writeToFile(outputFileName, jsonBuf)
623 
624  if errorFileName is not None:
625  self.writeToFile(errorFileName, self.makeErrorFileContent())
626 
627  # self.pargs.verbose
628  if '-v' in self.argv or '--verbose' in self.argv:
629  self.logger.info("Finished, time: %s sec", str(time.time() - startTime))
630 
631  # Finish logging
632  self.logger.info(APP_CONSTS.LOGGER_DELIMITER_LINE)
633 
634 
635 
636  # #close method
637  # Method calls after application run
638  def close(self):
639  foundation.CementApp.close(self)
640 
641 
642  # # Write to file some data
643  #
644  # @param fileName- file name
645  # @param data - data string buffer
646  # @return - None
647  def writeToFile(self, fileName, data):
648  try:
649  if fileName is not None and isinstance(fileName, basestring) and fileName != "":
650  f = open(fileName, 'w')
651  f.write(data)
652  f.close()
653  else:
654  raise Exception("Bad parameter 'fileName' = '%s'" % str(fileName))
655 
656  except Exception, err:
657  self.logger.error("Write to file '%s' failed, error: '%s'", str(fileName), str(err))
658 
659 
660  # # Make error file content
661  # @param - None
662  # @return json string
664 
665  dataDict = {}
666  dataDict[self.ERROR_FILE_ERROR_MESSAGE_FIELD_NAME] = self.errorStr
667  dataDict[self.ERROR_FILE_ERROR_CODE_FIELD_NAME] = self.errorCode
668 
669  return json.dumps(dataDict)
def transportCommunications(self, task, eventObj, timeout)
Definition: DCC.py:179
def fillError(self, errorStr, errorCode, isLogging=True)
Definition: DCC.py:137
string ERROR_FILE_ERROR_CODE_FIELD_NAME
Definition: DCC.py:44
def __loadAppConfig(self, configName)
Definition: DCC.py:68
string MSG_ERROR_WRONG_CONFIG_FILE_NAME
Definition: DCC.py:39
def close(self)
Definition: DCC.py:638
errorStr
Definition: DCC.py:58
string MSG_ERROR_EMPTY_CONFIG_FILE_NAME
Definition: DCC.py:38
def generateEmptyResponse(self)
Definition: DCC.py:167
errorCode
Definition: DCC.py:57
connectionBuilder
Definition: DCC.py:54
def commandListSerialize(self, command, eventObj, eventObjClassName, listElementClassName, excpString)
Definition: DCC.py:328
eventBuilder
Definition: DCC.py:56
def setAttr(self, obj, name, valsDict, mode)
Definition: DCC.py:447
def makeErrorFileContent(self)
Definition: DCC.py:663
sendEventObject
Definition: DCC.py:60
string MSG_ERROR_READ_LOG_CONFIG
Definition: DCC.py:41
Class hides routines of bulding connection objects.
sendEventType
Definition: DCC.py:59
logger
Definition: DCC.py:61
string MSG_ERROR_LOAD_APP_CONFIG
Definition: DCC.py:40
def run(self)
Definition: DCC.py:545
def connectionInit(self, host, port)
Definition: DCC.py:150
def writeToFile(self, fileName, data)
Definition: DCC.py:647
def mergeDictList(self, iter1, iter2, mode)
Definition: DCC.py:482
def commandProcessingBuild(self, objectName)
Definition: DCC.py:366
def varDump(obj, stringify=True, strTypeMaxLen=256, strTypeCutSuffix='...', stringifyType=1, ignoreErrors=False, objectsHash=None, depth=0, indent=2, ensure_ascii=False, maxDepth=10)
Definition: Utils.py:410
def setup(self)
Definition: DCC.py:358
-mask-info
def commandProcessingFill(self, objectInst, fieldsJson)
Definition: DCC.py:404
def __init__(self, connectionBuilder=None)
Definition: DCC.py:51
string ERROR_FILE_ERROR_MESSAGE_FIELD_NAME
Definition: DCC.py:43
localConnection
Definition: DCC.py:55
Definition: join.py:1
def commandProcessingSerialize(self, command, eventObj)
Definition: DCC.py:343
def dictFillFromFile(self, d)
Definition: DCC.py:531
def commandProcessingDeserialize(self, command, fileName)
Definition: DCC.py:238
def __readLoggerConfig(self, configName)
Definition: DCC.py:120