HCE Project Python language Distributed Tasks Manager Application, Distributed Crawler Application and client API bindings.  2.0.0-chaika
Hierarchical Cluster Engine Python language binding
DTMA.py
Go to the documentation of this file.
1 '''
2 Created on Mar 25, 2014
3 
4 @package: dtm
5 @author: scorp
6 @link: http://hierarchical-cluster-engine.com/
7 @copyright: Copyright © 2013-2014 IOIX Ukraine
8 @license: http://hierarchical-cluster-engine.com/license/
9 '''
10 
11 import logging
12 #import logging.config
13 import json
14 import sys
15 import ConfigParser
16 from cement.core import foundation
17 from transport.ConnectionBuilderLight import ConnectionBuilderLight
18 from transport.Event import EventBuilder
19 from dtm.Constants import EVENT_TYPES as EVENT_TYPES
20 from DTMAObjectsFiller import DTMAObjectsFiller
21 import transport.Consts
22 import dtm.EventObjects
23 import DTMAExceptions
24 import Constants as CONSTANTS
25 import app.Consts as APP_CONSTS
26 import app.Utils as Utils
27 
28 
29 # Logger initialization
30 logger = logging.getLogger(APP_CONSTS.LOGGER_NAME)
31 #logger = logging.getLogger(CONSTANTS.APP_NAME)
32 
33 
34 
36 class DTMA(foundation.CementApp):
37 
38  class Meta: # pylint: disable=W0232, C1001
39  label = CONSTANTS.APP_NAME
40 
41 
42 
44  def __init__(self, connectionBuilder=None):
45  foundation.CementApp.__init__(self)
46  self.config = ConfigParser.ConfigParser()
47  self.connectionBuilder = connectionBuilder
48  self.localConnection = None
51  self.errorCode = CONSTANTS.ERROR_NOERROR
52  self.errorStr = ""
53 
54 
55 
57  def fillError(self, errorStr, errorCode):
58  self.errorCode = errorCode
59  self.errorStr = errorStr
60  logger.error(self.errorStr)
61 
62 
63 
65  def connectionInit(self):
66  if self.connectionBuilder is None:
68  host = str(self.config.get(CONSTANTS.APP_NAME, CONSTANTS.DTM_HOST))
69  if len(host) > 1 and host[0] == "\"":
70  host = host[1:]
71  if host[-1] == "\"":
72  host = host[0:-1]
73  port = str(self.config.get(CONSTANTS.APP_NAME, CONSTANTS.DTM_PORT))
74  addr = host + ":" + port
75  self.localConnection = self.connectionBuilder.build(transport.Consts.CLIENT_CONNECT, addr,
76  transport.Consts.TCP_TYPE)
77 
78 
79 
81  def generateEmptyResponse(self, task):
82  ret = []
83  obj = None
84  if task is None:
86  elif task == CONSTANTS.TASKS[0]:
88  elif task == CONSTANTS.TASKS[1]:
90  elif task == CONSTANTS.TASKS[2]:
92  elif task == CONSTANTS.TASKS[3]:
94  elif task == CONSTANTS.TASKS[4]:
96  elif task == CONSTANTS.TASKS[5]:
98  elif task == CONSTANTS.TASKS[6]:
99  obj = dtm.EventObjects.CustomResponse(None, None, None)
100 
101  obj.errorCode = self.errorCode
102  obj.errorMessage = self.errorStr
103  ret.append(obj)
104  return json.dumps([ret], default=lambda o: o.__dict__, sort_keys=True, indent=4)
105 
106 
107 
109  def configReader(self):
110  configReadList = []
111  if self.pargs.config is None:
112  configReadList = self.config.read(CONSTANTS.DEFAULT_CONFIG_NAME1)
113  if len(configReadList) == 0:
114  configReadList = self.config.read(CONSTANTS.DEFAULT_CONFIG_NAME2)
115  else:
116  configReadList = self.config.read(self.pargs.config)
117  return len(configReadList)
118 
119 
120 
122  def checkAdditionalArgs(self, cmd):
123  ret = True
124  if cmd in CONSTANTS.TASKS[1:3]:
125  if self.pargs.fields is None or self.pargs.classes is None:
126  self.fillError(CONSTANTS.ERROR_STR2, CONSTANTS.ERROR_ARGS2)
127  ret = False
128  elif cmd == CONSTANTS.TASKS[0] or cmd == CONSTANTS.TASKS[3]:
129  if self.pargs.classes is None:
130  optionName = [CONSTANTS.SERVER_CONFIG_OPTION_NAME, CONSTANTS.SERVER_CONFIG_OPTION_NAME2]\
131  [cmd == CONSTANTS.TASKS[3]]
132  try:
133  self.pargs.classes = str(self.config.get(CONSTANTS.SERVER_CONFIG_SECTION_NAME, optionName))
134  except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
135  self.fillError(CONSTANTS.ERROR_STR3.format(CONSTANTS.SERVER_CONFIG_SECTION_NAME, optionName),
136  CONSTANTS.ERROR_CONFIG_SECTION)
137  ret = False
138  return ret
139 
140 
141 
143  def transportCommunications(self, cmd, requestObjects):
144  responses = []
145  retEvent = None
146  eventType = None
147  timeout = None
148  responseTuple = []
149  emptyResponse = None
150  try:
151  timeout = self.config.get(CONSTANTS.APP_NAME, CONSTANTS.TCP_TIMEOUT_CONFIG_NAME)
152  except ConfigParser.NoSectionError:
153  timeout = CONSTANTS.TCP_TIMEOUT
154  except ConfigParser.NoOptionError:
155  timeout = CONSTANTS.TCP_TIMEOUT
156 
157  for requestObject in requestObjects:
158  if cmd == CONSTANTS.TASKS[0]:
159  eventType = EVENT_TYPES.ADMIN_FETCH_STAT_DATA
160  emptyResponse = dtm.EventObjects.AdminStatData(requestObject.className)
161  elif cmd == CONSTANTS.TASKS[1]:
162  eventType = EVENT_TYPES.ADMIN_SET_CONFIG_VARS
163  emptyResponse = dtm.EventObjects.AdminConfigVars(requestObject.className)
164  elif cmd == CONSTANTS.TASKS[2]:
165  eventType = EVENT_TYPES.ADMIN_GET_CONFIG_VARS
166  emptyResponse = dtm.EventObjects.AdminConfigVars(requestObject.className)
167  elif cmd == CONSTANTS.TASKS[3]:
168  eventType = EVENT_TYPES.ADMIN_STATE
169  emptyResponse = dtm.EventObjects.GeneralResponse()
170  elif cmd == CONSTANTS.TASKS[4]:
171  eventType = EVENT_TYPES.ADMIN_SUSPEND
172  emptyResponse = dtm.EventObjects.GeneralResponse()
173  elif cmd == CONSTANTS.TASKS[5]:
174  eventType = EVENT_TYPES.ADMIN_SYSTEM
175  emptyResponse = dtm.EventObjects.GeneralResponse()
176  elif cmd == CONSTANTS.TASKS[6]:
177  eventType = EVENT_TYPES.ADMIN_SQL_CUSTOM
178  emptyResponse = dtm.EventObjects.CustomResponse(None, None, None)
179 
180  event = self.eventBuilder.build(eventType, requestObject)
181  self.localConnection.send(event)
182  if self.localConnection.poll(timeout) == 0:
183  responseTuple = [emptyResponse, CONSTANTS.ERROR_NETWORK]
184  else:
185  retEvent = self.localConnection.recv()
186  responseTuple = [retEvent.eventObj, CONSTANTS.ERROR_NOERROR]
187  responses.append(responseTuple)
188  return responses
189 
190 
191 
194  def loadLogConfigFile(self):
195  global logger # pylint: disable=W0603
196  try:
197  logIniFileName = self.config.get(CONSTANTS.LOG_CONFIG_SECTION_NAME, CONSTANTS.LOG_CONFIG_OPTION_NAME)
198  if logIniFileName != None:
199  logging.config.fileConfig(logIniFileName, disable_existing_loggers=False)
200  except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
201  self.errorStr = CONSTANTS.ERROR_STR8
202  self.errorCode = CONSTANTS.ERROR_LOG_SECTION_ERROR
203  except Exception, err:
204  self.errorStr = CONSTANTS.ERROR_STR10 + ': ' + str(err)
205  self.errorCode = CONSTANTS.ERROR_LOG_INIT
206  logger = Utils.MPLogger().getLogger()
207 
208 
209 
210 
212  def setup(self):
213  foundation.CementApp.setup(self)
214 
215 
216 
218  def run(self):
219  foundation.CementApp.run(self)
220  isHelpArg = False
221  if '-h' in self.argv or '--help' in self.argv:
222  isHelpArg = True
223  requestObjects = None
224  jsonBuf = None
225  if self.configReader() > 0:
226  self.loadLogConfigFile()
227 
228  if self.errorCode == CONSTANTS.ERROR_NOERROR:
229  try:
230  self.connectionInit()
231  except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
232  self.fillError(CONSTANTS.ERROR_STR3, CONSTANTS.ERROR_CONFIG_SECTION)
233  if not isHelpArg and self.pargs.cmd is None:
234  self.fillError(CONSTANTS.ERROR_STR1, CONSTANTS.ERROR_ARGS1)
235  elif self.pargs.cmd is not None and self.pargs.cmd in CONSTANTS.TASKS and \
236  self.checkAdditionalArgs(self.pargs.cmd):
237  try:
238  requestObjects = self.dTMAObjectsFiller.generateObjectsList(self.pargs.cmd, self.pargs.fields,
239  self.pargs.classes)
240  responseObjects = self.transportCommunications(self.pargs.cmd, requestObjects)
241  jsonBuf = json.dumps(responseObjects, default=lambda o: o.__dict__, sort_keys=True, indent=4)
243  self.fillError(CONSTANTS.ERROR_STR4, CONSTANTS.ERROR_FIELDS_ARG)
245  self.fillError(CONSTANTS.ERROR_STR5, CONSTANTS.ERROR_CLASSES_ARG)
246  else:
247  self.errorStr = CONSTANTS.ERROR_STR9
248  self.errorCode = CONSTANTS.ERROR_NO_CONFIG
249 
250  if jsonBuf is None:
251  jsonBuf = self.generateEmptyResponse(self.pargs.cmd)
252  sys.stdout.write(jsonBuf)
253  # Finish logging
254  logger.info(APP_CONSTS.LOGGER_DELIMITER_LINE)
255 
256 
257 
259  def close(self):
260  foundation.CementApp.close(self)
def checkAdditionalArgs(self, cmd)
checkAdditionalArgs method Method checks additions mandatory argument's present
Definition: DTMA.py:122
localConnection
Definition: DTMA.py:48
def transportCommunications(self, cmd, requestObjects)
taskProcessingDeserialize method Method serializes incoming task to the JSON string ...
Definition: DTMA.py:143
def loadLogConfigFile(self)
load logging load logging configuration (log file, log level, filters)
Definition: DTMA.py:194
DTMA Class contents main functional of DTMA application, class inherits from foundation.CementApp.
Definition: DTMA.py:36
connectionBuilder
Definition: DTMA.py:47
GeneralResponse event object, represents general state response for multipurpose usage.
dTMAObjectsFiller
Definition: DTMA.py:50
def configReader(self)
configReader method Method try to read config file by prereared paths, return count of readed configs...
Definition: DTMA.py:109
AdminConfigVars event object, for admin set or get config variables from any of threaded classes or a...
DTMCObjectsFiller Class contents serialize/deserialize methods for incoming "DTMA" commands...
def fillError(self, errorStr, errorCode)
fillError method calls from error-code point from main processing (...from event handlers) ...
Definition: DTMA.py:57
Class hides routines of bulding connection objects.
def close(self)
close method Method calls after application run
Definition: DTMA.py:259
AdminStatData event object, for admin fetch stat fields and possible data from any threaded classes i...
def setup(self)
setup method Method calls before run application
Definition: DTMA.py:212
def __init__(self, connectionBuilder=None)
constructor initialise all class variable and recieve connectionBuilder as param(not mandatory) ...
Definition: DTMA.py:44
DTMAExceptions module keepts DTMA module native exceptions.
def connectionInit(self)
connectionInit method initializes internal variables that containts network connections/communication...
Definition: DTMA.py:65
def run(self)
run method Method contains main application functionality
Definition: DTMA.py:218
def generateEmptyResponse(self, task)
generateEmptyResponse method If here was some critical error, we generate empty response here ...
Definition: DTMA.py:81