hce-node application
1.4.3
HCE Hierarchical Cluster Engine node application
|
#include <DataProcessorData.hpp>
Classes | |
class | TasksStateNotificationFunctor |
Public Member Functions | |
DataProcessorData (const std::string &nodeName, std::string name, zmq::context_t &ctx, int64_t startedAt, unsigned char logPriority, const std::string &iniFile) | |
~DataProcessorData () | |
Public Member Functions inherited from HCE::handlers::Handler | |
Handler (const std::string &name, zmq::context_t &context, const std::string &clientIdentity, const std::string &connectionString, int64_t startedAt, unsigned char logPriority, const std::string &iniFile="") | |
virtual | ~Handler () |
void | shutdown (void) |
Get handler instance name. | |
std::string | getName (void) |
Protected Member Functions | |
int | initialize (void) |
void | deinitialize (void) |
void * | process (void) |
void | handleInternalMessage (void) |
std::string | processDataRequest (const std::string &messageBody) |
std::string | processAdminCommands (std::string &command, std::string ¶meters) |
std::string | processAdminCommandsSphinx (std::string &command) |
Node name setter. | |
void | setNodeName (const std::string &nodeName) |
Node name getter. | |
std::string | getNodeName (void) |
Plain text requests data processor. Used for test purposes as a minimal simple data generator and processor as well as messages handling algorithms. | |
std::string | processDataRequestPlainText (const std::string &messageBody) |
Json requests data processor. Used to check the request target type and chose proper metod to handle: Sphinx, sql, etc... | |
std::string | processDataRequestJson (const std::string &messageBody) |
Sphinx requests data processor by Sphinx FO | |
std::string | processDataRequestSphinx (const std::string &messageBody) |
Sphinx FO initialization from ini file | |
void | dataStorageInitSphinx (const std::string &iniFile) |
Node initialization from ini file | |
void | nodeInit (const std::string &iniFile) |
Check initialization and set default value if necessary. | |
void | checkInit (void) |
Prepare dump node properties. | |
void | saveNodeProperties (void) |
Prepare dump properties for Sphinx FO. | |
void | saveSphinxProperties (void) |
Prepare dump properties for DRCE FO. | |
void | saveDRCEProperties (void) |
Restore node properties. | |
void | restoreNodeProperties (void) |
Restore properties for Sphinx FO. | |
void | restoreSphinxProperties (void) |
Restore properties for DRCE FO. | |
void | restoreDRCEProperties (void) |
Get current user home dir | |
std::string | getDataStorageSphinxProperties (void) |
Get data storage properties for DRCE FO | |
std::string | getDataStorageDRCEProperties (void) |
DRCE requests data processor by DRCE FO | |
std::string | processDataRequestDRCE (const std::string &messageBody) |
DRCE FO initialization from ini file | |
void | dataStorageInitDRCE (const std::string &iniFile) |
Rebuilds tasks state notification connection. | |
bool | rebuildTasksStateNotificationConnection () |
Get property information. | |
std::string | getPropertyInformation (void) |
ZMQ Sockets | |
void | setStatClientResponses (unsigned long long statClientResponses) |
unsigned long long | getStatClientResponses (void) |
void | setStatSphinxRequestsSearch (unsigned long long statSphinxRequestsSearch) |
unsigned long long | getStatSphinxRequestsSearch (void) |
void | setStatSphinxRequestsAdmin (unsigned long long statSphinxRequestsAdmin) |
unsigned long long | getStatSphinxRequestsAdmin (void) |
void | setStatSphinxTimeTotalSearch (unsigned long long statSphinxTimeTotalSearch) |
unsigned long long | getStatSphinxTimeTotalSearch (void) |
void | setStatSphinxTimeTotalAdmin (unsigned long long statSphinxTimeTotalAdmin) |
unsigned long long | getStatSphinxTimeTotalAdmin (void) |
void | setStatDRCERequests (unsigned long long statDRCERequests) |
unsigned long long | getStatDRCERequests (void) |
void | setStatDRCETimeTotal (unsigned long long statDRCETimeTotal) |
unsigned long long | getStatDRCETimeTotal (void) |
Sphinx FO instance | |
void | setDataProcessingMode (unsigned int dataProcessingMode) |
unsigned int | getDataProcessingMode (void) |
void | setDataProcessingFakeResults (unsigned int dataProcessingFakeResults) |
unsigned int | getDataProcessingFakeResults (void) |
void | setDataProcessingFakeResultsMax (unsigned int dataProcessingFakeResultsMax) |
unsigned int | getDataProcessingFakeResultsMax (void) |
Node number. | |
void | setNodeNumber (unsigned int NodeNumber) |
unsigned int | getNodeNumber (void) |
Tasks state notification update service. | |
void | setStateNotificationHost (const std::string &stateNotificationHost) |
std::string | getStateNotificationHost (void) |
void | setStateNotificationPort (const std::string &stateNotificationPort) |
std::string | getStateNotificationPort (void) |
Protected Member Functions inherited from HCE::handlers::Handler | |
void | setName (const std::string &name) |
<Set handler instance name | |
void | setClientIdentity (const std::string &clientIdentity) |
Get client ID. | |
std::string | getClientIdentity (void) |
Set start time mark. | |
void | setStartedAt (int64_t startedAt) |
Get start time mark. | |
int64_t | getStartedAt (void) |
Set log priority level. | |
void | setLogPriority (unsigned int logPriority) |
Get log priority level. | |
unsigned int | getLogPriority (void) |
Set ini file name. | |
void | setIniFile (const std::string &iniFile) |
Get ini file name. | |
std::string | getIniFile (void) |
Set ZMQ connections rebuild tries max number. | |
void | setBindTriesMaxNumber (const unsigned int &bindTriesMaxNumber) |
Get ZMQ connections rebuild tries max number. | |
unsigned int | getBindTriesMaxNumber (void) |
Set ZMQ connections rebuild tries delay. | |
void | setBindTriesDelay (const unsigned int &bindTriesDelay) |
Get ZMQ connections rebuild tries delay. | |
unsigned int | getBindTriesDelay (void) |
Set ZMQ connection string. | |
void | setConnectionString (const std::string &connectionString) |
Get ZMQ connection string. | |
std::string | getConnectionString (void) |
Set flag for polling condition. | |
void | setInProgress (bool in_progress) |
Get flag for polling condition. | |
bool | getInProgress (void) const |
Set poll timeout value. | |
void | setPollTimeout (unsigned int pollTimeout) |
Get poll timeout value. | |
unsigned int | getPollTimeout (void) |
Set property interval between resend Property values. | |
void | setPropertyInterval (unsigned int propertyInterval) |
Get property interval between resend Property values. | |
unsigned int | getPropertyInterval (void) |
Set interval between dump run-time variables. | |
void | setDumpInterval (unsigned int dumpInterval) |
Get interval between dump run-time variables. | |
unsigned int | getDumpInterval (void) |
Set flag for allowed restore variables use dump. | |
void | setDumpAllowRestore (unsigned int dumpAllowRestore) |
Get interval between dump run-time variables. | |
unsigned int | getDumpAllowRestore (void) |
void | logFlush (unsigned int priority=NODE_LOG_MODE_DEFAULT) |
Node init from file. | |
void | initHandler (const std::string &iniFile, LoggerStream &logger) |
void | handleAdminMessage (void) |
void | processAdminMessage (zmsg &msg) |
void | rebuildAdminClientSocket (void) |
bool | rebuildServerConnection (zmq::socket_t *&sock_p, const std::string &connectionString, const int type) |
void | rebuildClientConnection (zmq::socket_t *&sock_p, const std::string &connectionString, const int type, const std::string &identity="") |
bool | rebuildClientConnection (zmq::socket_t *&sock_p, const std::string &connectionString, const int type, const std::string &identity, int linger, const std::string &readyMessage="", const std::string &byeMessage=NODE_MSG_BYE) |
void | sendByeMessage (zmq::socket_t *&sock_p, const std::string &identity="", const std::string &byeMessage=NODE_MSG_BYE) |
std::string | getProperties (void) |
Get property information. | |
void | sendPeriodicProperties (void) |
Safe create directories if necessary. | |
void | createDirectories (const std::string &name) |
Get file name use for dump and restore properties values. | |
std::string | getDumpFileName (void) |
Get file name use for dump and restore statistic values. | |
std::string | getStatFileName (void) |
Dump properties variables. | |
bool | dumpProperties (void) |
Dump run-time variables at regular intervals. | |
bool | dumpRunTimeVariables (void) |
void | handleExternalMessage (void) |
void | processControlMessage (zmsg &msg) |
void | processDataMessage (zmsg &msg) |
void | logPeriodicStatistics (void) |
void | heartbit (void) |
void | registerClient (const std::string &identity) |
Admin socket. |
Static Protected Member Functions | |
static std::string | getUserHomeDir (void) |
Get data storage properties for Sphinx FO |
Protected Attributes | |
zmq::socket_t * | _inprocDataSock |
zmq::socket_t * | _tasksStateNotificationSock |
Setters and getters for stat variables. | |
HCE::sphinx::SphinxFunctionalObject * | _sphinxFO |
DRCE FO instance | |
HCE::drce::DRCEFunctionalObject * | _drceFO |
Node data processing mode, 0 - regular with involving of FO, 1 - fake w/o involving FO and random results generation | |
TasksStateNotificationFunctor * | _tasksStateNotificationFunctor |
Protected Attributes inherited from HCE::handlers::Handler | |
zmq::socket_t * | _inprocAdminSock |
Storage for run-time property variables. | |
HandlerProperties | handlerProperties |
Storage for run-time statistic variables. | |
HandlerProperties | statProperties |
ZMQ context. | |
zmq::context_t & | _context |
std::bitset< 4 > | _logMask |
std::string | _logTimeFormat |
std::stringstream | log |
LoggerStream | logger |
Main loop of sockets polling condition. | |
bool | _inProgress |
Send out periodical property values to Admin class at regular intervals. | |
int64_t | _propertyIntervalAt |
Dump run-time variables at regular intervals. | |
int64_t | _dumpIntervalAt |
Dump dir name. | |
std::string | _dumpDir |
Additional Inherited Members | |
Static Public Member Functions inherited from HCE::handlers::Handler | |
static void * | main (void *that) |
Definition at line 57 of file DataProcessorData.hpp.
HCE::handlers::DataProcessorData::DataProcessorData | ( | const std::string & | nodeName, |
std::string | name, | ||
zmq::context_t & | ctx, | ||
int64_t | startedAt, | ||
unsigned char | logPriority, | ||
const std::string & | iniFile | ||
) |
<Init all the rest
Definition at line 5 of file DataProcessorData.cpp.
HCE::handlers::DataProcessorData::~DataProcessorData | ( | ) |
|
protected |
Prepare dump node properties.
<Check all stat variables
Definition at line 205 of file DataProcessorData.cpp.
|
protected |
Rebuilds tasks state notification connection.
<DRCE FO init parameters
<Process FO's ini file
<Log some error of config file processing
<Log some error of config file processing
<Create DRCE FO and init
<Log some error of DRCE FO
<Set tasks queue dump period interval (in msec.)
<Set resource monitor time period (in msec.)
<Set mandatory environment variables
<Set max threads count for resource extractor
<Set max processes count for resource extractor
<Set max disk size for resource extractor
<Set max virtual memory size for resource extractor
<Set max rss memory size for resource extractor
<Create connection socket and connect to tasks state notifications service
<Create instance or the notification functor
<Set notification function to DRCE FO instance
<Set flag reset error state for notification
Definition at line 563 of file DataProcessorData.cpp.
|
protected |
Node initialization from ini file
<Sphinx FO init parameters
<Process FO's ini file
<Log some error of config file processing
<Log some error of config file processing
<Create Sphinx Sphinx FO and init
<Log some error of Sphinx FO
<Set ranker
Definition at line 455 of file DataProcessorData.cpp.
|
protectedvirtual |
Implements HCE::handlers::Handler.
Definition at line 57 of file DataProcessorData.cpp.
|
protected |
Definition at line 169 of file DataProcessorData.cpp.
|
protected |
Node number.
Definition at line 177 of file DataProcessorData.cpp.
|
protected |
Definition at line 161 of file DataProcessorData.cpp.
|
protected |
DRCE requests data processor by DRCE FO
Definition at line 1286 of file DataProcessorData.cpp.
|
protected |
Get data storage properties for DRCE FO
Definition at line 1266 of file DataProcessorData.cpp.
|
protected |
Plain text requests data processor. Used for test purposes as a minimal simple data generator and processor as well as messages handling algorithms.
Definition at line 97 of file DataProcessorData.cpp.
|
protected |
Tasks state notification update service.
Definition at line 185 of file DataProcessorData.cpp.
|
protectedvirtual |
ZMQ Sockets
Reimplemented from HCE::handlers::Handler.
Definition at line 1247 of file DataProcessorData.cpp.
|
protected |
Definition at line 105 of file DataProcessorData.cpp.
|
protected |
Definition at line 145 of file DataProcessorData.cpp.
|
protected |
Sphinx FO instance
Definition at line 153 of file DataProcessorData.cpp.
|
protected |
Definition at line 193 of file DataProcessorData.cpp.
|
protected |
Definition at line 201 of file DataProcessorData.cpp.
|
protected |
Definition at line 121 of file DataProcessorData.cpp.
|
protected |
Definition at line 113 of file DataProcessorData.cpp.
|
protected |
Definition at line 137 of file DataProcessorData.cpp.
|
protected |
Definition at line 129 of file DataProcessorData.cpp.
|
staticprotected |
Get data storage properties for Sphinx FO
Definition at line 810 of file DataProcessorData.cpp.
|
protected |
<Fetch message
<Get identity and body of request message message
<Process data request
<Create response message
<Send response
Reimplemented from HCE::handlers::Handler.
Definition at line 1151 of file DataProcessorData.cpp.
|
protectedvirtual |
<Init sockets
<Node init
<Check initialization and set default value if necessary
<Sphinx FO init
<DRCE FO init
<Restore node properties
<Restore Sphinx FO properties
<Restore DRCE FO properties
<Rebuild admin client socket
<Rebuild inproc_proxy connection
Implements HCE::handlers::Handler.
Definition at line 19 of file DataProcessorData.cpp.
|
protected |
Check initialization and set default value if necessary.
<Node init parameters
<Process Node's ini file
<Log some error of config file processing
<Log some error of config file processing
Definition at line 822 of file DataProcessorData.cpp.
|
protectedvirtual |
<Enter main loop of processing of message
<Init polling pool
<Poll inproc_data server
<Handle income data message
<Dump run-time variables at regular intervals
Implements HCE::handlers::Handler.
Definition at line 1401 of file DataProcessorData.cpp.
|
protectedvirtual |
<Process commands
<Make property information
<Parse command parameters and fetch processing mode value
<Sphinx functional object command
<DRCE functional object command
<Prepare response message
<Prepare response message
<Unsupported command
Reimplemented from HCE::handlers::Handler.
Definition at line 1182 of file DataProcessorData.cpp.
|
protected |
Node name setter.
<Create input object for Sphinx FO
<Reset searcher error state to get next message to start from scratch
<Execute admin request
<Log some error of Sphinx FO
<Reset indexer error state to get next message to start from scratch
<Log some error of indexer request execution
<Log some error of indexer request execution
<Return default response with general error
Definition at line 1349 of file DataProcessorData.cpp.
|
protected |
<Possible json message
<Plain text message
Definition at line 879 of file DataProcessorData.cpp.
|
protected |
DRCE FO initialization from ini file
<Create input object for DRCE FO
<Return empty string to reducer in case of DRCE state error
<Reset DRCE error state to get next message to start from scratch
<Execute DRCE request
<Log some error of DRCE FO
<Reset DRCE error state to get next message to start from scratch
<Log some error of DRCE request execution
<Return DRCE results data json string of data from the DRCEOutputJsonMessage object
<Register DRCE request time
<Return empty json string to reducer in case of de-serialisation error to direct the processing on json type
<Fill error code and error message for default json
<Register DRCE requests number stat
Definition at line 1036 of file DataProcessorData.cpp.
|
protected |
Sphinx requests data processor by Sphinx FO
<If serialization error return default result to reducer
<If serialization error return default result to reducer
Definition at line 897 of file DataProcessorData.cpp.
|
protected |
Json requests data processor. Used to check the request target type and chose proper metod to handle: Sphinx, sql, etc...
<Parse request and fetch items number
<Generate items
<Randomize items values
<Sord items
<Serialize items
Definition at line 1121 of file DataProcessorData.cpp.
|
protected |
Sphinx FO initialization from ini file
<If serialization error return default result to reducer
<Create input object for Sphinx FO
<Return empty string to reducer in case of searcher state error
<Reset searcher error state to get next message to start from scratch
<Execute search request
<Log some error of Sphinx FO
<Reset searcher error state to get next message to start from scratch
<Log some error of search request execution
<Return Sphinx results data json string of data from Sphinx FO
<Register Sphinx request time
<Return empty json string to reducer in case of de-serialisation error to direct the processing on json type
Definition at line 955 of file DataProcessorData.cpp.
|
protected |
Get property information.
Definition at line 441 of file DataProcessorData.cpp.
|
protected |
Get current user home dir
Definition at line 377 of file DataProcessorData.cpp.
|
protected |
Restore properties for Sphinx FO.
Definition at line 287 of file DataProcessorData.cpp.
|
protected |
Restore properties for DRCE FO.
Definition at line 335 of file DataProcessorData.cpp.
|
protected |
Restore node properties.
<Dump stat variables
Definition at line 260 of file DataProcessorData.cpp.
|
protected |
Prepare dump properties for Sphinx FO.
Definition at line 229 of file DataProcessorData.cpp.
|
protected |
Prepare dump properties for DRCE FO.
Definition at line 245 of file DataProcessorData.cpp.
|
protected |
Definition at line 165 of file DataProcessorData.cpp.
|
protected |
Definition at line 173 of file DataProcessorData.cpp.
|
protected |
Definition at line 157 of file DataProcessorData.cpp.
|
protected |
Node name getter.
Definition at line 92 of file DataProcessorData.cpp.
|
protected |
Definition at line 181 of file DataProcessorData.cpp.
|
protected |
Definition at line 101 of file DataProcessorData.cpp.
|
protected |
Definition at line 141 of file DataProcessorData.cpp.
|
protected |
Definition at line 149 of file DataProcessorData.cpp.
|
protected |
Definition at line 189 of file DataProcessorData.cpp.
|
protected |
Definition at line 197 of file DataProcessorData.cpp.
|
protected |
Definition at line 117 of file DataProcessorData.cpp.
|
protected |
Definition at line 109 of file DataProcessorData.cpp.
|
protected |
Definition at line 133 of file DataProcessorData.cpp.
|
protected |
Definition at line 125 of file DataProcessorData.cpp.
|
protected |
Node data processing mode, 0 - regular with involving of FO, 1 - fake w/o involving FO and random results generation
Definition at line 166 of file DataProcessorData.hpp.
|
protected |
Definition at line 135 of file DataProcessorData.hpp.
|
protected |
DRCE FO instance
Definition at line 163 of file DataProcessorData.hpp.
|
protected |
Definition at line 225 of file DataProcessorData.hpp.
|
protected |
Setters and getters for stat variables.
Definition at line 138 of file DataProcessorData.hpp.