hce-node application  1.4.3
HCE Hierarchical Cluster Engine node application
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros
SphinxSearcher.cpp
Go to the documentation of this file.
1 #include <string.h>
2 #include <stdio.h>
3 #include <sstream>
4 #include <iterator>
5 #include <time.h>
6 #include <sys/time.h>
7 #include <unistd.h>
8 #include <Poco/AutoPtr.h>
9 
10 #include "SphinxSearcher.hpp"
11 #include "SphinxInputJsonMessage.hpp"
12 #include "SphinxOutputJsonMessage.hpp"
13 #include "SphinxError.hpp"
14 #include "ProcExec.hpp"
15 #include "SphinxAdminCommand.hpp"
16 #include "SphinxConfigCreator.hpp"
17 #include "SphinxWeightCalculator.hpp"
18 #include "SphinxInputJsonMessageSearch.hpp"
19 #include "SphinxMessageConst.hpp"
20 
21 namespace HCE
22 {
23 namespace sphinx
24 {
25 //-------------------------------------------------------------------
26 SphinxSearcher::SphinxSearcher(SphinxFunctionalObject& fObj_, bool startSearchd_, bool stopSearchd_)
27 :SphinxSearchInterface(),
28  client(nullptr), field_names(nullptr), field_weights(nullptr), fieldsCount(0), index_names(nullptr), index_weights(nullptr),
29  indexesCount(0), fObj(fObj_), message(fObj.getCustomMessage()),
30  serverHost(sphinx_search_const::defaultHost), serverPort(sphinx_search_const::defaultPort), indexFileName(""),
31  _StopSearchd(stopSearchd_), _isConnected(false), _isStoppedSearchd(true), timeoutedRequests(0), defaultRequestTimeout(input_json_message_const::defaultTimeoutValue),
32  allowedFields(), fullSchemaNames(), requestInfo(), resultData(), resultSerializator(resultData), resultMemoryManager(resultData)
33 {
34  client = sphinx_create(SPH_TRUE);
35  if (!client)
36  errorMsg = message(message_const::FAILED_TO_CREATE_CLIENT);
37 }
38 //-------------------------------------------------------------------
39 SphinxSearcher::~SphinxSearcher(void)
40 {
41  if (_StopSearchd)
42  stopSearchd();
43  sphinx_close(client);
44  sphinx_destroy(client);
45  cleanup(field_names, field_weights, fieldsCount);
46  cleanup(index_names, index_weights, indexesCount);
47 }
48 //-------------------------------------------------------------------
49 void SphinxSearcher::resizeResultData(size_t matchesCount, size_t attributesCount)
50 {
51  resultMemoryManager.resize(matchesCount, attributesCount);
52 }
53 //-------------------------------------------------------------------
54 void SphinxSearcher::setAllowedFields(std::vector<std::string>& queryExtendedFields)
55 {
56  allowedFields.clear();
57  for (size_t i=0;i<queryExtendedFields.size();++i)
58  allowedFields.insert(queryExtendedFields[i]);
59 }
60 //-------------------------------------------------------------------
61 bool SphinxSearcher::isAllowedFields(const std::string& extendedFieldName)
62 {
63  return (allowedFields.find(extendedFieldName)!=allowedFields.end());
64 }
65 //-------------------------------------------------------------------
66 void SphinxSearcher::setFullSchemaNames(std::vector<std::string>& schemaNames)
67 {
68  fullSchemaNames.clear();
69  for (size_t i=0;i<schemaNames.size();++i)
70  fullSchemaNames.insert(schemaNames[i]);
71 }
72 //-------------------------------------------------------------------
73 bool SphinxSearcher::isExistSchemaNames(const std::string& schemaName)
74 {
75  return (fullSchemaNames.find(schemaName)!=fullSchemaNames.end());
76 }
77 //-------------------------------------------------------------------
78 bool SphinxSearcher::startSearchd(void)
79 {
80  _isError = false;
81  std::string configFile = fObj.getDataDir()+"/"+fObj.getIndexName()+"/"+sphinx_search_const::sphinxConfigName;
82 
83  fObj.log(LoggerStream::Priority::PRIO_INFORMATION) << message(message_const::START_SEARCHD_READ_CONFIG, configFile) << flush;
84 
85  try
86  {
87  stopSearchd();
88 
89  int count = sphinx_search_const::startStopSearchdLimit;
90  fObj.log(LoggerStream::Priority::PRIO_INFORMATION) << message(message_const::WAIT_START_SEARCHD) << flush;
91 
92  while (!isActiveSearchd() && --count>0)
93  {
94  std::stringstream outMsg, errMsg;
95  const std::string cmd("searchd");
96  std::vector<std::string> args = {"--config", configFile};
97  ProcExec procExec(cmd, args);
98  procExec.exec(outMsg, errMsg);
99  if (procExec.isError())
100  throw Poco::Exception(message(message_const::START_SEARCHD_FAIL, errMsg.str()));
101 
102  if ((outMsg.str().find(sphinx_admin_command_const::procExecErrorMsg)!=std::string::npos) ||
103  (outMsg.str().find(sphinx_admin_command_const::procExecFatalMsg)!=std::string::npos))
104  throw Poco::Exception(message(message_const::START_SEARCHD_FAIL, outMsg.str()));
105 
106  if ((errMsg.str().find(sphinx_admin_command_const::procExecErrorMsg)!=std::string::npos) ||
107  (errMsg.str().find(sphinx_admin_command_const::procExecFatalMsg)!=std::string::npos))
108  throw Poco::Exception(message(message_const::START_SEARCHD_FAIL, errMsg.str()));
109 
110  sleep(1);
111  fObj.log(LoggerStream::Priority::PRIO_INFORMATION) << ".";
112  fObj.log(LoggerStream::Priority::PRIO_INFORMATION).flush();
113  fObj.log(LoggerStream::Priority::PRIO_INFORMATION) << flush;
114  }
115  fObj.log(LoggerStream::Priority::PRIO_INFORMATION) << message(message_const::TRY_EASY_START) << flush;
116  if (!easyStart())
117  {
118  errorMsg.clear();
119  errorCode = NO_ERROR;
120  sleep(1);
121  if (!easyStart())
122  throw Poco::Exception(errorMsg);
123  }
124 
125  _isError = false;
126  _isStoppedSearchd = false;
127  }
128  catch(Poco::Exception& e)
129  {
130  _isError = true;
131  errorMsg = e.message();
132  errorCode = ERROR_START_SEARCHD;
133 
134  if (!easyStart())
135  {
136  errorMsg.clear();
137  errorCode = NO_ERROR;
138  sleep(1);
139  if (!easyStart())
140  errorMsg.append("\n").append(e.message());
141  }
142  fObj.log(LoggerStream::Priority::PRIO_INFORMATION) << errorMsg << flush;
143  }
144  return !_isError;
145 }
146 //-------------------------------------------------------------------
147 bool SphinxSearcher::stopSearchd(void)
148 {
149  _isError = false;
150  std::string configFile = fObj.getDataDir()+"/"+fObj.getIndexName()+"/"+sphinx_search_const::sphinxConfigName;
151  try
152  {
153  if (!close())
154  fObj.resetError();
155 
156  if (!isStoppedSearchd())
157  {
158  std::stringstream outMsg, errMsg;
159  const std::string cmd("searchd");
160  std::vector<std::string> args = {"--stop", "--config", configFile};
161  ProcExec procExec(cmd, args);
162  if (!procExec.exec(outMsg, errMsg))
163  throw Poco::Exception(message(message_const::STOP_SEARCHD, errMsg.str()));
164 
165  _isStoppedSearchd = true;
166  }
167  _isError = false;
168  }
169  catch(Poco::Exception& e)
170  {
171  _isError = true;
172  errorMsg = e.message();
173  errorCode = ERROR_STOP_SEARCHD;
174  }
175  return !_isError;
176 }
177 //-------------------------------------------------------------------
178 bool SphinxSearcher::isActiveSearchd()
179 {
180  bool result = false;
181  IndexStatusSearchd indexStatusSearchd(fObj);
182  indexStatusSearchd.setIndexName(fObj.getIndexName());
183  std::string outData;
184  if (indexStatusSearchd.execute())
185  {
186  outData = indexStatusSearchd.getResultData();
187  if (outData.find(sphinx_search_const::searchdGoodStatusFlag)!=std::string::npos)
188  result = true;
189  }
190  // indexStatus.isActiveSearchd(); // not used !!! method search PID-file on HDD
191  _isStoppedSearchd = !result;
192  return result;
193 }
194 //-------------------------------------------------------------------
195 void SphinxSearcher::setMaxResultsNumber(unsigned int maxResultsNumber) throw (Poco::Exception)
196 {
197  if (!client)
198  throw Poco::Exception(message(message_const::INVALID_CLIENT_HANDLE), ERROR_FUNCTIONAL_OBJECT);
199 
200  if (!sphinx_set_limits(client, 0, maxResultsNumber, maxResultsNumber, 0))
201  throw Poco::Exception(sphinx_error(client), ERROR_SET_LIMITS);
202 }
203 //-------------------------------------------------------------------
204 void SphinxSearcher::setLimits(unsigned int offset, unsigned int limit, unsigned int maxMatches, unsigned int cutoff) throw (Poco::Exception)
205 {
206  if (!client)
207  throw Poco::Exception(message(message_const::INVALID_CLIENT_HANDLE), ERROR_FUNCTIONAL_OBJECT);
208 
209  if (!sphinx_set_limits(client, offset, limit, maxMatches, cutoff))
210  throw Poco::Exception(sphinx_error(client), ERROR_SET_LIMITS);
211 }
212 //-------------------------------------------------------------------
213 void SphinxSearcher::applyFilter(Poco::SharedPtr<SphinxFilter> pFilter) throw (Poco::Exception)
214 {
215  if (!client)
216  throw Poco::Exception(message(message_const::INVALID_CLIENT_HANDLE), ERROR_FUNCTIONAL_OBJECT);
217 
218  if (pFilter.isNull())
219  throw Poco::Exception(message(message_const::INVALID_FILTER), ERROR_ADD_FILTER);
220 
221  bool exclude = (pFilter->getExcludeValue()==SphinxFilter::ExcludeType::etTrue);
222 
223  try
224  {
225  if (pFilter->getFilterType()==SphinxFilter::FilterType::ftSimple)
226  {
227  std::vector<long long> vFilters;
228  for (size_t i=0;i<pFilter->getAttributeValues().size();++i)
229  vFilters.push_back(std::stoll(pFilter->getAttributeValues()[i]));
230 
231  if (!addFilter(pFilter->getAttributeName(), vFilters, exclude))
232  throw Poco::Exception(message(message_const::ADD_FILTER, pFilter->getAttributeName())+message(message_const::ERROR, sphinx_error(client)), ERROR_ADD_FILTER);
233  }
234  else if(pFilter->getFilterType()==SphinxFilter::FilterType::ftRangeInteger)
235  {
236  if (pFilter->getAttributeValues().size() < 2)
237  throw Poco::Exception(message(message_const::INVALID_RANGE_OF_FILTER), ERROR_ADD_FILTER);
238 
239  if (!addFilterRange(pFilter->getAttributeName(), std::stoull(pFilter->getAttributeValues()[0]), std::stoull(pFilter->getAttributeValues()[1]), exclude))
240  throw Poco::Exception(message(message_const::ADD_FILTER, pFilter->getAttributeName())+message(message_const::ERROR, sphinx_error(client)), ERROR_ADD_FILTER);
241  }
242  else if(pFilter->getFilterType()==SphinxFilter::FilterType::ftRangeFloat)
243  {
244  if (pFilter->getAttributeValues().size() < 2)
245  throw Poco::Exception(message(message_const::INVALID_RANGE_OF_FILTER), ERROR_ADD_FILTER);
246 
247  if (!addFilterFloatRange(pFilter->getAttributeName(), std::stof(pFilter->getAttributeValues()[0]), std::stof(pFilter->getAttributeValues()[1]), exclude))
248  throw Poco::Exception(message(message_const::ADD_FILTER, pFilter->getAttributeName())+message(message_const::ERROR, sphinx_error(client)), ERROR_ADD_FILTER);
249  }
250  else if(pFilter->getFilterType()==SphinxFilter::FilterType::ftSimpleMulti)
251  {
252  for (size_t i=0;i<pFilter->getAttributeValues().size();++i)
253  {
254  if (!addFilter(pFilter->getAttributeName(), std::stoull(pFilter->getAttributeValues()[i]), exclude))
255  throw Poco::Exception(message(message_const::ADD_FILTER, pFilter->getAttributeName())+message(message_const::ERROR, sphinx_error(client)), ERROR_ADD_FILTER);
256  }
257  }
258  else
259  throw Poco::Exception(message(message_const::UNKNOWN_FILTER_TYPE, std::to_string(static_cast<unsigned int>(pFilter->getFilterType()))), ERROR_ADD_FILTER);
260  }
261  catch(std::exception& e)
262  {
263  throw Poco::Exception(e.what(), ERROR_ADD_FILTER);
264  }
265 }
266 //-------------------------------------------------------------------
267 bool SphinxSearcher::easyStart(int matchMode, int sortMode, int rankingMode, const std::string& rankexpr)
268 {
269  errorMsg.clear();
270  errorCode = NO_ERROR;
271  try
272  {
273  if (!client)
274  throw Poco::Exception(message(message_const::INVALID_CLIENT_HANDLE), ERROR_FUNCTIONAL_OBJECT);
275 
276  if (serverHost.empty())
277  throw Poco::Exception(message(message_const::SERVER_HOST_IS_EMPTY), ERROR_FUNCTIONAL_OBJECT);
278 
279  if (serverPort==0)
280  throw Poco::Exception(message(message_const::SERVER_PORT_IS_NULL), ERROR_FUNCTIONAL_OBJECT);
281 
282  if (!sphinx_set_server(client, serverHost.c_str(), serverPort))
283  throw Poco::Exception(sphinx_error(client), ERROR_SET_SERVER);
284 
285  fObj.log(LoggerStream::Priority::PRIO_INFORMATION) << message(message_const::SET_SERVER_SUCCESS)
286  << message(message_const::HOST, serverHost)
287  << message(message_const::PORT, serverPort) << flush;
288 
289  if (setMatchMode(matchMode))
290  {
291  if (setRankingMode(rankingMode, rankexpr))
292  {
293  if (setSortMode(sortMode))
294  {
295  if (!sphinx_open(client))
296  throw Poco::Exception(message(message_const::OPEN_CONNECTION_ERROR, sphinx_error(client)), ERROR_OPEN_CONNECTION);
297  else
298  _isConnected = true;
299  }
300  }
301  }
302 
303  fObj.log(LoggerStream::Priority::PRIO_INFORMATION) << message(message_const::EASY_START_SUCCESS)
304  << message(message_const::HOST, serverHost)
305  << message(message_const::PORT, serverPort) << flush;
306 
307  _isError = false;
308  }
309  catch(Poco::Exception& e)
310  {
311  _isError = true;
312  errorMsg = e.message();
313  errorCode = e.code();
314  fObj.log(LoggerStream::Priority::PRIO_INFORMATION) << message(message_const::EASY_START_FALLEN)
315  << message(message_const::HOST, serverHost)
316  << message(message_const::PORT, serverPort)
317  << message(message_const::ERROR, errorMsg) << flush;
318  }
319  return !_isError;
320 }
321 //-------------------------------------------------------------------
322 bool SphinxSearcher::open(void)
323 {
324  errorMsg.clear();
325  errorCode = NO_ERROR;
326  try
327  {
328  if (!client)
329  throw Poco::Exception(message(message_const::INVALID_CLIENT_HANDLE), ERROR_FUNCTIONAL_OBJECT);
330 
331  if (!sphinx_set_server(client, serverHost.c_str(), serverPort))
332  throw Poco::Exception(sphinx_error(client), ERROR_SET_SERVER);
333 
334  fObj.log(LoggerStream::Priority::PRIO_INFORMATION) << message(message_const::SET_SERVER_SUCCESS)
335  << message(message_const::HOST, serverHost)
336  << message(message_const::PORT, serverPort) << flush;
337 
338  if (!sphinx_open(client))
339  throw Poco::Exception(sphinx_error(client), ERROR_OPEN_CONNECTION);
340 
341  _isConnected = true;
342  _isError = false;
343  }
344  catch(Poco::Exception& e)
345  {
346  _isError = true;
347  errorMsg = e.displayText();
348  errorCode = e.code();
349  }
350  return !_isError;
351 }
352 //-------------------------------------------------------------------
353 bool SphinxSearcher::close(void)
354 {
355  errorMsg.clear();
356  errorCode = NO_ERROR;
357  try
358  {
359  if (!client)
360  throw Poco::Exception(message(message_const::INVALID_CLIENT_HANDLE), ERROR_FUNCTIONAL_OBJECT);
361 
362  if (!sphinx_close(client))
363  throw Poco::Exception(sphinx_error(client), ERROR_CLOSE_CONNECTION);
364 
365  _isConnected = false;
366  _isError = false;
367  }
368  catch(Poco::Exception& e)
369  {
370  _isError = true;
371  errorMsg = e.displayText();
372  errorCode = e.code();
373  }
374  return !_isError;
375 }
376 //-------------------------------------------------------------------
377 void SphinxSearcher::cleanup(char**& names, int*& weights, size_t& count)
378 {
379  if (names)
380  {
381  for (size_t i=0;i<count;++i)
382  if (names[i])
383  delete [] names[i];
384  delete [] names;
385  names = NULL;
386  }
387 
388  if (weights)
389  {
390  delete [] weights;
391  weights = NULL;
392  }
393  count = 0;
394  errorMsg.clear();
395 }
396 //-------------------------------------------------------------------
397 bool SphinxSearcher::setWeights(char**& names, int*& weights, size_t& count, std::vector <std::pair<std::string, int> >& Vec)
398 {
399  bool result = false;
400  if (client)
401  {
402  // free if need memory
403  cleanup(names, weights, count);
404  // allocate memory
405  count = Vec.size();
406  names = new char*[count];
407  weights = new int[count];
408  // copy data
409  for (size_t i=0;i<count;++i)
410  {
411  names[i] = new char[Vec[i].first.length()+1];
412  memset(names[i], 0, Vec[i].first.length()+1);
413  memcpy(names[i], Vec[i].first.c_str(), Vec[i].first.length());
414  weights[i] = Vec[i].second;
415  }
416  result = true;
417  }
418  return result;
419 }
420 //-------------------------------------------------------------------
421 bool SphinxSearcher::setFieldWeights(std::vector <std::pair<std::string, int> >& Vec)
422 {
423  bool result = false;
424 
425  if (setWeights(field_names, field_weights, fieldsCount, Vec))
426  result = sphinx_set_field_weights(client, fieldsCount, (const char**)field_names, field_weights);
427 
428  if (!result)
429  errorMsg = sphinx_error(client);
430 
431  return result;
432 }
433 //-------------------------------------------------------------------
434 bool SphinxSearcher::setIndexWeights(std::vector <std::pair<std::string, int> >& Vec)
435 {
436  bool result = false;
437 
438  if (setWeights(index_names, index_weights, indexesCount, Vec))
439  result = sphinx_set_index_weights(client, indexesCount, (const char**)index_names, index_weights);
440 
441  if (!result)
442  errorMsg = sphinx_error(client);
443 
444  if (Vec.size() && indexFileName.empty())
445  result = setIndexFileName(Vec.begin()->first);
446 
447  return result;
448 }
449 //-------------------------------------------------------------------
450 void SphinxSearcher::getWeights(char**& names, int*& weights, size_t& count, std::vector <std::pair<std::string, int> >& Vec)
451 {
452  Vec.clear();
453  for (size_t i=0;i<count;++i)
454  {
455  if (names[i]!=nullptr)
456  Vec.push_back(std::make_pair <std::string, int> (static_cast<std::string>(names[i]), static_cast<int>(weights[i])));
457  }
458 }
459 //-------------------------------------------------------------------
460 void SphinxSearcher::getFieldWeights(std::vector <std::pair<std::string, int> >& Vec)
461 {
462  getWeights(field_names, field_weights, fieldsCount, Vec);
463 }
464 //-------------------------------------------------------------------
465 void SphinxSearcher::getIndexWeights(std::vector <std::pair<std::string, int> >& Vec)
466 {
467  getWeights(index_names, index_weights, indexesCount, Vec);
468 }
469 //-------------------------------------------------------------------
470 std::vector <std::pair<std::string, int> >::iterator SphinxSearcher::find(std::vector <std::pair<std::string, int> >& Vec, const std::string& name)
471 {
472  std::vector <std::pair<std::string, int> >::iterator iter;
473  for (iter=Vec.begin();iter!=Vec.end();++iter)
474  {
475  if ((*iter).first == name)
476  break;
477  }
478  return iter;
479 }
480 //-------------------------------------------------------------------
481 // cppcheck-suppress unusedFunction
482 bool SphinxSearcher::isExistField(const std::string& name)
483 {
484  std::vector <std::pair<std::string, int> > Vec;
485  this->getFieldWeights(Vec);
486  return (this->find(Vec, name) != Vec.end());
487 }
488 //-------------------------------------------------------------------
489 // cppcheck-suppress unusedFunction
490 bool SphinxSearcher::isExistIndexFile(const std::string& name)
491 {
492  std::vector <std::pair<std::string, int> > Vec;
493  this->getIndexWeights(Vec);
494  return (this->find(Vec, name) != Vec.end());
495 }
496 //-------------------------------------------------------------------
497 // cppcheck-suppress unusedFunction
498 int SphinxSearcher::getFieldWeight(const std::string& name)
499 {
500  std::vector <std::pair<std::string, int> > Vec;
501  this->getFieldWeights(Vec);
502  std::vector <std::pair<std::string, int> >::iterator iter = this->find(Vec, name);
503  return (iter!=Vec.end())?(*iter).second:0;
504 }
505 //-------------------------------------------------------------------
506 // cppcheck-suppress unusedFunction
507 int SphinxSearcher::getIndexWeight(const std::string& name)
508 {
509  std::vector <std::pair<std::string, int> > Vec;
510  this->getIndexWeights(Vec);
511  std::vector <std::pair<std::string, int> >::iterator iter = this->find(Vec, name);
512  return (iter!=Vec.end())?(*iter).second:0;
513 }
514 //-------------------------------------------------------------------
515 // cppcheck-suppress unusedFunction
516 bool SphinxSearcher::addField(const std::string& name, int weight)
517 {
518  errorMsg.clear();
519  bool result = false;
520 
521  std::vector <std::pair<std::string, int> > Vec;
522  getFieldWeights(Vec);
523 
524  std::vector <std::pair<std::string, int> >::iterator iter = this->find(Vec, name);
525  if (iter==Vec.end())
526  {
527  Vec.push_back(std::make_pair (name, weight));
528  result = setFieldWeights(Vec);
529  }
530  else
531  errorMsg = message(message_const::FIELD_ALREADY_EXIST, name);
532 
533  return result;
534 }
535 //-------------------------------------------------------------------
536 // cppcheck-suppress unusedFunction
537 bool SphinxSearcher::setField(const std::string& name, int weight)
538 {
539  errorMsg.clear();
540  bool result = false;
541 
542  std::vector <std::pair<std::string, int> > Vec;
543  getFieldWeights(Vec);
544 
545  std::vector <std::pair<std::string, int> >::iterator iter = this->find(Vec, name);
546  if (iter!=Vec.end())
547  {
548  (*iter).second = weight;
549  result = setFieldWeights(Vec);
550  }
551  else
552  errorMsg = message(message_const::FIELD_NOT_FOUND, name);
553 
554  return result;
555 }
556 //-------------------------------------------------------------------
557 // cppcheck-suppress unusedFunction
558 bool SphinxSearcher::delField(const std::string& name)
559 {
560  errorMsg.clear();
561  bool result = false;
562 
563  std::vector <std::pair<std::string, int> > Vec;
564  getFieldWeights(Vec);
565 
566  std::vector <std::pair<std::string, int> >::iterator iter = this->find(Vec, name);
567  if (iter!=Vec.end())
568  {
569  Vec.erase(iter);
570  result = setFieldWeights(Vec);
571  }
572  else
573  errorMsg = message(message_const::FIELD_NOT_FOUND, name);
574 
575  return result;
576 }
577 //-------------------------------------------------------------------
578 // cppcheck-suppress unusedFunction
579 bool SphinxSearcher::addIndexFile(const std::string& name, int weight)
580 {
581  errorMsg.clear();
582  bool result = false;
583 
584  std::vector <std::pair<std::string, int> > Vec;
585  getIndexWeights(Vec);
586 
587  std::vector <std::pair<std::string, int> >::iterator iter = this->find(Vec, name);
588  if (iter==Vec.end())
589  {
590  Vec.push_back(std::make_pair (name, weight));
591  result = setIndexWeights(Vec);
592  }
593  else
594  errorMsg =message(message_const::INDEX_ALREADY_EXIST, name);
595 
596  return result;
597 }
598 //-------------------------------------------------------------------
599 // cppcheck-suppress unusedFunction
600 bool SphinxSearcher::setIndexFile(const std::string& name, int weight)
601 {
602  errorMsg.clear();
603  bool result = false;
604 
605  std::vector <std::pair<std::string, int> > Vec;
606  getIndexWeights(Vec);
607 
608  std::vector <std::pair<std::string, int> >::iterator iter = this->find(Vec, name);
609  if (iter!=Vec.end())
610  {
611  (*iter).second = weight;
612  result = setIndexWeights(Vec);
613  }
614  else
615  errorMsg = message(message_const::INDEX_NOT_FOUND, name);
616 
617  return result;
618 }
619 //-------------------------------------------------------------------
620 // cppcheck-suppress unusedFunction
621 bool SphinxSearcher::delIndexFile(const std::string& name)
622 {
623  errorMsg.clear();
624  bool result = false;
625 
626  std::vector <std::pair<std::string, int> > Vec;
627  getIndexWeights(Vec);
628 
629  std::vector <std::pair<std::string, int> >::iterator iter = this->find(Vec, name);
630  if (iter!=Vec.end())
631  {
632  Vec.erase(iter);
633  result = setIndexWeights(Vec);
634  }
635  else
636  errorMsg = message(message_const::INDEX_NOT_FOUND, name);
637 
638  return result;
639 }
640 //-------------------------------------------------------------------
641 bool SphinxSearcher::setMatchMode(int mode)
642 {
643  bool result = false;
644  try
645  {
646  if (!client)
647  throw Poco::Exception(message(message_const::INVALID_CLIENT_HANDLE), ERROR_FUNCTIONAL_OBJECT);
648 
649  if (!sphinx_set_match_mode(client, mode))
650  throw Poco::Exception(sphinx_error(client), ERROR_SET_MATCH_MODE);
651 
652  result = true;
653  }
654  catch(Poco::Exception& e)
655  {
656  errorMsg = e.displayText();
657  errorCode = e.code();
658  }
659  return result;
660 }
661 //-------------------------------------------------------------------
662 bool SphinxSearcher::setSortMode(int mode, const std::string& sortby)
663 {
664  bool result = false;
665  try
666  {
667  if (!client)
668  throw Poco::Exception(message(message_const::INVALID_CLIENT_HANDLE), ERROR_FUNCTIONAL_OBJECT);
669 
670  if (sortby.empty())
671  {
672  if (!sphinx_set_sort_mode(client, SPH_SORT_RELEVANCE, nullptr))
673  throw Poco::Exception(sphinx_error(client), ERROR_SET_SORT_MODE);
674  }
675  else
676  {
677  if (!sphinx_set_sort_mode(client, mode, sortby.c_str()))
678  throw Poco::Exception(sphinx_error(client), ERROR_SET_SORT_MODE);
679  }
680  result = true;
681  }
682  catch(Poco::Exception& e)
683  {
684  errorMsg = e.displayText();
685  errorCode = e.code();
686  }
687  return result;
688 }
689 //-------------------------------------------------------------------
690 bool SphinxSearcher::setRankingMode(int ranker, const std::string& rankexpr)
691 {
692  bool result = false;
693  try
694  {
695  if (!client)
696  throw Poco::Exception(message(message_const::INVALID_CLIENT_HANDLE), ERROR_FUNCTIONAL_OBJECT);
697 
698  if (rankexpr.empty())
699  {
700  if (!sphinx_set_ranking_mode(client, ranker, nullptr))
701  throw Poco::Exception(sphinx_error(client), ERROR_SET_RANKING_MODE);
702  }
703  else
704  {
705  if (!sphinx_set_ranking_mode(client, ranker, rankexpr.c_str()))
706  throw Poco::Exception(sphinx_error(client), ERROR_SET_RANKING_MODE);
707  }
708  result = true;
709  }
710  catch(Poco::Exception& e)
711  {
712  errorMsg = e.displayText();
713  errorCode = e.code();
714  }
715  return result;
716 }
717 //-------------------------------------------------------------------
718 void SphinxSearcher::setMaxQueryTime(unsigned long maxQueryTime) throw (Poco::Exception)
719 {
720  if (!client)
721  throw Poco::Exception(message(message_const::INVALID_CLIENT_HANDLE), ERROR_FUNCTIONAL_OBJECT);
722 
723  if (!sphinx_set_max_query_time(client, maxQueryTime))
724  throw Poco::Exception(sphinx_error(client), ERROR_SET_QUERY_TIME);
725 }
726 //-------------------------------------------------------------------
727 bool SphinxSearcher::setIndexFileName(const std::string& indexFileName_)
728 {
729  bool result = false;
730  for (size_t i=0;i<indexesCount;++i)
731  {
732  if (indexFileName_ == index_names[i])
733  {
734  indexFileName = indexFileName_;
735  result = true;
736  break;
737  }
738  }
739  if (!result)
740  {
741  if (indexesCount)
742  errorMsg = message(message_const::INDEX_NAME, indexFileName)+message(message_const::NOT_FOUND_IN_CONFIG);
743  else
744  errorMsg = message(message_const::LIST_OF_INDEXES_IS_EMPTY);
745  }
746  return result;
747 }
748 //-------------------------------------------------------------------
749 bool SphinxSearcher::searchDocument(const std::string& query, SphinxResultDataMemoryManager& resultMemoryManager, SphinxRequestInfo& request)
750 {
751  bool result = false;
752  if (client)
753  {
754  sphinx_result* res = 0;
755  int i, j, k, mva_len;
756  unsigned int* mva;
757  result = true;
758 
759  for (size_t z=0;z<indexesCount;++z)
760  {
761  if (!indexFileName.empty() && indexFileName!=index_names[z])
762  continue;
763 
764  const char* indexName = index_names[z];
765  res = sphinx_query(client, query.c_str(), indexName, nullptr);
766 
767  if (!res)
768  {
769  errorMsg = sphinx_error(client);
770  fObj.log(LoggerStream::Priority::PRIO_INFORMATION) << message(message_const::SEARCHD_ERROR, errorMsg) << flush;
771  result = false;
772  }
773  else
774  {
775  std::string warningMsg(sphinx_warning(client));
776  if (!warningMsg.empty())
777  {
778  fObj.log(LoggerStream::Priority::PRIO_INFORMATION) << message(message_const::SEARCHD_WARNING, warningMsg) << flush;
779  ++timeoutedRequests;
780  }
781 
782  request.setQuery(query);
783  request.setTotal(res->total);
784  request.setTotalFound(res->total_found);
785  request.setTimeMsec(res->time_msec);
786 
787  for (i=0; i<res->num_words; ++i)
788  request.addWordInfo(WordInfo(res->words[i].word, res->words[i].hits, res->words[i].docs));
789 
790  for (i=0; i<res->num_matches; ++i)
791  {
792  resultMemoryManager.resetMatchInfo();
793  const unsigned long long sphinxDocumentId = sphinx_get_id(res, i);
794  resultMemoryManager.setMatchInfoDocId(sphinxDocumentId);
795  const std::string sphinxWeight = std::to_string(sphinx_get_weight(res, i));
796  resultMemoryManager.setMatchInfoWeight(sphinxWeight);
797  for (j=0; j<res->num_attrs; ++j)
798  {
799  if (allowedFields.empty() || isAllowedFields(res->attr_names[j]))
800  {
801  std::stringstream valueStr;
802 
803  switch(res->attr_types[j])
804  {
805  case SPH_ATTR_MULTI64:
806  case SPH_ATTR_MULTI:
807  mva = sphinx_get_mva(res, i, j);
808  mva_len = *mva++;
809  for (k=0; k<mva_len; ++k)
810  valueStr << (res->attr_types[j]==SPH_ATTR_MULTI ? mva[k] : (unsigned int)sphinx_get_mva64_value(mva, k));
811  break;
812  case SPH_ATTR_FLOAT: valueStr << sphinx_get_float(res, i, j);
813  break;
814  case SPH_ATTR_STRING: valueStr << sphinx_get_string(res, i, j);
815  break;
816  default: valueStr << (sphinx_int64_t)sphinx_get_int(res, i, j);
817  break;
818  }
819  if (!resultMemoryManager.addMatchInfoAttr(AttrInfo(res->attr_names[j], valueStr.str())))
820  break;
821  }
822  }
823  resultMemoryManager.addMatchInfoAttr(AttrInfo(search_result_json_const::sphinxWeight, sphinxWeight)); // save weight as attribute
824  resultMemoryManager.addMatchInfoAttr(AttrInfo(search_result_json_const::sphinxDocumentId, std::to_string(sphinxDocumentId))); // save document ID as attribute
825  if (!resultMemoryManager.applyMatchInfo())
826  break;
827  }
828  }
829  }
830  }
831  if (!result)
832  {
833  bool activeSearchd = isActiveSearchd();
834  if (!activeSearchd)
835  {
836  IndexStatusSearchd indexStatusSearchd(fObj);
837  indexStatusSearchd.setIndexName(fObj.getIndexName());
838  bool existsPidFile = indexStatusSearchd.isExistPidFile();
839 
840  fObj.log(LoggerStream::Priority::PRIO_INFORMATION) << message(message_const::SEARCHD_STOPPED_PID_FILE_EXIST, ((existsPidFile)?"true":"false")) << flush;
841 
842  if (existsPidFile)
843  {
844  errorCode = ERROR_SEARCHD_STOPED_PID_FILE_EXISTS;
845  errorMsg = message(message_const::SEARCHD_STOPPED_BUT_PID_FILE_EXIST, errorMsg);
846  }
847  else
848  {
849  errorCode = ERROR_NOT_ACTIVITY_SEARCHD;
850  errorMsg = message(message_const::SEARCHD_NOT_ACTIVITY, errorMsg);
851  }
852  }
853  else
854  errorCode = ERROR_SEARCH_REQUEST;
855  }
856  _isError = !result;
857  return result;
858 }
859 //-------------------------------------------------------------------
860 bool SphinxSearcher::addFilter(const std::string& attrName, long long value, bool exclude)
861 {
862  return sphinx_add_filter(client, attrName.c_str(), 1, &value, exclude);
863 }
864 //-------------------------------------------------------------------
865 bool SphinxSearcher::addFilter(const std::string& attrName, std::vector<long long>& values, bool exclude)
866 {
867  bool result = false;
868  long long* pVal = new long long[values.size()];
869  if (pVal)
870  {
871  for (size_t i=0;i<values.size();++i)
872  pVal[i] = values[i];
873  result = sphinx_add_filter(client, attrName.c_str(), values.size(), pVal, exclude);
874  delete [] pVal;
875  }
876  return result;
877 }
878 //-------------------------------------------------------------------
879 bool SphinxSearcher::addFilterRange(const std::string& attrName, long long umin, long long umax, bool exclude)
880 {
881  return sphinx_add_filter_range(client, attrName.c_str(), umin, umax, exclude);
882 }
883 //-------------------------------------------------------------------
884 bool SphinxSearcher::addFilterFloatRange(const std::string& attrName, float fmin, float fmax, bool exclude)
885 {
886  return sphinx_add_filter_float_range(client, attrName.c_str(), fmin, fmax, exclude);
887 }
888 //-------------------------------------------------------------------
889 void SphinxSearcher::resetFilters(void)
890 {
891  sphinx_reset_filters(client);
892 }
893 //-------------------------------------------------------------------
894 std::string SphinxSearcher::Process(const std::string& json)
895 {
896  _isError = false;
897  errorMsg.clear();
898  errorCode = NO_ERROR;
899  std::string resultData;
900  timeval tvStart, tvStop;
901  gettimeofday(&tvStart, 0); //gettimeofday does not support TZ adjust on Linux.
902 
903  try
904  {
905  SphinxInputJsonMessage inputMessage(json);
906  if (inputMessage.isError())
907  throw Poco::Exception(inputMessage.getErrorMsg(), PARSE_ERROR);
908 
909  if (inputMessage.getType() == SphinxInputJsonMessage::MessageType::mtSearch)
910  {
911  resultData = makeSearchCommand(inputMessage.getData());
912  if (resultData.empty())
913  {
914  SphinxDefaultJSON defaultJson;
915  resultData = defaultJson.getJSON();
916  }
917  }
918  else
919  throw Poco::Exception(message(message_const::INCORRECT_JSON_TYPE), COMMAND_ERROR);
920  }
921  catch(Poco::Exception& e)
922  {
923  _isError = true;
924  errorCode = e.code();
925  errorMsg = e.displayText();
926 
927  SphinxDefaultJSON defaultJson;
928  resultData = defaultJson.getJSON();
929  }
930 
931  std::string resJson;
932  gettimeofday(&tvStop, 0);
933 
934  SphinxOutputJsonMessage outputMessage;
935  outputMessage.setErrorCode(errorCode);
936  outputMessage.setErrorMessage(errorMsg);
937  outputMessage.setData(resultData);
938  outputMessage.setTime(getTimeInterval(tvStart, tvStop)/1000);
939  bool success = outputMessage.serialize(resJson);
940  if (!_isError)
941  _isError = !success;
942 
943  return resJson;
944 }
945 //-------------------------------------------------------------------
946 std::string SphinxSearcher::makeSearchCommand(const std::string& json) throw (Poco::Exception, std::exception)
947 {
948  resultMemoryManager.reset();
949  requestInfo.clear();
950  resultData.clearRequestInfo();
951 
952  _isError = false;
953  errorMsg.clear();
954  errorCode = NO_ERROR;
955 
956  SphinxInputJsonMessageSearch messageSearch(json);
957  if (messageSearch.isError())
958  throw Poco::Exception(messageSearch.getErrorMsg(), ERROR_FUNCTIONAL_OBJECT);
959 
960  unsigned int offsetResults = 0;
961  unsigned int limitResults = 0;
962  unsigned int cutoffResults = 0;
963  unsigned int orderBy = 0;
964  unsigned int maxResultsNumber = input_json_message_const::defaultMaxResultsNumber;
965  unsigned long long timeoutValue = defaultRequestTimeout;
966 
967  setAllowedFields(messageSearch.getExternalFields()); // set list of external fields need get in search request
968  std::vector<std::string>& orderFieldsNames = messageSearch.getOrderFields(); // set list of external fields use in calculate weight
969  std::string weightAlgirithm(""); // type of calculate weight algorithm in string
970 
971  try
972  {
973  // read query parameters
974  for (size_t k=0;k<messageSearch.getQueryParameters().size();++k)
975  {
976  try
977  {
978  if (insensitiveCompare(messageSearch.getQueryParameters()[k].first, input_json_message_const::queryId))
979  requestInfo.setQueryId(std::stoul(messageSearch.getQueryParameters()[k].second, nullptr, 0));
980 
981  if (insensitiveCompare(messageSearch.getQueryParameters()[k].first, input_json_message_const::JsonType))
982  resultSerializator.setJsonType(std::stoul(messageSearch.getQueryParameters()[k].second, nullptr, 0));
983 
984  std::string sortFieldName;
985  if (insensitiveCompare(messageSearch.getQueryParameters()[k].first, input_json_message_const::sortFieldName))
986  {
987  sortFieldName = messageSearch.getQueryParameters()[k].second;
988  if (!sortFieldName.empty())
989  if (!isExistSchemaNames(sortFieldName))
990  throw Poco::Exception(message(message_const::FIELD_NOT_FOUND_IN_CURRENT_SCHEMA, sortFieldName), ERROR_SEARCH_REQUEST);
991  }
992 
993  if (insensitiveCompare(messageSearch.getQueryParameters()[k].first, input_json_message_const::sortOrderBy))
994  {
995  // set different sort modes
996  unsigned int sortOrderBy = std::stoul(messageSearch.getQueryParameters()[k].second);
997  if (!setSortMode(sortOrderBy, sortFieldName))
998  throw Poco::Exception(errorMsg, errorCode);
999  }
1000 
1001  if (insensitiveCompare(messageSearch.getQueryParameters()[k].first, input_json_message_const::offsetResultsNumber))
1002  offsetResults = std::stoul(messageSearch.getQueryParameters()[k].second);
1003 
1004  if (insensitiveCompare(messageSearch.getQueryParameters()[k].first, input_json_message_const::limitResultsNumber))
1005  limitResults = std::stoul(messageSearch.getQueryParameters()[k].second);
1006 
1007  if (insensitiveCompare(messageSearch.getQueryParameters()[k].first, input_json_message_const::cutoffResultNumber))
1008  cutoffResults = std::stoul(messageSearch.getQueryParameters()[k].second);
1009 
1010  if (insensitiveCompare(messageSearch.getQueryParameters()[k].first, input_json_message_const::maxResultsNumber))
1011  maxResultsNumber = std::stoul(messageSearch.getQueryParameters()[k].second);
1012 
1013  if (insensitiveCompare(messageSearch.getQueryParameters()[k].first, input_json_message_const::timeoutValueName))
1014  timeoutValue = std::stoul(messageSearch.getQueryParameters()[k].second);
1015  }
1016  catch(std::exception& e)
1017  {
1018  throw Poco::Exception("'"+messageSearch.getQueryParameters()[k].first+"' "+e.what(), ERROR_SEARCH_REQUEST);
1019  }
1020  }
1021 
1022  // read order parameters
1023  for (size_t k=0;k<messageSearch.getOrderParameters().size();++k)
1024  {
1025  try
1026  {
1027  if (insensitiveCompare(messageSearch.getOrderParameters()[k].first, input_json_message_const::weightAlgorithm))
1028  weightAlgirithm = messageSearch.getOrderParameters()[k].second;
1029 
1030  if (insensitiveCompare(messageSearch.getOrderParameters()[k].first, input_json_message_const::orderBy))
1031  orderBy = std::stoul(messageSearch.getOrderParameters()[k].second);
1032  }
1033  catch(std::exception& e)
1034  {
1035  throw Poco::Exception("'"+messageSearch.getOrderParameters()[k].first+"' "+e.what(), ERROR_SEARCH_REQUEST);
1036  }
1037  }
1038  }
1039  catch(Poco::Exception& e)
1040  {
1041  _isError = true;
1042  fObj.log(LoggerStream::Priority::PRIO_INFORMATION) << message(message_const::ERROR_MSG, e.message()) << message(message_const::ERROR_CODE, e.code()) << flush;
1043  throw Poco::Exception(message(message_const::GETTING_PARAMETERS_FAIL, e.message()), ERROR_SEARCH_REQUEST);
1044  }
1045 
1046  if (limitResults==0)
1047  limitResults = maxResultsNumber;
1048  // set different limits
1049  setLimits(offsetResults, limitResults, maxResultsNumber, cutoffResults);
1050 
1051  // cleanup filters
1052  resetFilters();
1053  // add filters
1054  for (size_t i=0;i<messageSearch.getFilters().getFilters().size();++i)
1055  applyFilter(messageSearch.getFilters().getFilters()[i]);
1056  // set max allowed query time
1057  setMaxQueryTime(timeoutValue);
1058 
1059  if (!searchDocument(messageSearch.getQueryString(), resultMemoryManager, requestInfo))
1060  throw Poco::Exception(getErrorMsg(), ERROR_SEARCH_REQUEST);
1061 
1062  requestInfo.setNodeName(fObj.getNodeName());
1063  requestInfo.setMaxResultsNumber(maxResultsNumber);
1064  requestInfo.setOrderBy(orderBy);
1065  resultData.addRequestInfo(std::forward<SphinxRequestInfo>(requestInfo));
1066 
1067  SphinxWeightCalculator calculator;
1068  CalculateStrategyFactory factory;
1069  calculator.setStrategy(factory.create(weightAlgirithm));
1070  calculator.calculate(orderFieldsNames, resultData);
1071  fObj.log(LoggerStream::Priority::PRIO_INFORMATION) << calculator.getLogString();
1072 
1073  std::string resultJson;
1074  if (!resultSerializator.serialize(resultJson, orderFieldsNames.size(), fObj.getMinNumberFieldsPacking()))
1075  throw Poco::Exception(resultSerializator.getErrorMsg(), ERROR_SEARCH_REQUEST);
1076 
1077  return resultJson;
1078 }
1079 //-------------------------------------------------------------------
1080 SphinxOutputJsonMessage SphinxSearcher::Process(SphinxInputJsonMessage& inputJsonMessage)
1081 {
1082  _isError = false;
1083  errorMsg.clear();
1084  errorCode = NO_ERROR;
1085  std::string resultData;
1086  timeval tvStart, tvStop;
1087  gettimeofday(&tvStart, 0); //gettimeofday does not support TZ adjust on Linux.
1088 
1089  try
1090  {
1091  if (inputJsonMessage.getType() != SphinxInputJsonMessage::MessageType::mtSearch)
1092  throw Poco::Exception(message(message_const::INCORRECT_JSON_TYPE), COMMAND_ERROR);
1093 
1094  resultData = makeSearchCommand(inputJsonMessage.getData());
1095  if (resultData.empty())
1096  {
1097  SphinxDefaultJSON defaultJson;
1098  resultData = defaultJson.getJSON();
1099  }
1100  }
1101  catch(Poco::Exception& e)
1102  {
1103  _isError = true;
1104  errorCode = e.code();
1105  errorMsg = e.displayText();
1106 
1107  SphinxDefaultJSON defaultJson;
1108  resultData = defaultJson.getJSON();
1109  }
1110  gettimeofday(&tvStop, 0);
1111 
1112  SphinxOutputJsonMessage outputMessage;
1113  outputMessage.setErrorCode(errorCode);
1114  outputMessage.setErrorMessage(errorMsg);
1115  outputMessage.setData(resultData);
1116  outputMessage.setTime(getTimeInterval(tvStart, tvStop)/1000);
1117 
1118  return outputMessage;
1119 }
1120 //-------------------------------------------------------------------
1121 //-------------------------------------------------------------------
1122 } // end namespace sphinx
1123 } // end namespace HCE
1124 
1125