HCE project C++ developers source code library  1.1.1
HCE project developer library
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros
SphinxSearcher.cpp
Go to the documentation of this file.
1 #include <string.h>
2 #include <stdio.h>
3 #include <sstream>
4 #include <iterator>
5 #include <time.h>
6 #include <sys/time.h>
7 #include <unistd.h>
8 #include <Poco/AutoPtr.h>
9 
10 #include "SphinxSearcher.hpp"
11 #include "SphinxInputJsonMessage.hpp"
12 #include "SphinxOutputJsonMessage.hpp"
13 #include "SphinxError.hpp"
14 #include "ProcExec.hpp"
15 #include "SphinxAdminCommand.hpp"
16 #include "SphinxConfigCreator.hpp"
17 #include "SphinxWeightCalculator.hpp"
18 #include "SphinxInputJsonMessageSearch.hpp"
19 #include "SphinxMessageConst.hpp"
20 
21 namespace HCE
22 {
23 namespace sphinx
24 {
25 //-------------------------------------------------------------------
26 SphinxSearcher::SphinxSearcher(SphinxFunctionalObject& fObj_, bool startSearchd_, bool stopSearchd_)
27 :SphinxSearchInterface(),
28  client(nullptr), field_names(nullptr), field_weights(nullptr), fieldsCount(0), index_names(nullptr), index_weights(nullptr),
29  indexesCount(0), fObj(fObj_), message(fObj.getCustomMessage()),
30  serverHost(sphinx_search_const::defaultHost), serverPort(sphinx_search_const::defaultPort), indexFileName(""),
31  _StopSearchd(stopSearchd_), _isConnected(false), timeoutedRequests(0), defaultRequestTimeout(input_json_message_const::defaultTimeoutValue),
32  allowedFields(), fullSchemaNames(), requestInfo(), resultData(), resultSerializator(resultData), resultMemoryManager(resultData)
33 {
34  client = sphinx_create(SPH_TRUE);
35  if (!client)
36  errorMsg = message(message_const::FAILED_TO_CREATE_CLIENT);
37 }
38 //-------------------------------------------------------------------
39 SphinxSearcher::~SphinxSearcher(void)
40 {
41  if (_StopSearchd)
42  stopSearchd();
43  sphinx_close(client);
44  sphinx_destroy(client);
45  cleanup(field_names, field_weights, fieldsCount);
46  cleanup(index_names, index_weights, indexesCount);
47 }
48 //-------------------------------------------------------------------
49 void SphinxSearcher::resizeResultData(size_t matchesCount, size_t attributesCount)
50 {
51  resultMemoryManager.resize(matchesCount, attributesCount);
52 }
53 //-------------------------------------------------------------------
54 void SphinxSearcher::setAllowedFields(std::vector<std::string>& queryExtendedFields)
55 {
56  allowedFields.clear();
57  for (size_t i=0;i<queryExtendedFields.size();++i)
58  allowedFields.insert(queryExtendedFields[i]);
59 }
60 //-------------------------------------------------------------------
61 bool SphinxSearcher::isAllowedFields(const std::string& extendedFieldName)
62 {
63  return (allowedFields.find(extendedFieldName)!=allowedFields.end());
64 }
65 //-------------------------------------------------------------------
66 void SphinxSearcher::setFullSchemaNames(std::vector<std::string>& schemaNames)
67 {
68  fullSchemaNames.clear();
69  for (size_t i=0;i<schemaNames.size();++i)
70  fullSchemaNames.insert(schemaNames[i]);
71 }
72 //-------------------------------------------------------------------
73 bool SphinxSearcher::isExistSchemaNames(const std::string& schemaName)
74 {
75  return (fullSchemaNames.find(schemaName)!=fullSchemaNames.end());
76 }
77 //-------------------------------------------------------------------
78 bool SphinxSearcher::startSearchd(void)
79 {
80  _isError = false;
81  std::string configFile = fObj.getDataDir()+"/"+fObj.getIndexName()+"/"+sphinx_search_const::sphinxConfigName;
82 
83  fObj.log() << message(message_const::START_SEARCHD_READ_CONFIG, configFile) << std::endl;
84 
85  try
86  {
87  stopSearchd();
88 
89  int count = sphinx_search_const::startStopSearchdLimit;
90  fObj.log() << message(message_const::WAIT_START_SEARCHD) << std::endl;
91 
92  while (!isActiveSearchd() && --count>0)
93  {
94  std::stringstream outMsg, errMsg;
95  const std::string cmd("searchd");
96  std::vector<std::string> args = {"--config", configFile};
97  ProcExec procExec(cmd, args);
98  procExec.exec(outMsg, errMsg);
99  if (procExec.isError())
100  throw Poco::Exception(message(message_const::START_SEARCHD_FAIL, errMsg.str()));
101 
102  if ((outMsg.str().find(sphinx_admin_command_const::procExecErrorMsg)!=std::string::npos) ||
103  (outMsg.str().find(sphinx_admin_command_const::procExecFatalMsg)!=std::string::npos))
104  throw Poco::Exception(message(message_const::START_SEARCHD_FAIL, outMsg.str()));
105 
106  if ((errMsg.str().find(sphinx_admin_command_const::procExecErrorMsg)!=std::string::npos) ||
107  (errMsg.str().find(sphinx_admin_command_const::procExecFatalMsg)!=std::string::npos))
108  throw Poco::Exception(message(message_const::START_SEARCHD_FAIL, errMsg.str()));
109 
110  sleep(1);
111  fObj.log() << ".";
112  fObj.log().flush();
113  fObj.log() << std::endl;
114  }
115  fObj.log() << message(message_const::TRY_EASY_START) << std::endl;
116  if (!easyStart())
117  {
118  errorMsg.clear();
119  errorCode = NO_ERROR;
120  sleep(1);
121  if (!easyStart())
122  throw Poco::Exception(errorMsg);
123  }
124 
125  _isError = false;
126  }
127  catch(Poco::Exception& e)
128  {
129  _isError = true;
130  errorMsg = e.message();
131  errorCode = ERROR_START_SEARCHD;
132 
133  if (!easyStart())
134  {
135  errorMsg.clear();
136  errorCode = NO_ERROR;
137  sleep(1);
138  if (!easyStart())
139  errorMsg.append("\n").append(e.message());
140  }
141  fObj.log() << errorMsg << std::endl;
142  }
143  return !_isError;
144 }
145 //-------------------------------------------------------------------
146 bool SphinxSearcher::stopSearchd(void)
147 {
148  _isError = false;
149  std::string configFile = fObj.getDataDir()+"/"+fObj.getIndexName()+"/"+sphinx_search_const::sphinxConfigName;
150  try
151  {
152  if (!close())
153  fObj.resetError();
154 
155  if (isActiveSearchd())
156  {
157  if (!_StopSearchd)
158  fObj.log() << message(message_const::WAIT_STOP_SEARCHD) << std::endl;
159 
160  std::stringstream outMsg, errMsg;
161  const std::string cmd("searchd");
162  std::vector<std::string> args = {"--stop", "--config", configFile};
163  ProcExec procExec(cmd, args);
164  if (!procExec.exec(outMsg, errMsg))
165  throw Poco::Exception(message(message_const::STOP_SEARCHD, errMsg.str()));
166 
167  int count=sphinx_search_const::startStopSearchdLimit;
168  while(isActiveSearchd() && --count>0)
169  {
170  sleep(1);
171  if (!_StopSearchd)
172  {
173  fObj.log() << ".";
174  fObj.log().flush();
175  }
176  }
177  if (!_StopSearchd)
178  fObj.log() << std::endl;
179  }
180  _isError = false;
181  }
182  catch(Poco::Exception& e)
183  {
184  _isError = true;
185  errorMsg = e.message();
186  errorCode = ERROR_STOP_SEARCHD;
187  }
188  return !_isError;
189 }
190 //-------------------------------------------------------------------
191 bool SphinxSearcher::isActiveSearchd()
192 {
193  bool result = false;
194  IndexStatusSearchd indexStatusSearchd(fObj);
195  indexStatusSearchd.setIndexName(fObj.getIndexName());
196  std::string outData;
197  if (indexStatusSearchd.execute())
198  {
199  outData = indexStatusSearchd.getResultData();
200  if (outData.find(sphinx_search_const::searchdGoodStatusFlag)!=std::string::npos)
201  result = true;
202  }
203  // indexStatus.isActiveSearchd(); // not used !!! method search PID-file on HDD
204 
205  return result;
206 }
207 //-------------------------------------------------------------------
208 void SphinxSearcher::setMaxResultsNumber(unsigned int maxResultsNumber) throw (Poco::Exception)
209 {
210  if (!client)
211  throw Poco::Exception(message(message_const::INVALID_CLIENT_HANDLE), ERROR_FUNCTIONAL_OBJECT);
212 
213  if (!sphinx_set_limits(client, 0, maxResultsNumber, maxResultsNumber, 0))
214  throw Poco::Exception(sphinx_error(client), ERROR_SET_LIMITS);
215 }
216 //-------------------------------------------------------------------
217 void SphinxSearcher::setLimits(unsigned int offset, unsigned int limit, unsigned int maxMatches, unsigned int cutoff) throw (Poco::Exception)
218 {
219  if (!client)
220  throw Poco::Exception(message(message_const::INVALID_CLIENT_HANDLE), ERROR_FUNCTIONAL_OBJECT);
221 
222  if (!sphinx_set_limits(client, offset, limit, maxMatches, cutoff))
223  throw Poco::Exception(sphinx_error(client), ERROR_SET_LIMITS);
224 }
225 //-------------------------------------------------------------------
226 void SphinxSearcher::applyFilter(Poco::SharedPtr<SphinxFilter> pFilter) throw (Poco::Exception)
227 {
228  if (!client)
229  throw Poco::Exception(message(message_const::INVALID_CLIENT_HANDLE), ERROR_FUNCTIONAL_OBJECT);
230 
231  if (pFilter.isNull())
232  throw Poco::Exception(message(message_const::INVALID_FILTER), ERROR_ADD_FILTER);
233 
234  bool exclude = (pFilter->getExcludeValue()==SphinxFilter::ExcludeType::etTrue);
235 
236  try
237  {
238  if (pFilter->getFilterType()==SphinxFilter::FilterType::ftSimple)
239  {
240  std::vector<long long> vFilters;
241  for (size_t i=0;i<pFilter->getAttributeValues().size();++i)
242  vFilters.push_back(std::stoll(pFilter->getAttributeValues()[i]));
243 
244  if (!addFilter(pFilter->getAttributeName(), vFilters, exclude))
245  throw Poco::Exception(message(message_const::ADD_FILTER, pFilter->getAttributeName())+message(message_const::ERROR, sphinx_error(client)), ERROR_ADD_FILTER);
246  }
247  else if(pFilter->getFilterType()==SphinxFilter::FilterType::ftRangeInteger)
248  {
249  if (pFilter->getAttributeValues().size() < 2)
250  throw Poco::Exception(message(message_const::INVALID_RANGE_OF_FILTER), ERROR_ADD_FILTER);
251 
252  if (!addFilterRange(pFilter->getAttributeName(), std::stoull(pFilter->getAttributeValues()[0]), std::stoull(pFilter->getAttributeValues()[1]), exclude))
253  throw Poco::Exception(message(message_const::ADD_FILTER, pFilter->getAttributeName())+message(message_const::ERROR, sphinx_error(client)), ERROR_ADD_FILTER);
254  }
255  else if(pFilter->getFilterType()==SphinxFilter::FilterType::ftRangeFloat)
256  {
257  if (pFilter->getAttributeValues().size() < 2)
258  throw Poco::Exception(message(message_const::INVALID_RANGE_OF_FILTER), ERROR_ADD_FILTER);
259 
260  if (!addFilterFloatRange(pFilter->getAttributeName(), std::stof(pFilter->getAttributeValues()[0]), std::stof(pFilter->getAttributeValues()[1]), exclude))
261  throw Poco::Exception(message(message_const::ADD_FILTER, pFilter->getAttributeName())+message(message_const::ERROR, sphinx_error(client)), ERROR_ADD_FILTER);
262  }
263  else if(pFilter->getFilterType()==SphinxFilter::FilterType::ftSimpleMulti)
264  {
265  for (size_t i=0;i<pFilter->getAttributeValues().size();++i)
266  {
267  if (!addFilter(pFilter->getAttributeName(), std::stoull(pFilter->getAttributeValues()[i]), exclude))
268  throw Poco::Exception(message(message_const::ADD_FILTER, pFilter->getAttributeName())+message(message_const::ERROR, sphinx_error(client)), ERROR_ADD_FILTER);
269  }
270  }
271  else
272  throw Poco::Exception(message(message_const::UNKNOWN_FILTER_TYPE, std::to_string(static_cast<unsigned int>(pFilter->getFilterType()))), ERROR_ADD_FILTER);
273  }
274  catch(std::exception& e)
275  {
276  throw Poco::Exception(e.what(), ERROR_ADD_FILTER);
277  }
278 }
279 //-------------------------------------------------------------------
280 bool SphinxSearcher::easyStart(int matchMode, int sortMode, int rankingMode, const std::string& rankexpr)
281 {
282  errorMsg.clear();
283  errorCode = NO_ERROR;
284  try
285  {
286  if (!client)
287  throw Poco::Exception(message(message_const::INVALID_CLIENT_HANDLE), ERROR_FUNCTIONAL_OBJECT);
288 
289  if (serverHost.empty())
290  throw Poco::Exception(message(message_const::SERVER_HOST_IS_EMPTY), ERROR_FUNCTIONAL_OBJECT);
291 
292  if (serverPort==0)
293  throw Poco::Exception(message(message_const::SERVER_PORT_IS_NULL), ERROR_FUNCTIONAL_OBJECT);
294 
295  if (!sphinx_set_server(client, serverHost.c_str(), serverPort))
296  throw Poco::Exception(sphinx_error(client), ERROR_SET_SERVER);
297 
298  fObj.log() << message(message_const::SET_SERVER_SUCCESS)
299  << message(message_const::HOST, serverHost)
300  << message(message_const::PORT, serverPort) << std::endl;
301 
302  if (setMatchMode(matchMode))
303  {
304  if (setRankingMode(rankingMode, rankexpr))
305  {
306  if (setSortMode(sortMode))
307  {
308  if (!sphinx_open(client))
309  throw Poco::Exception(message(message_const::OPEN_CONNECTION_ERROR, sphinx_error(client)), ERROR_OPEN_CONNECTION);
310  else
311  _isConnected = true;
312  }
313  }
314  }
315 
316  fObj.log() << message(message_const::EASY_START_SUCCESS)
317  << message(message_const::HOST, serverHost)
318  << message(message_const::PORT, serverPort) << std::endl;
319 
320  _isError = false;
321  }
322  catch(Poco::Exception& e)
323  {
324  _isError = true;
325  errorMsg = e.message();
326  errorCode = e.code();
327  fObj.log() << message(message_const::EASY_START_FALLEN)
328  << message(message_const::HOST, serverHost)
329  << message(message_const::PORT, serverPort)
330  << message(message_const::ERROR, errorMsg) << std::endl;
331  }
332  return !_isError;
333 }
334 //-------------------------------------------------------------------
335 bool SphinxSearcher::open(void)
336 {
337  errorMsg.clear();
338  errorCode = NO_ERROR;
339  try
340  {
341  if (!client)
342  throw Poco::Exception(message(message_const::INVALID_CLIENT_HANDLE), ERROR_FUNCTIONAL_OBJECT);
343 
344  if (!sphinx_set_server(client, serverHost.c_str(), serverPort))
345  throw Poco::Exception(sphinx_error(client), ERROR_SET_SERVER);
346 
347  fObj.log() << message(message_const::SET_SERVER_SUCCESS)
348  << message(message_const::HOST, serverHost)
349  << message(message_const::PORT, serverPort) << std::endl;
350 
351  if (!sphinx_open(client))
352  throw Poco::Exception(sphinx_error(client), ERROR_OPEN_CONNECTION);
353 
354  _isConnected = true;
355  _isError = false;
356  }
357  catch(Poco::Exception& e)
358  {
359  _isError = true;
360  errorMsg = e.displayText();
361  errorCode = e.code();
362  }
363  return !_isError;
364 }
365 //-------------------------------------------------------------------
366 bool SphinxSearcher::close(void)
367 {
368  errorMsg.clear();
369  errorCode = NO_ERROR;
370  try
371  {
372  if (!client)
373  throw Poco::Exception(message(message_const::INVALID_CLIENT_HANDLE), ERROR_FUNCTIONAL_OBJECT);
374 
375  if (!sphinx_close(client))
376  throw Poco::Exception(sphinx_error(client), ERROR_CLOSE_CONNECTION);
377 
378  _isConnected = false;
379  _isError = false;
380  }
381  catch(Poco::Exception& e)
382  {
383  _isError = true;
384  errorMsg = e.displayText();
385  errorCode = e.code();
386  }
387  return !_isError;
388 }
389 //-------------------------------------------------------------------
390 void SphinxSearcher::cleanup(char**& names, int*& weights, size_t& count)
391 {
392  if (names)
393  {
394  for (size_t i=0;i<count;++i)
395  if (names[i])
396  delete [] names[i];
397  delete [] names;
398  names = NULL;
399  }
400 
401  if (weights)
402  {
403  delete [] weights;
404  weights = NULL;
405  }
406  count = 0;
407  errorMsg.clear();
408 }
409 //-------------------------------------------------------------------
410 bool SphinxSearcher::setWeights(char**& names, int*& weights, size_t& count, std::vector <std::pair<std::string, int> >& Vec)
411 {
412  bool result = false;
413  if (client)
414  {
415  // free if need memory
416  cleanup(names, weights, count);
417  // allocate memory
418  count = Vec.size();
419  names = new char*[count];
420  weights = new int[count];
421  // copy data
422  for (size_t i=0;i<count;++i)
423  {
424  names[i] = new char[Vec[i].first.length()+1];
425  memset(names[i], 0, Vec[i].first.length()+1);
426  memcpy(names[i], Vec[i].first.c_str(), Vec[i].first.length());
427  weights[i] = Vec[i].second;
428  }
429  result = true;
430  }
431  return result;
432 }
433 //-------------------------------------------------------------------
434 bool SphinxSearcher::setFieldWeights(std::vector <std::pair<std::string, int> >& Vec)
435 {
436  bool result = false;
437 
438  if (setWeights(field_names, field_weights, fieldsCount, Vec))
439  result = sphinx_set_field_weights(client, fieldsCount, (const char**)field_names, field_weights);
440 
441  if (!result)
442  errorMsg = sphinx_error(client);
443 
444  return result;
445 }
446 //-------------------------------------------------------------------
447 bool SphinxSearcher::setIndexWeights(std::vector <std::pair<std::string, int> >& Vec)
448 {
449  bool result = false;
450 
451  if (setWeights(index_names, index_weights, indexesCount, Vec))
452  result = sphinx_set_index_weights(client, indexesCount, (const char**)index_names, index_weights);
453 
454  if (!result)
455  errorMsg = sphinx_error(client);
456 
457  if (Vec.size() && indexFileName.empty())
458  result = setIndexFileName(Vec.begin()->first);
459 
460  return result;
461 }
462 //-------------------------------------------------------------------
463 void SphinxSearcher::getWeights(char**& names, int*& weights, size_t& count, std::vector <std::pair<std::string, int> >& Vec)
464 {
465  Vec.clear();
466  for (size_t i=0;i<count;++i)
467  {
468  if (names[i]!=nullptr)
469  Vec.push_back(std::make_pair <std::string, int> (static_cast<std::string>(names[i]), static_cast<int>(weights[i])));
470  }
471 }
472 //-------------------------------------------------------------------
473 void SphinxSearcher::getFieldWeights(std::vector <std::pair<std::string, int> >& Vec)
474 {
475  getWeights(field_names, field_weights, fieldsCount, Vec);
476 }
477 //-------------------------------------------------------------------
478 void SphinxSearcher::getIndexWeights(std::vector <std::pair<std::string, int> >& Vec)
479 {
480  getWeights(index_names, index_weights, indexesCount, Vec);
481 }
482 //-------------------------------------------------------------------
483 std::vector <std::pair<std::string, int> >::iterator SphinxSearcher::find(std::vector <std::pair<std::string, int> >& Vec, const std::string& name)
484 {
485  std::vector <std::pair<std::string, int> >::iterator iter;
486  for (iter=Vec.begin();iter!=Vec.end();++iter)
487  {
488  if ((*iter).first == name)
489  break;
490  }
491  return iter;
492 }
493 //-------------------------------------------------------------------
494 // cppcheck-suppress unusedFunction
495 bool SphinxSearcher::isExistField(const std::string& name)
496 {
497  std::vector <std::pair<std::string, int> > Vec;
498  this->getFieldWeights(Vec);
499  return (this->find(Vec, name) != Vec.end());
500 }
501 //-------------------------------------------------------------------
502 // cppcheck-suppress unusedFunction
503 bool SphinxSearcher::isExistIndexFile(const std::string& name)
504 {
505  std::vector <std::pair<std::string, int> > Vec;
506  this->getIndexWeights(Vec);
507  return (this->find(Vec, name) != Vec.end());
508 }
509 //-------------------------------------------------------------------
510 // cppcheck-suppress unusedFunction
511 int SphinxSearcher::getFieldWeight(const std::string& name)
512 {
513  std::vector <std::pair<std::string, int> > Vec;
514  this->getFieldWeights(Vec);
515  std::vector <std::pair<std::string, int> >::iterator iter = this->find(Vec, name);
516  return (iter!=Vec.end())?(*iter).second:0;
517 }
518 //-------------------------------------------------------------------
519 // cppcheck-suppress unusedFunction
520 int SphinxSearcher::getIndexWeight(const std::string& name)
521 {
522  std::vector <std::pair<std::string, int> > Vec;
523  this->getIndexWeights(Vec);
524  std::vector <std::pair<std::string, int> >::iterator iter = this->find(Vec, name);
525  return (iter!=Vec.end())?(*iter).second:0;
526 }
527 //-------------------------------------------------------------------
528 // cppcheck-suppress unusedFunction
529 bool SphinxSearcher::addField(const std::string& name, int weight)
530 {
531  errorMsg.clear();
532  bool result = false;
533 
534  std::vector <std::pair<std::string, int> > Vec;
535  getFieldWeights(Vec);
536 
537  std::vector <std::pair<std::string, int> >::iterator iter = this->find(Vec, name);
538  if (iter==Vec.end())
539  {
540  Vec.push_back(std::make_pair (name, weight));
541  result = setFieldWeights(Vec);
542  }
543  else
544  errorMsg = message(message_const::FIELD_ALREADY_EXIST, name);
545 
546  return result;
547 }
548 //-------------------------------------------------------------------
549 // cppcheck-suppress unusedFunction
550 bool SphinxSearcher::setField(const std::string& name, int weight)
551 {
552  errorMsg.clear();
553  bool result = false;
554 
555  std::vector <std::pair<std::string, int> > Vec;
556  getFieldWeights(Vec);
557 
558  std::vector <std::pair<std::string, int> >::iterator iter = this->find(Vec, name);
559  if (iter!=Vec.end())
560  {
561  (*iter).second = weight;
562  result = setFieldWeights(Vec);
563  }
564  else
565  errorMsg = message(message_const::FIELD_NOT_FOUND, name);
566 
567  return result;
568 }
569 //-------------------------------------------------------------------
570 // cppcheck-suppress unusedFunction
571 bool SphinxSearcher::delField(const std::string& name)
572 {
573  errorMsg.clear();
574  bool result = false;
575 
576  std::vector <std::pair<std::string, int> > Vec;
577  getFieldWeights(Vec);
578 
579  std::vector <std::pair<std::string, int> >::iterator iter = this->find(Vec, name);
580  if (iter!=Vec.end())
581  {
582  Vec.erase(iter);
583  result = setFieldWeights(Vec);
584  }
585  else
586  errorMsg = message(message_const::FIELD_NOT_FOUND, name);
587 
588  return result;
589 }
590 //-------------------------------------------------------------------
591 // cppcheck-suppress unusedFunction
592 bool SphinxSearcher::addIndexFile(const std::string& name, int weight)
593 {
594  errorMsg.clear();
595  bool result = false;
596 
597  std::vector <std::pair<std::string, int> > Vec;
598  getIndexWeights(Vec);
599 
600  std::vector <std::pair<std::string, int> >::iterator iter = this->find(Vec, name);
601  if (iter==Vec.end())
602  {
603  Vec.push_back(std::make_pair (name, weight));
604  result = setIndexWeights(Vec);
605  }
606  else
607  errorMsg =message(message_const::INDEX_ALREADY_EXIST, name);
608 
609  return result;
610 }
611 //-------------------------------------------------------------------
612 // cppcheck-suppress unusedFunction
613 bool SphinxSearcher::setIndexFile(const std::string& name, int weight)
614 {
615  errorMsg.clear();
616  bool result = false;
617 
618  std::vector <std::pair<std::string, int> > Vec;
619  getIndexWeights(Vec);
620 
621  std::vector <std::pair<std::string, int> >::iterator iter = this->find(Vec, name);
622  if (iter!=Vec.end())
623  {
624  (*iter).second = weight;
625  result = setIndexWeights(Vec);
626  }
627  else
628  errorMsg = message(message_const::INDEX_NOT_FOUND, name);
629 
630  return result;
631 }
632 //-------------------------------------------------------------------
633 // cppcheck-suppress unusedFunction
634 bool SphinxSearcher::delIndexFile(const std::string& name)
635 {
636  errorMsg.clear();
637  bool result = false;
638 
639  std::vector <std::pair<std::string, int> > Vec;
640  getIndexWeights(Vec);
641 
642  std::vector <std::pair<std::string, int> >::iterator iter = this->find(Vec, name);
643  if (iter!=Vec.end())
644  {
645  Vec.erase(iter);
646  result = setIndexWeights(Vec);
647  }
648  else
649  errorMsg = message(message_const::INDEX_NOT_FOUND, name);
650 
651  return result;
652 }
653 //-------------------------------------------------------------------
654 bool SphinxSearcher::setMatchMode(int mode)
655 {
656  bool result = false;
657  try
658  {
659  if (!client)
660  throw Poco::Exception(message(message_const::INVALID_CLIENT_HANDLE), ERROR_FUNCTIONAL_OBJECT);
661 
662  if (!sphinx_set_match_mode(client, mode))
663  throw Poco::Exception(sphinx_error(client), ERROR_SET_MATCH_MODE);
664 
665  result = true;
666  }
667  catch(Poco::Exception& e)
668  {
669  errorMsg = e.displayText();
670  errorCode = e.code();
671  }
672  return result;
673 }
674 //-------------------------------------------------------------------
675 bool SphinxSearcher::setSortMode(int mode, const std::string& sortby)
676 {
677  bool result = false;
678  try
679  {
680  if (!client)
681  throw Poco::Exception(message(message_const::INVALID_CLIENT_HANDLE), ERROR_FUNCTIONAL_OBJECT);
682 
683  if (sortby.empty())
684  {
685  if (!sphinx_set_sort_mode(client, SPH_SORT_RELEVANCE, nullptr))
686  throw Poco::Exception(sphinx_error(client), ERROR_SET_SORT_MODE);
687  }
688  else
689  {
690  if (!sphinx_set_sort_mode(client, mode, sortby.c_str()))
691  throw Poco::Exception(sphinx_error(client), ERROR_SET_SORT_MODE);
692  }
693  result = true;
694  }
695  catch(Poco::Exception& e)
696  {
697  errorMsg = e.displayText();
698  errorCode = e.code();
699  }
700  return result;
701 }
702 //-------------------------------------------------------------------
703 bool SphinxSearcher::setRankingMode(int ranker, const std::string& rankexpr)
704 {
705  bool result = false;
706  try
707  {
708  if (!client)
709  throw Poco::Exception(message(message_const::INVALID_CLIENT_HANDLE), ERROR_FUNCTIONAL_OBJECT);
710 
711  if (rankexpr.empty())
712  {
713  if (!sphinx_set_ranking_mode(client, ranker, nullptr))
714  throw Poco::Exception(sphinx_error(client), ERROR_SET_RANKING_MODE);
715  }
716  else
717  {
718  if (!sphinx_set_ranking_mode(client, ranker, rankexpr.c_str()))
719  throw Poco::Exception(sphinx_error(client), ERROR_SET_RANKING_MODE);
720  }
721  result = true;
722  }
723  catch(Poco::Exception& e)
724  {
725  errorMsg = e.displayText();
726  errorCode = e.code();
727  }
728  return result;
729 }
730 //-------------------------------------------------------------------
731 void SphinxSearcher::setMaxQueryTime(unsigned long maxQueryTime) throw (Poco::Exception)
732 {
733  if (!client)
734  throw Poco::Exception(message(message_const::INVALID_CLIENT_HANDLE), ERROR_FUNCTIONAL_OBJECT);
735 
736  if (!sphinx_set_max_query_time(client, maxQueryTime))
737  throw Poco::Exception(sphinx_error(client), ERROR_SET_QUERY_TIME);
738 }
739 //-------------------------------------------------------------------
740 bool SphinxSearcher::setIndexFileName(const std::string& indexFileName_)
741 {
742  bool result = false;
743  for (size_t i=0;i<indexesCount;++i)
744  {
745  if (indexFileName_ == index_names[i])
746  {
747  indexFileName = indexFileName_;
748  result = true;
749  break;
750  }
751  }
752  if (!result)
753  {
754  if (indexesCount)
755  errorMsg = message(message_const::INDEX_NAME, indexFileName)+message(message_const::NOT_FOUND_IN_CONFIG);
756  else
757  errorMsg = message(message_const::LIST_OF_INDEXES_IS_EMPTY);
758  }
759  return result;
760 }
761 //-------------------------------------------------------------------
762 bool SphinxSearcher::searchDocument(const std::string& query, SphinxResultDataMemoryManager& resultMemoryManager, SphinxRequestInfo& request)
763 {
764  bool result = false;
765  if (client)
766  {
767  sphinx_result* res = 0;
768  int i, j, k, mva_len;
769  unsigned int* mva;
770  result = true;
771 
772  for (size_t z=0;z<indexesCount;++z)
773  {
774  if (!indexFileName.empty() && indexFileName!=index_names[z])
775  continue;
776 
777  const char* indexName = index_names[z];
778  res = sphinx_query(client, query.c_str(), indexName, nullptr);
779 
780  if (!res)
781  {
782  errorMsg = sphinx_error(client);
783  fObj.log() << message(message_const::SEARCHD_ERROR, errorMsg) << std::endl;
784  result = false;
785  }
786  else
787  {
788  std::string warningMsg(sphinx_warning(client));
789  if (!warningMsg.empty())
790  {
791  fObj.log() << message(message_const::SEARCHD_WARNING, warningMsg) << std::endl;
792  ++timeoutedRequests;
793  }
794 
795  request.setQuery(query);
796  request.setTotal(res->total);
797  request.setTotalFound(res->total_found);
798  request.setTimeMsec(res->time_msec);
799 
800  for (i=0; i<res->num_words; ++i)
801  request.addWordInfo(WordInfo(res->words[i].word, res->words[i].hits, res->words[i].docs));
802 
803  for (i=0; i<res->num_matches; ++i)
804  {
805  resultMemoryManager.resetMatchInfo();
806  const unsigned long long sphinxDocumentId = sphinx_get_id(res, i);
807  resultMemoryManager.setMatchInfoDocId(sphinxDocumentId);
808  const std::string sphinxWeight = std::to_string(sphinx_get_weight(res, i));
809  resultMemoryManager.setMatchInfoWeight(sphinxWeight);
810  for (j=0; j<res->num_attrs; ++j)
811  {
812  if (allowedFields.empty() || isAllowedFields(res->attr_names[j]))
813  {
814  std::stringstream valueStr;
815 
816  switch(res->attr_types[j])
817  {
818  case SPH_ATTR_MULTI64:
819  case SPH_ATTR_MULTI:
820  mva = sphinx_get_mva(res, i, j);
821  mva_len = *mva++;
822  for (k=0; k<mva_len; ++k)
823  valueStr << (res->attr_types[j]==SPH_ATTR_MULTI ? mva[k] : (unsigned int)sphinx_get_mva64_value(mva, k));
824  break;
825  case SPH_ATTR_FLOAT: valueStr << sphinx_get_float(res, i, j);
826  break;
827  case SPH_ATTR_STRING: valueStr << sphinx_get_string(res, i, j);
828  break;
829  default: valueStr << (sphinx_int64_t)sphinx_get_int(res, i, j);
830  break;
831  }
832  if (!resultMemoryManager.addMatchInfoAttr(AttrInfo(res->attr_names[j], valueStr.str())))
833  break;
834  }
835  }
836  resultMemoryManager.addMatchInfoAttr(AttrInfo(search_result_json_const::sphinxWeight, sphinxWeight)); // save weight as attribute
837  resultMemoryManager.addMatchInfoAttr(AttrInfo(search_result_json_const::sphinxDocumentId, std::to_string(sphinxDocumentId))); // save document ID as attribute
838  if (!resultMemoryManager.applyMatchInfo())
839  break;
840  }
841  }
842  }
843  }
844  if (!result)
845  {
846  bool activeSearchd = isActiveSearchd();
847  if (!activeSearchd)
848  {
849  IndexStatusSearchd indexStatusSearchd(fObj);
850  indexStatusSearchd.setIndexName(fObj.getIndexName());
851  bool existsPidFile = indexStatusSearchd.isExistPidFile();
852 
853  fObj.log() << message(message_const::SEARCHD_STOPPED_PID_FILE_EXIST, ((existsPidFile)?"true":"false")) << std::endl;
854 
855  if (existsPidFile)
856  {
857  errorCode = ERROR_SEARCHD_STOPED_PID_FILE_EXISTS;
858  errorMsg = message(message_const::SEARCHD_STOPPED_BUT_PID_FILE_EXIST, errorMsg);
859  }
860  else
861  {
862  errorCode = ERROR_NOT_ACTIVITY_SEARCHD;
863  errorMsg = message(message_const::SEARCHD_NOT_ACTIVITY, errorMsg);
864  }
865  }
866  else
867  errorCode = ERROR_SEARCH_REQUEST;
868  }
869  _isError = !result;
870  return result;
871 }
872 //-------------------------------------------------------------------
873 bool SphinxSearcher::addFilter(const std::string& attrName, long long value, bool exclude)
874 {
875  return sphinx_add_filter(client, attrName.c_str(), 1, &value, exclude);
876 }
877 //-------------------------------------------------------------------
878 bool SphinxSearcher::addFilter(const std::string& attrName, std::vector<long long>& values, bool exclude)
879 {
880  bool result = false;
881  long long* pVal = new long long[values.size()];
882  if (pVal)
883  {
884  for (size_t i=0;i<values.size();++i)
885  pVal[i] = values[i];
886  result = sphinx_add_filter(client, attrName.c_str(), values.size(), pVal, exclude);
887  delete [] pVal;
888  }
889  return result;
890 }
891 //-------------------------------------------------------------------
892 bool SphinxSearcher::addFilterRange(const std::string& attrName, long long umin, long long umax, bool exclude)
893 {
894  return sphinx_add_filter_range(client, attrName.c_str(), umin, umax, exclude);
895 }
896 //-------------------------------------------------------------------
897 bool SphinxSearcher::addFilterFloatRange(const std::string& attrName, float fmin, float fmax, bool exclude)
898 {
899  return sphinx_add_filter_float_range(client, attrName.c_str(), fmin, fmax, exclude);
900 }
901 //-------------------------------------------------------------------
902 void SphinxSearcher::resetFilters(void)
903 {
904  sphinx_reset_filters(client);
905 }
906 //-------------------------------------------------------------------
907 std::string SphinxSearcher::Process(const std::string& json)
908 {
909  _isError = false;
910  errorMsg.clear();
911  errorCode = NO_ERROR;
912  std::string resultData;
913  timeval tvStart, tvStop;
914  gettimeofday(&tvStart, 0); //gettimeofday does not support TZ adjust on Linux.
915 
916  try
917  {
918  SphinxInputJsonMessage inputMessage(json);
919  if (inputMessage.isError())
920  throw Poco::Exception(inputMessage.getErrorMsg(), PARSE_ERROR);
921 
922  if (inputMessage.getType() == SphinxInputJsonMessage::MessageType::mtSearch)
923  {
924  resultData = makeSearchCommand(inputMessage.getData());
925  if (resultData.empty())
926  {
927  SphinxDefaultJSON defaultJson;
928  resultData = defaultJson.getJSON();
929  }
930  }
931  else
932  throw Poco::Exception(message(message_const::INCORRECT_JSON_TYPE), COMMAND_ERROR);
933  }
934  catch(Poco::Exception& e)
935  {
936  _isError = true;
937  errorCode = e.code();
938  errorMsg = e.displayText();
939 
940  SphinxDefaultJSON defaultJson;
941  resultData = defaultJson.getJSON();
942  }
943 
944  std::string resJson;
945  gettimeofday(&tvStop, 0);
946 
947  SphinxOutputJsonMessage outputMessage;
948  outputMessage.setErrorCode(errorCode);
949  outputMessage.setErrorMessage(errorMsg);
950  outputMessage.setData(resultData);
951  outputMessage.setTime(getTimeInterval(tvStart, tvStop)/1000);
952  bool success = outputMessage.serialize(resJson);
953  if (!_isError)
954  _isError = !success;
955 
956  return resJson;
957 }
958 //-------------------------------------------------------------------
959 std::string SphinxSearcher::makeSearchCommand(const std::string& json) throw (Poco::Exception, std::exception)
960 {
961  resultMemoryManager.reset();
962  requestInfo.clear();
963  resultData.clearRequestInfo();
964 
965  _isError = false;
966  errorMsg.clear();
967  errorCode = NO_ERROR;
968 
969  SphinxInputJsonMessageSearch messageSearch(json);
970  if (messageSearch.isError())
971  throw Poco::Exception(messageSearch.getErrorMsg(), ERROR_FUNCTIONAL_OBJECT);
972 
973  unsigned int offsetResults = 0;
974  unsigned int limitResults = 0;
975  unsigned int cutoffResults = 0;
976  unsigned int orderBy = 0;
977  unsigned int maxResultsNumber = input_json_message_const::defaultMaxResultsNumber;
978  unsigned long long timeoutValue = defaultRequestTimeout;
979 
980  setAllowedFields(messageSearch.getExternalFields()); // set list of external fields need get in search request
981  std::vector<std::string>& orderFieldsNames = messageSearch.getOrderFields(); // set list of external fields use in calculate weight
982  std::string weightAlgirithm(""); // type of calculate weight algorithm in string
983 
984  try
985  {
986  // read query parameters
987  for (size_t k=0;k<messageSearch.getQueryParameters().size();++k)
988  {
989  try
990  {
991  if (insensitiveCompare(messageSearch.getQueryParameters()[k].first, input_json_message_const::queryId))
992  requestInfo.setQueryId(std::stoul(messageSearch.getQueryParameters()[k].second, nullptr, 0));
993 
994  if (insensitiveCompare(messageSearch.getQueryParameters()[k].first, input_json_message_const::JsonType))
995  resultSerializator.setJsonType(std::stoul(messageSearch.getQueryParameters()[k].second, nullptr, 0));
996 
997  std::string sortFieldName;
998  if (insensitiveCompare(messageSearch.getQueryParameters()[k].first, input_json_message_const::sortFieldName))
999  {
1000  sortFieldName = messageSearch.getQueryParameters()[k].second;
1001  if (!sortFieldName.empty())
1002  if (!isExistSchemaNames(sortFieldName))
1003  throw Poco::Exception(message(message_const::FIELD_NOT_FOUND_IN_CURRENT_SCHEMA, sortFieldName), ERROR_SEARCH_REQUEST);
1004  }
1005 
1006  if (insensitiveCompare(messageSearch.getQueryParameters()[k].first, input_json_message_const::sortOrderBy))
1007  {
1008  // set different sort modes
1009  unsigned int sortOrderBy = std::stoul(messageSearch.getQueryParameters()[k].second);
1010  if (!setSortMode(sortOrderBy, sortFieldName))
1011  throw Poco::Exception(errorMsg, errorCode);
1012  }
1013 
1014  if (insensitiveCompare(messageSearch.getQueryParameters()[k].first, input_json_message_const::offsetResultsNumber))
1015  offsetResults = std::stoul(messageSearch.getQueryParameters()[k].second);
1016 
1017  if (insensitiveCompare(messageSearch.getQueryParameters()[k].first, input_json_message_const::limitResultsNumber))
1018  limitResults = std::stoul(messageSearch.getQueryParameters()[k].second);
1019 
1020  if (insensitiveCompare(messageSearch.getQueryParameters()[k].first, input_json_message_const::cutoffResultNumber))
1021  cutoffResults = std::stoul(messageSearch.getQueryParameters()[k].second);
1022 
1023  if (insensitiveCompare(messageSearch.getQueryParameters()[k].first, input_json_message_const::maxResultsNumber))
1024  maxResultsNumber = std::stoul(messageSearch.getQueryParameters()[k].second);
1025 
1026  if (insensitiveCompare(messageSearch.getQueryParameters()[k].first, input_json_message_const::timeoutValueName))
1027  timeoutValue = std::stoul(messageSearch.getQueryParameters()[k].second);
1028  }
1029  catch(std::exception& e)
1030  {
1031  throw Poco::Exception("'"+messageSearch.getQueryParameters()[k].first+"' "+e.what(), ERROR_SEARCH_REQUEST);
1032  }
1033  }
1034 
1035  // read order parameters
1036  for (size_t k=0;k<messageSearch.getOrderParameters().size();++k)
1037  {
1038  try
1039  {
1040  if (insensitiveCompare(messageSearch.getOrderParameters()[k].first, input_json_message_const::weightAlgorithm))
1041  weightAlgirithm = messageSearch.getOrderParameters()[k].second;
1042 
1043  if (insensitiveCompare(messageSearch.getOrderParameters()[k].first, input_json_message_const::orderBy))
1044  orderBy = std::stoul(messageSearch.getOrderParameters()[k].second);
1045  }
1046  catch(std::exception& e)
1047  {
1048  throw Poco::Exception("'"+messageSearch.getOrderParameters()[k].first+"' "+e.what(), ERROR_SEARCH_REQUEST);
1049  }
1050  }
1051  }
1052  catch(Poco::Exception& e)
1053  {
1054  _isError = true;
1055  fObj.log() << message(message_const::ERROR_MSG, e.message()) << message(message_const::ERROR_CODE, e.code()) << std::endl;
1056  throw Poco::Exception(message(message_const::GETTING_PARAMETERS_FAIL, e.message()), ERROR_SEARCH_REQUEST);
1057  }
1058 
1059  if (limitResults==0)
1060  limitResults = maxResultsNumber;
1061  // set different limits
1062  setLimits(offsetResults, limitResults, maxResultsNumber, cutoffResults);
1063 
1064  // cleanup filters
1065  resetFilters();
1066  // add filters
1067  for (size_t i=0;i<messageSearch.getFilters().getFilters().size();++i)
1068  applyFilter(messageSearch.getFilters().getFilters()[i]);
1069  // set max allowed query time
1070  setMaxQueryTime(timeoutValue);
1071 
1072  if (!searchDocument(messageSearch.getQueryString(), resultMemoryManager, requestInfo))
1073  throw Poco::Exception(getErrorMsg(), ERROR_SEARCH_REQUEST);
1074 
1075  requestInfo.setNodeName(fObj.getNodeName());
1076  requestInfo.setMaxResultsNumber(maxResultsNumber);
1077  requestInfo.setOrderBy(orderBy);
1078  resultData.addRequestInfo(std::forward<SphinxRequestInfo>(requestInfo));
1079 
1080  SphinxWeightCalculator calculator;
1081  CalculateStrategyFactory factory;
1082  calculator.setStrategy(factory.create(weightAlgirithm));
1083  calculator.calculate(orderFieldsNames, resultData);
1084  fObj.log() << calculator.getLogString();
1085 
1086  std::string resultJson;
1087  if (!resultSerializator.serialize(resultJson, orderFieldsNames.size(), fObj.getMinNumberFieldsPacking()))
1088  throw Poco::Exception(resultSerializator.getErrorMsg(), ERROR_SEARCH_REQUEST);
1089 
1090  return resultJson;
1091 }
1092 //-------------------------------------------------------------------
1093 SphinxOutputJsonMessage SphinxSearcher::Process(SphinxInputJsonMessage& inputJsonMessage)
1094 {
1095  _isError = false;
1096  errorMsg.clear();
1097  errorCode = NO_ERROR;
1098  std::string resultData;
1099  timeval tvStart, tvStop;
1100  gettimeofday(&tvStart, 0); //gettimeofday does not support TZ adjust on Linux.
1101 
1102  try
1103  {
1104  if (inputJsonMessage.getType() != SphinxInputJsonMessage::MessageType::mtSearch)
1105  throw Poco::Exception(message(message_const::INCORRECT_JSON_TYPE), COMMAND_ERROR);
1106 
1107  resultData = makeSearchCommand(inputJsonMessage.getData());
1108  if (resultData.empty())
1109  {
1110  SphinxDefaultJSON defaultJson;
1111  resultData = defaultJson.getJSON();
1112  }
1113  }
1114  catch(Poco::Exception& e)
1115  {
1116  _isError = true;
1117  errorCode = e.code();
1118  errorMsg = e.displayText();
1119 
1120  SphinxDefaultJSON defaultJson;
1121  resultData = defaultJson.getJSON();
1122  }
1123  gettimeofday(&tvStop, 0);
1124 
1125  SphinxOutputJsonMessage outputMessage;
1126  outputMessage.setErrorCode(errorCode);
1127  outputMessage.setErrorMessage(errorMsg);
1128  outputMessage.setData(resultData);
1129  outputMessage.setTime(getTimeInterval(tvStart, tvStop)/1000);
1130 
1131  return outputMessage;
1132 }
1133 //-------------------------------------------------------------------
1134 //-------------------------------------------------------------------
1135 } // end namespace sphinx
1136 } // end namespace HCE
1137 
1138