hce-node application  1.4.3
HCE Hierarchical Cluster Engine node application
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros
SphinxSearchTest.cpp
Go to the documentation of this file.
1 #include <gtest/gtest.h>
2 #include <iomanip>
3 #include <set>
4 
5 #include "SphinxResultDataSerializator.hpp"
6 #include "SphinxSearchTest.hpp"
7 #include "SphinxSearcher.hpp"
8 #include "SphinxInputJsonMessage.hpp"
9 #include "SphinxOutputJsonMessage.hpp"
10 #include "SphinxError.hpp"
11 #include "SphinxFunctionalObject.hpp"
12 #include "SphinxResultDataSerializator.hpp"
13 #include "SphinxWeightCalculator.hpp"
14 #include "SphinxInputJsonMessageSearch.hpp"
15 #include "SphinxPrintStatus.hpp"
16 
17 namespace HCE
18 {
19 namespace sphinx
20 {
21 namespace tests
22 {
23 //---------------------------------------------------------------------------------------
24 SphinxPrintStatus printStartMessage("information", " - START");
25 //---------------------------------------------------------------------------------------
26 void SphinxSearchTest::testSerialization(void)
27 {
28  const std::string json = "{\"MI\":[{\"At\":[{\"uid\":\"2\"},{\"sid\":\"1\"},{\"lang\":\"255\"}],\"Id\":\"9659189393397617664\",\"W\":\"00000000000000000002000000000000000000ff00000000000000000001\"}],\"RI\":[{\"WI\":[{\"d\":5231,\"h\":28086,\"w\":\"appl\"}],\"f\":5231,\"max\":1,\"node\":\"test_node\",\"order\":2,\"q\":\"apple\",\"qid\":123,\"r\":1,\"time\":1}]}";
29 
30  SphinxResultData resultData, outResultData;
31  SphinxResultDataSerializator resultSerializator(resultData), outResultSerializator(outResultData);
32 
33  bool ret = resultSerializator.unserialize(json);
34  if (!ret)
35  std::cout << "ErrorCode: " << resultSerializator.getErrorCode() << "\tErrorMsg: " << resultSerializator.getErrorMsg() << std::endl;
36 
37  ASSERT_TRUE(ret);
38  ASSERT_TRUE(resultSerializator.getErrorCode() == NO_ERROR);
39  ASSERT_TRUE(resultSerializator.getErrorMsg().empty());
40  ASSERT_TRUE(resultSerializator.getJsonType() == (FILL_MATCH_INFO | FILL_REQUEST_INFO | FILL_ATTRIBUTES | FILL_WORD_INFO));
41 
42  printSuccess("Unserialize (FILL_MATCH_INFO | FILL_REQUEST_INFO | FILL_ATTRIBUTES | FILL_WORD_INFO)");
43 
44  std::string jsonString;
45  ret = resultSerializator.serialize((FILL_MATCH_INFO | FILL_REQUEST_INFO | FILL_ATTRIBUTES | FILL_WORD_INFO), jsonString);
46  if (!ret)
47  std::cout << "ErrorCode: " << resultSerializator.getErrorCode() << "\tErrorMsg: " << resultSerializator.getErrorMsg() << std::endl;
48 
49  ASSERT_TRUE(ret);
50  ASSERT_TRUE(resultSerializator.getErrorCode() == NO_ERROR);
51  ASSERT_TRUE(resultSerializator.getErrorMsg().empty());
52  std::cout << " json: " << json << std::endl;
53  std::cout << "jsonString: " << jsonString << std::endl;
54 
55 // assert(jsonString=="{\"MI\":[{\"At\":[{\"uid\":\"2\"},{\"sid\":\"1\"},{\"lang\":\"255\"}],\"Id\":\"9659189393397617664\",\"W\":\":2:ff:1\"}],\"RI\":[{\"WI\":[{\"d\":5231,\"h\":28086,\"w\":\"appl\"}],\"f\":5231,\"max\":1,\"node\":\"test_node\",\"order\":2,\"q\":\"apple\",\"qid\":123,\"r\":1,\"time\":1}]}");
56  printSuccess("Serialize (FILL_MATCH_INFO | FILL_REQUEST_INFO | FILL_ATTRIBUTES | FILL_WORD_INFO)");
57 
58  ret = resultSerializator.serialize((FILL_MATCH_INFO | FILL_REQUEST_INFO | FILL_ATTRIBUTES), jsonString);
59  if (!ret)
60  std::cout << "ErrorCode: " << resultSerializator.getErrorCode() << "\tErrorMsg: " << resultSerializator.getErrorMsg() << std::endl;
61 
62  ASSERT_TRUE(ret);
63  ASSERT_TRUE(resultSerializator.getErrorCode() == NO_ERROR);
64  ASSERT_TRUE(resultSerializator.getErrorMsg().empty());
65  printSuccess("Serialize (FILL_MATCH_INFO | FILL_REQUEST_INFO | FILL_ATTRIBUTES)");
66 
67  ret = outResultSerializator.unserialize(jsonString);
68  if (!ret)
69  std::cout << "ErrorCode: " << outResultSerializator.getErrorCode() << "\tErrorMsg: " << outResultSerializator.getErrorMsg() << std::endl;
70 
71  ASSERT_TRUE(ret);
72  ASSERT_TRUE(outResultSerializator.getErrorCode() == NO_ERROR);
73  ASSERT_TRUE(outResultSerializator.getErrorMsg().empty());
74 
75  const size_t requestCount = outResultData.getRequestInfoCount();
76  for (size_t i=0;i<requestCount;++i)
77  {
78  ASSERT_TRUE(outResultData.getRequestInfoItem(i).getWordsCount()==0);
79  }
80  printSuccess("Unserialize (FILL_MATCH_INFO | FILL_REQUEST_INFO | FILL_ATTRIBUTES)");
81 
82  ret = resultSerializator.serialize((FILL_MATCH_INFO | FILL_REQUEST_INFO), jsonString);
83  if (!ret)
84  std::cout << "ErrorCode: " << resultSerializator.getErrorCode() << "\tErrorMsg: " << resultSerializator.getErrorMsg() << std::endl;
85 
86  ASSERT_TRUE(ret);
87  ASSERT_TRUE(resultSerializator.getErrorCode() == NO_ERROR);
88  ASSERT_TRUE(resultSerializator.getErrorMsg().empty());
89  printSuccess("Serialize (FILL_MATCH_INFO | FILL_REQUEST_INFO)");
90 
91  ret = outResultSerializator.unserialize(jsonString);
92  if (!ret)
93  std::cout << "ErrorCode: " << outResultSerializator.getErrorCode() << "\tErrorMsg: " << outResultSerializator.getErrorMsg() << std::endl;
94 
95  ASSERT_TRUE(ret);
96  ASSERT_TRUE(outResultSerializator.getErrorCode() == NO_ERROR);
97  ASSERT_TRUE(outResultSerializator.getErrorMsg().empty());
98 
99  const size_t matchesCount = outResultData.getMatchInfoCount();
100  for (size_t i=0;i<matchesCount;++i)
101  {
102  ASSERT_TRUE(outResultData.getMatchInfoItem(i).getAttributesCount()==0);
103  }
104  printSuccess("Unserialize (FILL_MATCH_INFO | FILL_REQUEST_INFO)");
105 
106  ret = resultSerializator.serialize((FILL_MATCH_INFO), jsonString);
107  if (!ret)
108  std::cout << "ErrorCode: " << resultSerializator.getErrorCode() << "\tErrorMsg: " << resultSerializator.getErrorMsg() << std::endl;
109 
110  ASSERT_TRUE(ret);
111  ASSERT_TRUE(resultSerializator.getErrorCode() == NO_ERROR);
112  ASSERT_TRUE(resultSerializator.getErrorMsg().empty());
113  printSuccess("Serialize (FILL_MATCH_INFO)");
114 
115  ret = outResultSerializator.unserialize(jsonString);
116  if (!ret)
117  std::cout << "ErrorCode: " << outResultSerializator.getErrorCode() << "\tErrorMsg: " << outResultSerializator.getErrorMsg() << std::endl;
118 
119  ASSERT_TRUE(ret);
120  ASSERT_TRUE(outResultSerializator.getErrorCode() == NO_ERROR);
121  ASSERT_TRUE(outResultSerializator.getErrorMsg().empty());
122  ASSERT_TRUE(outResultData.getRequestInfoCount()==0);
123  printSuccess("Unserialize (FILL_MATCH_INFO)");
124 
125  ret = resultSerializator.serialize((NO_FILL), jsonString);
126  if (!ret)
127  std::cout << "ErrorCode: " << resultSerializator.getErrorCode() << "\tErrorMsg: " << resultSerializator.getErrorMsg() << std::endl;
128 
129  assert(ret);
130  assert(resultSerializator.getErrorCode() == NO_ERROR);
131  assert(resultSerializator.getErrorMsg().empty());
132  printSuccess("Serialize (NO_FILL)");
133 
134  ret = outResultSerializator.unserialize(jsonString);
135  if (!ret)
136  std::cout << "ErrorCode: " << outResultSerializator.getErrorCode() << "\tErrorMsg: " << outResultSerializator.getErrorMsg() << std::endl;
137 
138  ASSERT_TRUE(ret);
139  ASSERT_TRUE(outResultSerializator.getErrorCode() == NO_ERROR);
140  ASSERT_TRUE(outResultSerializator.getErrorMsg().empty());
141  ASSERT_TRUE(outResultData.getMatchInfoCount() == 0);
142  printSuccess("Unserialize (NO_FILL)");
143 
144  SphinxDefaultJSON defaultJson(1, 1);
145  if (defaultJson.isError())
146  std::cout << "ErrorCode: " << defaultJson.getErrorCode() << "\tErrorMsg: " << defaultJson.getErrorMsg() << std::endl;
147 
148  ASSERT_FALSE(defaultJson.isError());
149  ASSERT_TRUE(defaultJson.getErrorCode() == NO_ERROR);
150  ASSERT_TRUE(defaultJson.getErrorMsg().empty());
151  ASSERT_FALSE(defaultJson.getJSON().empty());
152  printSuccess("Generate default JSON");
153 
154  ret = defaultJson.makeJSON();
155  if (!ret)
156  std::cout << "ErrorCode: " << defaultJson.getErrorCode() << "\tErrorMsg: " << defaultJson.getErrorMsg() << std::endl;
157 
158  ASSERT_TRUE(ret);
159  ASSERT_TRUE(defaultJson.getErrorCode() == NO_ERROR);
160  ASSERT_TRUE(defaultJson.getErrorMsg().empty());
161  ASSERT_FALSE(defaultJson.getJSON().empty());
162 
163  printSuccess("Test SphinxResultDataSerializator");
164 }
165 void SphinxSearchTest::testSerializationExt(void)
166 {
167  const std::string query = "q";
168  const unsigned int totalRes = 10;
169  const unsigned int totalFound = 10;
170  const unsigned int time = 1;
171 
172 
173  Poco::SharedPtr<SphinxResultData> sphinxResultData;
174  WordInfo word1 = WordInfo("word1", 10, 3);
175  WordInfo word2 = WordInfo("word2", 1, 1);
176 
177  SphinxMatchInfo sphinxMatchInfo1;
178  sphinxMatchInfo1.setDocId(7);
179  sphinxMatchInfo1.setWeightFromDecString("111");
180 
181  SphinxMatchInfo sphinxMatchInfo2;
182  sphinxMatchInfo2.setDocId(9);
183  sphinxMatchInfo2.setWeightFromNumeric(222);
184 
185  AttrInfo pAttrInfo1("first", "value");
186 
187  sphinxMatchInfo1.addAttrInfo(pAttrInfo1);
188  sphinxMatchInfo2.addAttrInfo(pAttrInfo1);
189 
190  SphinxRequestInfo sphinxRequestInfo;
191  sphinxRequestInfo.setQuery(query);
192  sphinxRequestInfo.setTotal(totalRes);
193  sphinxRequestInfo.setTotalFound(totalFound);
194  sphinxRequestInfo.setTimeMsec(time);
195  sphinxRequestInfo.addWordInfo(word1);
196  sphinxRequestInfo.addWordInfo(word2);
197 
198  sphinxResultData.assign(new SphinxResultData());
199  sphinxResultData->addMatchInfo(sphinxMatchInfo1);
200  sphinxResultData->addMatchInfo(sphinxMatchInfo2);
201  sphinxResultData->addRequestInfo(sphinxRequestInfo);
202 
203 
204  const size_t matchInfoCount = sphinxResultData->getMatchInfoCount();
205  size_t attributesCount = 0;
206 
207  if (matchInfoCount)
208  attributesCount = sphinxResultData->getMatchInfoItem(0).getAttributesCount();
209 
210  SphinxResultDataSerializator resultSerializator(*sphinxResultData);
211 
212  resultSerializator.setJsonType(FILL_MATCH_INFO | FILL_REQUEST_INFO | FILL_ATTRIBUTES | FILL_WORD_INFO);
213 
214  std::string resultJson;
215  bool ret = resultSerializator.serialize(resultJson, attributesCount, attributesCount);
216  if (resultSerializator.isError())
217  std::cout << "Error: " << resultSerializator.getErrorMsg() << std::endl;
218  ASSERT_TRUE(ret);
219 
220  std::cout << "JSON: " << resultJson << std::endl;
221 
222  sphinxResultData->reset();
223  ret = resultSerializator.unserialize(resultJson);
224  if (resultSerializator.isError())
225  std::cout << "Error: " << resultSerializator.getErrorMsg() << std::endl;
226  ASSERT_TRUE(ret);
227 
228  const size_t macthInfoCount = sphinxResultData->getMatchInfoCount();
229  ASSERT_EQ(macthInfoCount, 2);
230  ASSERT_EQ(sphinxResultData->getMatchInfoItem(0).getWeight(), toHexString(111));
231  ASSERT_EQ(sphinxResultData->getMatchInfoItem(1).getWeight(), toHexString(222));
232  printSuccess("Test SerializationExt");
233 }
234 //---------------------------------------------------------------------------------------
235 void SphinxSearchTest::testMemoryManager(void)
236 {
237  SphinxResultData resultData;
238  SphinxResultDataMemoryManager resultMemoryManager(resultData);
239 
240  size_t matchesCount = 100;
241  size_t attributesCount = 10;
242 
243  resultMemoryManager.resize(matchesCount, attributesCount);
244  for (size_t i=0;i<(matchesCount+attributesCount);++i) // max limit oversized
245  {
246  resultMemoryManager.setMatchInfoDocId(i+1);
247  resultMemoryManager.setMatchInfoWeight(toHexString(i+1));
248  for (size_t j=0;j<matchesCount;++j) // max limit oversized
249  resultMemoryManager.addMatchInfoAttr(AttrInfo("at_"+std::to_string(i+1), "val_"+std::to_string(i+1)));
250  resultMemoryManager.applyMatchInfo();
251  }
252 
253  ASSERT_EQ(resultData.getMatchInfoCount(), matchesCount);
254  for (size_t i=0;i<matchesCount;++i)
255  assert(resultData.getMatchInfoItem(i).getAttributesCount() == attributesCount);
256  printSuccess("Allocated memory for 100 MatchInfo each have 10 attributes");
257 
258  matchesCount = 10;
259  attributesCount = 5;
260 
261  resultMemoryManager.resize(matchesCount, attributesCount);
262  for (size_t i=0;i<(matchesCount+attributesCount);++i) // max limit oversized
263  {
264  resultMemoryManager.setMatchInfoDocId(i+1);
265  resultMemoryManager.setMatchInfoWeight(toHexString(i+1));
266  for (size_t j=0;j<matchesCount;++j) // max limit oversized
267  resultMemoryManager.addMatchInfoAttr(AttrInfo("at_"+std::to_string(i+1), "val_"+std::to_string(i+1)));
268  resultMemoryManager.applyMatchInfo();
269  }
270 
271  ASSERT_EQ(resultData.getMatchInfoCount(), matchesCount);
272  for (size_t i=0;i<matchesCount;++i)
273  ASSERT_EQ(resultData.getMatchInfoItem(i).getAttributesCount(), attributesCount);
274  printSuccess("Reallocated memory for 10 MatchInfo each have 5 attributes");
275 
276  resultMemoryManager.reset();
277  ASSERT_EQ(resultData.getMatchInfoCount(), 0);
278  for (size_t i=0;i<matchesCount;++i)
279  ASSERT_EQ(resultData.getMatchInfoItem(i).getAttributesCount(), 0);
280  printSuccess("Reset allocated memory");
281 
282  for (size_t i=0;i<(matchesCount/2);++i) // half max limit
283  {
284  resultMemoryManager.setMatchInfoDocId(i+1);
285  resultMemoryManager.setMatchInfoWeight(toHexString(i+1));
286  for (size_t j=0;j<(attributesCount/2);++j) // half max limit
287  resultMemoryManager.addMatchInfoAttr(AttrInfo("at_"+std::to_string(i+1), "val_"+std::to_string(i+1)));
288  resultMemoryManager.applyMatchInfo();
289  }
290 
291  ASSERT_EQ(resultData.getMatchInfoCount(), matchesCount/2);
292  for (size_t i=0;i<(matchesCount/2);++i)
293  ASSERT_EQ(resultData.getMatchInfoItem(i).getAttributesCount(), attributesCount/2);
294  printSuccess("Filling half allocated memory");
295 
296  resultMemoryManager.resize(1, 0);
297  for (size_t i=0;i<matchesCount;++i) // max limit oversized
298  {
299  resultMemoryManager.setMatchInfoDocId(i+1);
300  resultMemoryManager.setMatchInfoWeight(toHexString(i+1));
301  for (size_t j=0;j<(attributesCount);++j) // max limit oversized
302  resultMemoryManager.addMatchInfoAttr(AttrInfo("at_"+std::to_string(i+1), "val_"+std::to_string(i+1)));
303  resultMemoryManager.applyMatchInfo();
304  }
305 
306  ASSERT_EQ(resultData.getMatchInfoCount(), 1);
307  ASSERT_EQ(resultData.getMatchInfoItem(0).getAttributesCount(), 0);
308  printSuccess("Resize allocated memory for 1 MatchInfo and without attributes");
309 
310  resultMemoryManager.resize(0, 0);
311  for (size_t i=0;i<matchesCount;++i) // max limit oversized
312  {
313  resultMemoryManager.setMatchInfoDocId(i+1);
314  resultMemoryManager.setMatchInfoWeight(toHexString(i+1));
315  for (size_t j=0;j<(attributesCount);++j) // max limit oversized
316  resultMemoryManager.addMatchInfoAttr(AttrInfo("at_"+std::to_string(i+1), "val_"+std::to_string(i+1)));
317  resultMemoryManager.applyMatchInfo();
318  }
319  ASSERT_EQ(resultData.getMatchInfoCount(), 0);
320  printSuccess("Resize allocated memory for 0 MatchInfo and without attributes");
321 
322  const size_t MATCHES_SZ = 2;
323  const size_t ATTRIBUTES_SZ = 3;
324  resultMemoryManager.resize(MATCHES_SZ, ATTRIBUTES_SZ);
325  unsigned long long docId[MATCHES_SZ] = {123, 456};
326  std::string weight[MATCHES_SZ] = {"00000000000000000001", "00000000000000000002"};
327  std::vector<std::pair<std::string, std::string> > attr[ATTRIBUTES_SZ] = {{
328  {"uid", "67"},
329  {"qid", "123"},
330  {"lang", "255"}
331  }, {
332  {"uid", "87"},
333  {"qid", "45"},
334  {"lang", "1"}
335  }};
336 
337  // add to SphinxResultData information about 2 matches
338  for (size_t i=0;i<MATCHES_SZ;++i)
339  {
340  resultMemoryManager.setMatchInfoDocId(docId[i]);
341  resultMemoryManager.setMatchInfoWeight(weight[i]);
342  for (size_t j=0;j<ATTRIBUTES_SZ;++j)
343  resultMemoryManager.addMatchInfoAttr(AttrInfo(attr[i][j].first, attr[i][j].second));
344  resultMemoryManager.applyMatchInfo();
345  }
346 
347  // check correct content of SphinxResultData
348  matchesCount = resultData.getMatchInfoCount();
349  for (size_t i=0;i<matchesCount;++i)
350  {
351  ASSERT_EQ(resultData.getMatchInfoItem(i).getDocId(), docId[i]);
352  ASSERT_EQ(resultData.getMatchInfoItem(i).getWeight(), weight[i]);
353  attributesCount = resultData.getMatchInfoItem(i).getAttributesCount();
354  for (size_t j=0;j<attributesCount;++j)
355  {
356  ASSERT_EQ(resultData.getMatchInfoItem(i).getAttrItem(j).name, attr[i][j].first);
357  ASSERT_EQ(resultData.getMatchInfoItem(i).getAttrItem(j).value, attr[i][j].second);
358  }
359  }
360  printSuccess("Test SphinxResultDataMemoryMananger");
361 }
362 //---------------------------------------------------------------------------------------
363 void SphinxSearchTest::makeSearchRequest(SphinxFunctionalObject& fObj, const std::string& query,
364  const std::vector<std::pair<std::string, std::string> >& queryParameters,
365  const std::vector<std::pair<std::string, std::string> >& orderParameters,
366  const std::vector<std::string>& externalFields,
367  const std::vector<std::string>& orderFields,
368  SphinxResultData& resultData)
369 {
370  resultData.clear();
371  SphinxResultDataSerializator resultSerializator(resultData);
372 
373  std::string json;
374  SphinxInputJsonMessageSearch msgSearch;
375  // prepare before request
376  msgSearch.setQueryString(query);
377  for (size_t i=0;i<queryParameters.size();++i)
378  msgSearch.addQueryParameters(queryParameters[i].first, queryParameters[i].second);
379 
380  for (size_t i=0;i<orderParameters.size();++i)
381  msgSearch.addOrderParameters(orderParameters[i].first, orderParameters[i].second);
382 
383  for (size_t i=0;i<externalFields.size();++i)
384  msgSearch.addExternalFields(externalFields[i]);
385 
386  for (size_t i=0;i<orderFields.size();++i)
387  msgSearch.addOrderFields(orderFields[i]);
388 
389  ASSERT_TRUE(msgSearch.serialize(json));
390  printSuccess("Serialize SphinxInputJsonMessageSearch");
391 
392  SphinxInputJsonMessage handler;
393  handler.setType(SphinxInputJsonMessage::MessageType::mtSearch);
394  handler.setData(json);
395 
396  std::stringstream str;
397  str << handler;
398 
399  std::string ret = fObj.Process(str.str());
400 
401  std::cout << "ret: " << ret << std::endl;
402 
403  SphinxOutputJsonMessage outputMessage(ret);
404  ASSERT_FALSE(outputMessage.isError());
405  ASSERT_TRUE(resultSerializator.unserialize(outputMessage.getData()));
406 
407  ASSERT_FALSE(resultSerializator.isError());
408  ASSERT_TRUE(resultSerializator.getErrorCode()==NO_ERROR);
409  ASSERT_TRUE(resultSerializator.getErrorMsg().empty());
410  printSuccess("Make Search Request");
411 }
412 //---------------------------------------------------------------------------------------
413 void SphinxSearchTest::testSearcherProcess(SphinxFunctionalObject& fObj, const std::string& query, unsigned int maxResults)
414 {
415  const std::string title = "Test Searcher Process";
416  printStartMessage(title);
417  ASSERT_TRUE(fObj.isLoadedConfig());
418 
419  SphinxResultData resultData;
420  const std::vector<std::pair<std::string, std::string> > queryParameters = {
421  {"queryId", "7475839"},
422  {"JsonType", "15"},
423  {"timeout", "5"},
424  {"max_results", std::to_string(maxResults)},
425  {"offset", "0"},
426  {"limit", "0"},
427  {"cutoff", "0"},
428  {"order_by", "0"},
429  {"ttl", "3"}
430  };
431  const std::vector<std::pair<std::string, std::string> > orderParameters = {
432  {"algorithm", "0"},
433  {"order_by", "2"}
434  };
435  const std::vector<std::string> externalFields = {"uid", "sid", "lang"};
436  const std::vector<std::string> orderFields = {"uid", "lang", "sid"};
437 
438  SphinxSearchTest::makeSearchRequest(fObj, query, queryParameters, orderParameters, externalFields, orderFields, resultData);
439 
440  if (!fObj.isError())
441  printSuccess(title);
442  else
443  printFail(title);
444 }
445 //-----------------------------------------------------------------------------
446 void SphinxSearchTest::testGetResultUseOffset(SphinxFunctionalObject& fObj, const std::string& query, unsigned int maxResults, unsigned int offset)
447 {
448  const std::string title = "Test Get Result Use Offset";
449  printStartMessage(title);
450  ASSERT_TRUE(fObj.isLoadedConfig());
451 
452  std::set<unsigned long long> ids;
453 
454  SphinxResultData resultData;
455  std::vector<std::pair<std::string, std::string> > queryParameters = {
456  {"queryId", "7475839"},
457  {"JsonType", "15"},
458  {"timeout", "5"},
459  {"max_results", std::to_string(maxResults)},
460  {"offset", std::to_string(offset)},
461  {"limit", "0"},
462  {"cutoff", "0"},
463  {"order_by", "1"},
464  {"ttl", "3"}
465  };
466  const std::vector<std::pair<std::string, std::string> > orderParameters = {
467  {"algorithm", "0"},
468  {"order_by", "2"}
469  };
470  const std::vector<std::string> externalFields = {"uid", "sid", "lang"};
471  const std::vector<std::string> orderFields = {"uid", "lang", "sid"};
472 
473  SphinxSearchTest::makeSearchRequest(fObj, query, queryParameters, orderParameters, externalFields, orderFields, resultData);
474 
475  const size_t requestInfoCount = resultData.getRequestInfoCount();
476  for (size_t i=0;i<requestInfoCount;++i)
477  {
478  const unsigned int totalFound = resultData.getRequestInfoItem(i).getTotalFound();
479  std::cout << "TOTAL FOUND: " << totalFound << std::endl;
480 
481  std::cout << "offset = " << offset << std::endl;
482 
483  for (size_t k=0;k<totalFound;k+=maxResults)
484  {
485  for (size_t j=0;j<queryParameters.size();++j)
486  {
487  if (queryParameters[j].first=="offset")
488  {
489  queryParameters[j].second = std::to_string(offset);
490  std::cout << "offset = " << queryParameters[j].second << std::endl;
491  }
492 
493  if (queryParameters[j].first=="max_results")
494  {
495  queryParameters[j].second = std::to_string(offset+maxResults);
496  std::cout << "max_results = " << queryParameters[j].second << std::endl;
497  }
498  }
499  SphinxSearchTest::makeSearchRequest(fObj, query, queryParameters, orderParameters, externalFields, orderFields, resultData);
500  const size_t matchInfoCount = resultData.getMatchInfoCount();
501  std::cout << "MATCH INFO COUNT: " << matchInfoCount << std::endl;
502  for (size_t q=0;q<matchInfoCount;++q)
503  ids.insert(resultData.getMatchInfoItem(q).getDocId());
504 
505  const size_t requestInfoCount = resultData.getRequestInfoCount();
506  for (size_t q=0;q<requestInfoCount;++q)
507  {
508  const unsigned int totalFound = resultData.getRequestInfoItem(q).getTotalFound();
509  std::cout << "TOTAL FOUND: " << totalFound << std::endl;
510  }
511  offset += (maxResults-1);
512  }
513  }
514 
515  std::cout << "UNIQUE ID FOUND: " << ids.size() << std::endl;
516 
517  if (!fObj.isError())
518  printSuccess(title);
519  else
520  printFail(title);
521 }
522 //-----------------------------------------------------------------------------
523 //-----------------------------------------------------------------------------
524 // cppcheck-suppress unusedFunction
525 unsigned int SphinxSearchTest::simpleSearchQuery(SphinxFunctionalObject& fObj, const std::string& query, size_t repeatCount)
526 {
527  unsigned int resultTime = 0;
528 
529  std::string json;
530  SphinxInputJsonMessageSearch msgSearch;
531  msgSearch.setQueryString(query);
532  msgSearch.addQueryParameters("queryId", "777");
533  msgSearch.addQueryParameters("JsonType", "7");
534  msgSearch.addQueryParameters("max_results", "5");
535  msgSearch.addQueryParameters("offset", "0");
536  msgSearch.addQueryParameters("limit", "0");
537  msgSearch.addQueryParameters("cutoff", "0");
538  assert(msgSearch.serialize(json));
539  printSuccess("Serialize SphinxInputJsonMessageSearch");
540 
541  SphinxInputJsonMessage inputJsonMessage;
542  inputJsonMessage.setType(SphinxInputJsonMessage::MessageType::mtSearch);
543  inputJsonMessage.setData(json);
544 
545  for (size_t i=0;i<repeatCount;++i)
546  {
547  SphinxOutputJsonMessage outputMessage = fObj.Process(inputJsonMessage);
548  if (!fObj.getErrorMsg().empty())
549  std::cout << "fObj.getErrorMsg() = " << fObj.getErrorMsg() << std::endl;
550  assert(fObj.getErrorMsg().empty());
551  assert(!outputMessage.isError());
552  resultTime += outputMessage.getTime();;
553  }
554  return resultTime;
555 }
556 //-----------------------------------------------------------------------------
557 void SphinxSearchTest::testSerializationSearchProcess(const std::string& json)
558 {
559  SphinxInputJsonMessage inputJsonMessage(json);
560 
561 // std::cout << "\njson: " << json << std::endl << std::endl;
562 
563 // std::cout << "inputJsonMessage.isError() = " << std::boolalpha << inputJsonMessage.isError() << std::endl;
564 // std::cout << "inputJsonMessage.getErrorCode() = " << inputJsonMessage.getErrorCode() << std::endl;
565 // std::cout << "inputJsonMessage.getErrorMsg(): '" << inputJsonMessage.getErrorMsg() << "'" << std::endl;
566  ASSERT_FALSE(inputJsonMessage.isError());
567  ASSERT_TRUE(inputJsonMessage.getErrorCode()==NO_ERROR);
568  ASSERT_TRUE(inputJsonMessage.getErrorMsg().empty());
569  printSuccess("Unserialize SphinxInputJsonMessage");
570 
571  SphinxInputJsonMessage::MessageType type = inputJsonMessage.getType();
572  std::string dataString = inputJsonMessage.getData();
573 
574 // std::cout << "\n dataString = " << dataString << std::endl << std::endl;
575 
576 
577  SphinxInputJsonMessageSearch msgSearch(dataString);
578 // std::cout << "msgSearch.isError() = " << std::boolalpha << msgSearch.isError() << std::endl;
579 // std::cout << "msgSearch.getErrorCode() = " << msgSearch.getErrorCode() << std::endl;
580 // std::cout << "msgSearch.getErrorMsg(): '" << msgSearch.getErrorMsg() << "'" << std::endl;
581  ASSERT_FALSE(msgSearch.isError());
582  ASSERT_TRUE(msgSearch.getErrorCode()==NO_ERROR);
583  ASSERT_TRUE(msgSearch.getErrorMsg().empty());
584  printSuccess("Unserialize SphinxInputJsonMessageSearch");
585 
586  std::string restoreString;
587  bool ret = msgSearch.serialize(restoreString);
588 
589 // std::cout << "\n restoreString = " << restoreString << std::endl << std::endl;
590 
591 // std::cout << "msgSearch.serialize() = " << std::boolalpha << ret << std::endl;
592 // std::cout << "msgSearch.isError() = " << std::boolalpha << msgSearch.isError() << std::endl;
593 // std::cout << "msgSearch.getErrorCode() = " << msgSearch.getErrorCode() << std::endl;
594 // std::cout << "msgSearch.getErrorMsg(): '" << msgSearch.getErrorMsg() << "'" << std::endl;
595  ASSERT_TRUE(ret);
596  ASSERT_FALSE(msgSearch.isError());
597  ASSERT_TRUE(msgSearch.getErrorCode()==NO_ERROR);
598  ASSERT_TRUE(msgSearch.getErrorMsg().empty());
599  printSuccess("Serialize SphinxInputJsonMessageSearch");
600 
601  std::string restoreJson;
602  SphinxInputJsonMessage restoreInputJsonMessage;
603  restoreInputJsonMessage.setType(type);
604  restoreInputJsonMessage.setData(restoreString);
605  ret = restoreInputJsonMessage.serialize(restoreJson);
606 // std::cout << "restoreInputJsonMessage.serialize() = " << std::boolalpha << ret << std::endl;
607 // std::cout << "restoreInputJsonMessage.isError() = " << std::boolalpha << restoreInputJsonMessage.isError() << std::endl;
608 // std::cout << "restoreInputJsonMessage.getErrorCode() = " << restoreInputJsonMessage.getErrorCode() << std::endl;
609 // std::cout << "restoreInputJsonMessage.getErrorMsg(): '" << restoreInputJsonMessage.getErrorMsg() << "'" << std::endl;
610  ASSERT_TRUE(ret);
611  ASSERT_FALSE(restoreInputJsonMessage.isError());
612  ASSERT_TRUE(restoreInputJsonMessage.getErrorCode()==NO_ERROR);
613  ASSERT_TRUE(restoreInputJsonMessage.getErrorMsg().empty());
614  printSuccess("Serialize SphinxInputJsonMessage");
615 
616  SphinxInputJsonMessageSearch restoreSearch(restoreInputJsonMessage.getData());
617 // std::cout << "restoreSearch.isError() = " << std::boolalpha << msgSearch.isError() << std::endl;
618 // std::cout << "restoreSearch.getErrorCode() = " << msgSearch.getErrorCode() << std::endl;
619 // std::cout << "restoreSearch.getErrorMsg(): '" << msgSearch.getErrorMsg() << "'" << std::endl;
620  ASSERT_FALSE(restoreSearch.isError());
621  ASSERT_TRUE(restoreSearch.getErrorCode()==NO_ERROR);
622  ASSERT_TRUE(restoreSearch.getErrorMsg().empty());
623  printSuccess("Unserialize SphinxInputJsonMessageSearch");
624 
625  // check validate
626  ASSERT_EQ(msgSearch.getQueryString(), restoreSearch.getQueryString());
627 
628  std::vector<std::pair<std::string, std::string> >& sourceParams = msgSearch.getQueryParameters();
629  std::vector<std::pair<std::string, std::string> >& restoreParams = restoreSearch.getQueryParameters();
630 
631 // std::cout << " sourceParams.size() = " << sourceParams.size() << std::endl;
632 // for (size_t i=0;i<sourceParams.size();++i)
633 // std::cout << i << ") " << sourceParams[i].first << " = " << sourceParams[i].second << std::endl;
634 
635 // std::cout << "restoreParams.size() = " << restoreParams.size() << std::endl;
636 // for (size_t i=0;i<restoreParams.size();++i)
637 // std::cout << i << ") " << restoreParams[i].first << " = " << restoreParams[i].second << std::endl;
638 
639  ASSERT_EQ(sourceParams.size(), restoreParams.size());
640 
641 
642  for (size_t i=0;i<sourceParams.size();++i)
643  {
644  bool found = false;
645  for (size_t k=0;k<restoreParams.size();++k)
646  if (sourceParams[i].first==restoreParams[k].first)
647  {
648  found = true;
649  break;
650  }
651  ASSERT_TRUE(found);
652  }
653 
654  SphinxFiltersArray& sourceFilters = msgSearch.getFilters();
655  SphinxFiltersArray& restoreFilters = restoreSearch.getFilters();
656  ASSERT_EQ(sourceFilters.getFilters().size(), restoreFilters.getFilters().size());
657  for (size_t i=0;i<sourceFilters.getFilters().size();++i)
658  {
659  ASSERT_EQ(sourceFilters.getFilters()[i]->getFilterType(), restoreFilters.getFilters()[i]->getFilterType());
660  ASSERT_EQ(sourceFilters.getFilters()[i]->getAttributeName(), restoreFilters.getFilters()[i]->getAttributeName());
661  ASSERT_EQ(sourceFilters.getFilters()[i]->getExcludeValue(), restoreFilters.getFilters()[i]->getExcludeValue());
662  ASSERT_EQ(sourceFilters.getFilters()[i]->getAttributeValues().size(), restoreFilters.getFilters()[i]->getAttributeValues().size());
663  for (size_t k=0;k<sourceFilters.getFilters()[i]->getAttributeValues().size();++k)
664  {
665  ASSERT_EQ(sourceFilters.getFilters()[i]->getAttributeValues()[k], restoreFilters.getFilters()[i]->getAttributeValues()[k]);
666  }
667  }
668  printSuccess("Test Serialization Search Process");
669 }
670 //-----------------------------------------------------------------------------
671 SphinxOutputJsonMessage SphinxSearchTest::searchWithFilter(SphinxFunctionalObject& fObj,
672  const std::string& query, std::vector<Poco::SharedPtr<SphinxFilter> >& filters)
673 {
674  assert(fObj.isLoadedConfig());
675 
676  std::string json;
677  SphinxInputJsonMessageSearch inputJsonMessageSearch;
678  inputJsonMessageSearch.setQueryString(query);
679  inputJsonMessageSearch.addQueryParameters("queryId", "1234");
680  inputJsonMessageSearch.addQueryParameters("jsonType", "15");
681  inputJsonMessageSearch.addQueryParameters("max_results", "100");
682 
683  for (size_t i=0;i<filters.size();++i)
684  inputJsonMessageSearch.addFilter(filters[i]);
685 
686  bool ret = inputJsonMessageSearch.serialize(json);
687  fObj.log(HCE::LoggerStream::PRIO_INFORMATION) << "SphinxInputJsonMessageSearch serialize => " << std::boolalpha << ret << std::endl;
688  assert(ret);
689 // std::cout << "json: " << json << std::endl;
690 
691  SphinxInputJsonMessage inputJsonMessage;
692  inputJsonMessage.setType(SphinxInputJsonMessage::MessageType::mtSearch);
693  inputJsonMessage.setData(json);
694 
695  ret = inputJsonMessage.serialize(json);
696  fObj.log(HCE::LoggerStream::PRIO_INFORMATION) << "\n\nSphinxInputJsonMessage serialize => " << std::boolalpha << ret << std::endl;
697  assert(ret);
698 // std::cout << "json: " << json << std::endl;
699 
700  SphinxOutputJsonMessage outputMessage = fObj.Process(inputJsonMessage);
701  if (!fObj.getErrorMsg().empty())
702  fObj.log(HCE::LoggerStream::PRIO_INFORMATION) << "fObj.getErrorMsg() = " << fObj.getErrorMsg() << std::endl;
703  if (fObj.getErrorCode()!=NO_ERROR)
704  fObj.log(HCE::LoggerStream::PRIO_INFORMATION) << "fObj.getErrorCode() = " << fObj.getErrorCode() << std::endl;
705  if (outputMessage.isError())
706  fObj.log(HCE::LoggerStream::PRIO_INFORMATION) << "outputMessage.isError() = " << outputMessage.isError() << std::endl;
707 
708  printSuccess("Search with filters");
709 
710  return outputMessage;
711 }
712 //-----------------------------------------------------------------------------
713 //-----------------------------------------------------------------------------
714 void SphinxSearchTest::testFilterSearch(SphinxFunctionalObject& fObj)
715 {
716  printStartMessage("Test use filter to Search Process");
717 
718  const std::string query = "some";
719  std::string json;
720  SphinxInputJsonMessageSearch msgSearch;
721  std::vector<std::string> attrs = {"123"};
722  msgSearch.addFilter((SphinxFilter::FilterType)0, "cdate", attrs, (SphinxFilter::ExcludeType)0);
723 
724  msgSearch.setQueryString(query);
725  msgSearch.addQueryParameters("queryId", "123");
726  msgSearch.addQueryParameters("JsonType", "15");
727  ASSERT_TRUE(msgSearch.serialize(json));
728  printSuccess("Serialize SphinxInputJsonMessageSearch");
729 
730  SphinxInputJsonMessage handler;
731  handler.setType(SphinxInputJsonMessage::MessageType::mtSearch);
732  handler.setData(json);
733 
734  std::stringstream str;
735  str << handler;
736 
737 // std::cout << "str.str(): '" << str.str() << "'\n";
738 
739  std::string outputJson = fObj.Process(str.str());
740 
741 // std::cout << "JSON: '" << outputJson << "'\n";
742 
743  SphinxOutputJsonMessage outputMessage(outputJson);
744  if (!fObj.getErrorMsg().empty())
745  std::cout << "fObj.getErrorMsg() = " << fObj.getErrorMsg() << std::endl;
746 
747  ASSERT_FALSE(fObj.isError());
748  ASSERT_TRUE(fObj.getErrorCode()==NO_ERROR);
749  ASSERT_TRUE(fObj.getErrorMsg().empty());
750  ASSERT_FALSE(outputMessage.isError());
751 
752  SphinxResultData Result;
753  SphinxResultDataSerializator ResultSerializator(Result);
754  ASSERT_TRUE(ResultSerializator.unserialize(outputMessage.getData()));
755  ASSERT_TRUE(ResultSerializator.getErrorCode()==NO_ERROR);
756  ASSERT_TRUE(ResultSerializator.getErrorMsg().empty());
757 
758 // std::cout << "Result:\n" << Result << std::endl;
759  printSuccess("Test use filter to Search Process");
760 }
761 //-----------------------------------------------------------------------------
762 void SphinxSearchTest::testWeightPacker(void)
763 {
764  const size_t fieldWidth = HEX_STRING_LENGTH;
765  size_t fieldsCount = 0;
766  std::stringstream str;
767  for (size_t i=45567;i<45570;++i)
768  {
769  str << HexString(i);
770  ++fieldsCount;
771  }
772 
773  std::string tmp = str.str();
774  SphinxWeightPacker packer(fieldsCount, fieldWidth);
775  for (size_t i=0;i<=fieldsCount+1;++i)
776  {
777  std::string packedData = packer.pack(tmp, i);
778  std::string unpackedData = packer.unpack(packedData);
779  ASSERT_EQ(tmp, unpackedData);
780  }
781 
782  tmp.resize(fieldsCount*fieldWidth+5);
783  for (size_t i=0;i<=fieldsCount+1;++i)
784  {
785  std::string packetData = packer.pack(tmp, i);
786  std::string unpackedData = packer.unpack(packetData);
787  ASSERT_TRUE(tmp.compare(0, fieldsCount*fieldWidth, unpackedData)==0 || tmp==unpackedData);
788  }
789 
790  tmp.resize(fieldsCount*fieldWidth-5);
791  for (size_t i=0;i<=fieldsCount+1;++i)
792  {
793  std::string packetData = packer.pack(tmp, i);
794  std::string unpackedData = packer.unpack(packetData);
795  ASSERT_TRUE(unpackedData.compare(0, fieldsCount*fieldWidth-5, tmp)==0);
796  }
797  printSuccess("Test Weight Packer");
798 }
799 //-----------------------------------------------------------------------------
800 void SphinxSearchTest::testWeightCalculator(void)
801 {
802  SphinxResultData resultData;
803  SphinxWeightCalculator calculator;
804 
805  SphinxMatchInfo matchInfo;
806  matchInfo.setDocId(2345);
807  matchInfo.setWeightFromDecString("2345");
808 
809  matchInfo.addAttrInfo("uid", "1");
810  matchInfo.addAttrInfo("sid", "2");
811  matchInfo.addAttrInfo("lang", "255");
812  resultData.addMatchInfo(matchInfo);
813 
814  const std::vector<std::string> orderFields = {"sid", "lang", "uid"};
815 
816  calculator.setStrategy(new CalculateStrategyDefault());
817  calculator.calculate(orderFields, resultData);
818 
819  size_t matchInfoCount = resultData.getMatchInfoCount();
820  for (size_t i=0;i<matchInfoCount;++i)
821  ASSERT_EQ(resultData.getMatchInfoItem(i).getWeight(), "00000000000000000929");
822 
823  resultData.clear();
824  resultData.addMatchInfo(matchInfo);
825 
826  calculator.setStrategy(new CalculateStrategyUseFieldNames());
827  calculator.calculate(orderFields, resultData);
828  matchInfoCount = resultData.getMatchInfoCount();
829  for (size_t i=0;i<matchInfoCount;++i)
830  ASSERT_EQ(resultData.getMatchInfoItem(i).getWeight(), "00000000000000000002000000000000000000ff00000000000000000001");
831 
832  printSuccess("Test Weight Calculator");
833 }
834 //-----------------------------------------------------------------------------
835 //-----------------------------------------------------------------------------
836 } // end namespace tests
837 } // end namespace sphinx
838 } // end namespace HCE
839 
840 
841 
842 
843