hce-node application  1.4.3
HCE Hierarchical Cluster Engine node application
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros
SphinxFunctionalObjectTest.cpp
Go to the documentation of this file.
1 #include <gtest/gtest.h>
2 #include <Poco/SharedPtr.h>
3 #include <fstream>
4 #include <assert.h>
5 #include <unistd.h>
6 #include <iomanip>
7 
8 #include "SphinxError.hpp"
9 #include "SphinxManageTest.hpp"
10 #include "SphinxSearchTest.hpp"
11 #include "SphinxIndexTest.hpp"
12 #include "SphinxMultiIndexTest.hpp"
13 
14 #include "SphinxInputJsonMessageSearch.hpp"
15 #include "SphinxInputJsonMessage.hpp"
16 #include "SphinxOutputJsonMessage.hpp"
17 #include "SphinxFunctionalObject.hpp"
18 #include "SphinxAdminCommand.hpp"
19 #include "SphinxStressTest.hpp"
20 
21 #include "SphinxFunctionalObjectTest.hpp"
22 #include "SphinxPrintStatus.hpp"
23 
24 namespace HCE
25 {
26 namespace sphinx
27 {
28 namespace tests
29 {
30 //-----------------------------------------------------------------------------
31 //-----------------------------------------------------------------------------
32 void SphinxFunctionalObjectTest::fullAdminCommandsTest(SphinxFunctionalObject& fObj)
33 {
34  const std::string indexName = "i003";
35 
36  fObj.setCheckIncomingDocuments(true);
37  fObj.setPackIncomingDocuments(true);
38 
39  SphinxManageTest::testIndexCreate(fObj, indexName);
40 
41  SphinxManageTest::testIndexCheck(fObj, indexName, true);
42  SphinxManageTest::testIndexCheck(fObj, "branch_0006", false);
43 
44  std::vector<std::string> branches;
45  /*
46  branches.push_back("_branch11");
47  branches.push_back("_branch12");
48  for (size_t i=0;i<branches.size();++i)
49  SphinxManageTest::testIndexStoreDataFile(fObj, indexName, branches[i]);
50  SphinxManageTest::testIndexStoreSchemaFile(fObj, indexName);
51  SphinxManageTest::testIndexRebuild(fObj, indexName, branches);
52  SphinxManageTest::testIndexMerge(fObj, indexName, branches);
53  */
54 
55  branches.clear();
56  branches.push_back("branch_0005");
57  branches.push_back("branch_0006");
58  branches.push_back("branch_0007");
59  for (size_t i=0;i<branches.size();++i)
60  {
61  SphinxManageTest::testIndexStoreDataFile(fObj, indexName, branches[i], "");
62  SphinxManageTest::testIndexStoreDataFile(fObj, indexName, branches[i]);
63  }
64  SphinxManageTest::testIndexStoreSchemaFile(fObj, indexName, "");
65  SphinxManageTest::testIndexStoreSchemaFile(fObj, indexName);
66  SphinxManageTest::testIndexRebuild(fObj, indexName, branches);
67 
68  SphinxManageTest::testIndexMaxDocId(fObj, indexName);
69  SphinxManageTest::testIndexMaxDocIdExtended(fObj, indexName);
70 
71  for (size_t i=0;i<branches.size();++i)
72  {
73  SphinxManageTest::testIndexStoreDataFile(fObj, indexName, branches[i]);
74  }
75  branches.push_back("branch_0008");
76  branches.push_back("branch_0009");
77  SphinxManageTest::testIndexRebuild(fObj, indexName, branches);
78 
79  branches.clear();
80  branches.push_back("*");
81  SphinxManageTest::testIndexRebuild(fObj, indexName, branches);
82 // SphinxManageTest::testIndexSetDataDir(fObj); // can use with or without it
83 
84  SphinxManageTest::testIndexMerge(fObj, indexName, branches);
85 
86  SphinxManageTest::testIndexStart(fObj, indexName);
87 
88  branches.clear();
89  branches.push_back("branch_0005");
90  SphinxManageTest::testIndexDeleteDataFile(fObj, indexName, branches);
91 
92  branches.clear();
93  branches.push_back("branch_0005");
94  branches.push_back("branch_0006");
95  branches.push_back("branch_0007");
96  SphinxManageTest::testIndexRebuild(fObj, indexName, branches);
97  SphinxManageTest::testIndexMerge(fObj, indexName, branches);
98  SphinxManageTest::testIndexStart(fObj, indexName);
99 
100  branches.clear();
101  branches.push_back("branch_0005");
102  SphinxManageTest::testIndexMerge(fObj, indexName, branches);
103  branches.push_back("branch_0006");
104  SphinxManageTest::testIndexMergeTrunk(fObj, indexName, branches);
105  branches.push_back("branch_0007");
106  SphinxManageTest::testIndexMergeTrunk(fObj, indexName, branches);
107  branches.push_back("branch_0008"); // not exists
108  branches.push_back("branch_0009"); // not exists
109  SphinxManageTest::testIndexMerge(fObj, indexName, branches);
110  SphinxManageTest::testIndexStart(fObj, indexName);
111  SphinxManageTest::testIndexStop(fObj, indexName);
112  SphinxManageTest::testIndexStart(fObj, indexName);
113 
114  SphinxManageTest::testIndexDeleteDataFile(fObj, indexName, branches);
115  for (size_t i=0;i<branches.size();++i)
116  SphinxManageTest::testIndexStoreDataFile(fObj, indexName, branches[i]);
117  branches.clear();
118  branches.push_back("*");
119  SphinxManageTest::testIndexDeleteDataFile(fObj, indexName, branches);
120  SphinxManageTest::testIndexDeleteSchemaFile(fObj, indexName);
121 
122  branches.clear();
123  branches.push_back("branch_0005");
124  branches.push_back("branch_0006");
125  for (size_t i=0;i<branches.size();++i)
126  SphinxManageTest::testIndexStoreDataFile(fObj, indexName, branches[i]);
127  SphinxManageTest::testIndexStoreSchemaFile(fObj, indexName);
128  for (size_t i=0;i<branches.size();++i)
129  SphinxManageTest::testIndexAppendDataFile(fObj, indexName, branches[i]);
130 
131  std::vector<unsigned long long> ids = { 123456789012, 123456789023, 123456789034, 123456789045, 123456789056, 123456789067, 123456789078, 123456789089, 123456789090, 1234567890101};
132  SphinxManageTest::testIndexDeleteDoc(fObj, indexName, ids);
133  SphinxManageTest::testIndexDeleteDocNumber(fObj, indexName);
134 
135  branches.clear();
136  branches.push_back("branch_0005");
137  branches.push_back("branch_0006");
138  branches.push_back("branch_0007");
139  for (size_t i=0;i<branches.size();++i)
140  SphinxManageTest::testIndexStoreDataFile(fObj, indexName, branches[i]);
141 
142  SphinxManageTest::testIndexDeleteDocNumber(fObj, indexName);
143  SphinxManageTest::testIndexPackDocData(fObj, indexName, branches);
144 
145  SphinxManageTest::testIndexRebuild(fObj, indexName, branches);
146  SphinxManageTest::testIndexMerge(fObj, indexName, branches);
147  branches.clear();
148  branches.push_back("*");
149  SphinxManageTest::testIndexPackDocData(fObj, indexName, branches);
150  SphinxManageTest::testIndexMerge(fObj, indexName, branches);
151 
152  SphinxManageTest::testIndexRemove(fObj, indexName);
153  SphinxManageTest::testIndexCheck(fObj, indexName, false);
154 
155  // prepare for next copy index
156  SphinxManageTest::testIndexCreate(fObj, indexName);
157  SphinxManageTest::testIndexCheck(fObj, indexName, true);
158  branches.clear();
159  branches.push_back("branch_0005");
160  branches.push_back("branch_0006");
161  branches.push_back("branch_0007");
162  for (size_t i=0;i<branches.size();++i)
163  SphinxManageTest::testIndexStoreDataFile(fObj, indexName, branches[i]);
164  SphinxManageTest::testIndexStoreSchemaFile(fObj, indexName);
165  SphinxManageTest::testIndexRebuild(fObj, indexName, branches);
166  SphinxManageTest::testIndexMerge(fObj, indexName, branches);
167  // make copy index
168  const std::string copyIndexName = "i004";
169  SphinxManageTest::testIndexStart(fObj, indexName);
170  SphinxManageTest::testIndexStop(fObj, indexName);
171  SphinxManageTest::testIndexStart(fObj, indexName);
172  SphinxManageTest::testIndexCopy(fObj, indexName, copyIndexName);
173  SphinxManageTest::testIndexCheck(fObj, copyIndexName, true);
174 
175  SphinxManageTest::testIndexStart(fObj, copyIndexName);
176 
177  // make rename index
178  const std::string renameIndexName = "i005";
179  SphinxManageTest::testIndexRename(fObj, copyIndexName, renameIndexName);
180  SphinxManageTest::testIndexCheck(fObj, copyIndexName, false);
181  fObj.resetError();
182  SphinxManageTest::testIndexCheck(fObj, renameIndexName, true);
183 
184  SphinxManageTest::testIndexStart(fObj, renameIndexName);
185  // make remove index
186  SphinxManageTest::testIndexRemove(fObj, renameIndexName);
187  SphinxManageTest::testIndexCheck(fObj, renameIndexName, false);
188  fObj.resetError();
189 
190  // set config property value
191  SphinxManageTest::testIndexSetConfigVar(fObj, indexName, "", "searchd", "test_param", "test_value1");
192  SphinxManageTest::testIndexSetConfigVar(fObj, indexName, "", "index branch_0001", "test_param", "test_value2");
193  SphinxManageTest::testIndexSetConfigVar(fObj, indexName, "branch_0005", "searchd", "test_param", "test_value1");
194  SphinxManageTest::testIndexSetConfigVar(fObj, indexName, "branch_0005", "index branch_0001", "test_param", "test_value2");
195 
196  // get config property value
197  std::string paramValue;
198  SphinxManageTest::testIndexGetConfigVar(fObj, indexName, "", "searchd", "test_param", paramValue);
199  SphinxManageTest::testIndexGetConfigVar(fObj, indexName, "", "index branch_0001", "test_param", paramValue);
200  SphinxManageTest::testIndexGetConfigVar(fObj, indexName, "branch_0005", "searchd", "test_param", paramValue);
201  SphinxManageTest::testIndexGetConfigVar(fObj, indexName, "branch_0005", "index branch_0001", "test_param", paramValue);
202 
203  // check validation of schema data and save in index if valid
204  SphinxManageTest::testIndexDeleteSchemaFile(fObj, indexName);
205 
206  // recreate index for test
207  SphinxManageTest::testIndexRemove(fObj, indexName);
208  SphinxManageTest::testIndexCheck(fObj, indexName, false);
209  fObj.resetError();
210  SphinxManageTest::testIndexCreate(fObj, indexName);
211  SphinxManageTest::testIndexCheck(fObj, indexName, true);
212  // get data about status of Sphinx searchd
213  for (size_t i=0;i<branches.size();++i)
214  SphinxManageTest::testIndexStoreDataFile(fObj, indexName, branches[i]);
215  SphinxManageTest::testIndexStoreSchemaFile(fObj, indexName);
216 
217  SphinxManageTest::testIndexCheckSchema(fObj, indexName);
218 
219  SphinxManageTest::testIndexRebuild(fObj, indexName, branches);
220  SphinxManageTest::testIndexMerge(fObj, indexName, branches);
221  SphinxManageTest::testIndexStart(fObj, indexName);
222 
223  SphinxManageTest::testIndexStatusSearchd(fObj, indexName);
224  SphinxManageTest::testIndexStatus(fObj, indexName);
225  SphinxManageTest::testIndexDataList(fObj, indexName);
226  SphinxManageTest::testIndexBranchesList(fObj, indexName);
227  SphinxManageTest::testIndexBranchesInfo(fObj, indexName, branches);
228  SphinxManageTest::testIndexBranchesStatus(fObj, indexName, branches);
229 
230  SphinxManageTest::testIndexConnect(fObj, indexName);
231  SphinxManageTest::testIndexDisconnect(fObj);
232  SphinxManageTest::testIndexDisconnect(fObj);
233  SphinxManageTest::testIndexConnect(fObj, indexName);
234  SphinxManageTest::testIndexConnect(fObj, indexName);
235 
236  SphinxManageTest::testIndexRemove(fObj, indexName);
237 }
238 //-----------------------------------------------------------------------------
239 void SphinxFunctionalObjectTest::changeSearchIndexTest(SphinxFunctionalObject& fObj)
240 {
241  const std::string indexName = "i003";
242 
243  // create first index
244  SphinxManageTest::testIndexCreate(fObj, indexName);
245  SphinxManageTest::testIndexCheck(fObj, indexName, true);
246  std::vector<std::string> branches = {"branch_0005"};
247  for (size_t i=0;i<branches.size();++i)
248  SphinxManageTest::testIndexStoreDataFile(fObj, indexName, branches[i]);
249  SphinxManageTest::testIndexStoreSchemaFile(fObj, indexName);
250  SphinxManageTest::testIndexRebuild(fObj, indexName, branches);
251  SphinxManageTest::testIndexMerge(fObj, indexName, branches);
252  SphinxManageTest::testIndexStart(fObj, indexName);
253  std::cout << "IndexStart >>>>>> isActiveSearchd = " << std::boolalpha << fObj.isActiveSearchd() << std::endl;
254  SphinxSearchTest::testSearcherProcess(fObj);
255  // make second index
256  const std::string newIndexName = "i004";
257  SphinxManageTest::testIndexCopy(fObj, indexName, newIndexName);
258  SphinxManageTest::testIndexStart(fObj, newIndexName);
259  std::cout << "IndexStart >>>>>> isActiveSearchd = " << std::boolalpha << fObj.isActiveSearchd() << std::endl;
260  SphinxSearchTest::testSearcherProcess(fObj);
261  SphinxManageTest::testIndexRemove(fObj, newIndexName);
262  SphinxManageTest::testIndexRemove(fObj, indexName);
263 }
264 //-----------------------------------------------------------------------------
265 void SphinxFunctionalObjectTest::autoGettingPortsSearchd(SphinxFunctionalObject& fObj)
266 {
267  const std::string nodeName = "test_node";
268 
269  const unsigned int SZ = 3;
270  const std::string indexName[SZ] = {"i003", "i004" , "i005"};
271  Poco::SharedPtr<SphinxFunctionalObject> pObj[SZ];
272 
273  for (unsigned int i=0;i<SZ;++i)
274  {
275  pObj[i] = new SphinxFunctionalObject(nodeName, fObj.getHomeDir());
276 
277  SphinxManageTest::testIndexCreate(*(pObj[i]), indexName[i]);
278  SphinxManageTest::testIndexCheck(*(pObj[i]), indexName[i], true);
279  std::vector<std::string> branches = {"branch_0005"};
280  for (size_t k=0;k<branches.size();++k)
281  SphinxManageTest::testIndexStoreDataFile(*(pObj[i]), indexName[i], branches[k]);
282  SphinxManageTest::testIndexStoreSchemaFile(*(pObj[i]), indexName[i]);
283  SphinxManageTest::testIndexRebuild(*(pObj[i]), indexName[i], branches);
284 // SphinxManageTest::testIndexStart(*(pObj[i]), indexName[i]);
285  SphinxManageTest::testIndexMerge(*(pObj[i]), indexName[i], branches); // TODO
286  SphinxManageTest::testIndexStart(*(pObj[i]), indexName[i]);
287 // SphinxManageTest::testIndexStatus(*(pObj[i]), indexName[i]);
288  SphinxSearchTest::testSearcherProcess(*(pObj[i]));
289  }
290  for (unsigned int i=0;i<SZ;++i)
291  SphinxManageTest::testIndexRemove(*(pObj[i]), indexName[i]);
292 
293  for (unsigned int i=0;i<SZ;++i)
294  ASSERT_FALSE(pObj[i]->isError());
295 }
296 //-----------------------------------------------------------------------------
297 void SphinxFunctionalObjectTest::fullIndexerTest(SphinxFunctionalObject& fObj)
298 {
299  SphinxIndexTest::testDataStorage();
300  SphinxIndexTest::testInputJsonMessage();
301  SphinxIndexTest::testIndexerProcess(fObj);
302 }
303 //-----------------------------------------------------------------------------
304 void SphinxFunctionalObjectTest::fullSearcherTest(SphinxFunctionalObject& fObj)
305 {
306  const std::string indexName = "i003";
307 
308  // create first index
309  SphinxManageTest::testIndexCreate(fObj, indexName);
310  SphinxManageTest::testIndexCheck(fObj, indexName, true);
311  std::vector<std::string> branches = {"branch_0005"};
312  for (size_t i=0;i<branches.size();++i)
313  SphinxManageTest::testIndexStoreDataFile(fObj, indexName, branches[i]);
314  SphinxManageTest::testIndexStoreSchemaFile(fObj, indexName);
315  SphinxManageTest::testIndexRebuild(fObj, indexName, branches);
316  SphinxManageTest::testIndexMerge(fObj, indexName, branches);
317  SphinxManageTest::testIndexStart(fObj, indexName);
318  ASSERT_TRUE(fObj.isActiveSearchd());
319 
320  SphinxSearchTest::testWeightPacker();
321  SphinxSearchTest::testWeightCalculator();
322 
323  SphinxSearchTest::testSerialization();
324  SphinxSearchTest::testSerializationExt();
325  SphinxSearchTest::testMemoryManager();
326  SphinxSearchTest::testSearcherProcess(fObj);
327  SphinxSearchTest::testFilterSearch(fObj);
328 
329  SphinxManageTest::testIndexRemove(fObj, indexName);
330  SphinxFunctionalObjectTest::testSerializationSearchProcess();
331 }
332 //-----------------------------------------------------------------------------
333 void SphinxFunctionalObjectTest::testSerializationSearchProcess(void)
334 {
335  std::string json =
336  "{\"type\":\"0\",\"data\":\"{\\\"q\\\":\\\"post\\\",\\\"filters\\\":\\\"[{\\\\\\\"type\\\\\\\":0,\\\\\\\"attribute\\\\\\\":\\\\\\\"afflags7ex\\\\\\\",\\\\\\\"values\\\\\\\":[\\\\\\\"141665\\\\\\\"],\\\\\\\"exclude\\\\\\\":\\\\\\\"0\\\\\\\"}]\\\",\\\"max_results\\\":10,\\\"sort_mode\\\":1,\\\"parameters\\\":[{\\\"queryId\\\":\\\"2829799777\\\"},{\\\"JsonType\\\":7},{\\\"sort_by\\\":\\\"weight\\\"}]}\"}";
337  SphinxSearchTest::testSerializationSearchProcess(json);
338 
339  json = "{\"type\":\"0\",\"data\":\"{\\\"q\\\":\\\"post\\\",\\\"filters\\\":\\\"[]\\\",\\\"max_results\\\":10,\\\"sort_mode\\\":1,\\\"parameters\\\":[{\\\"queryId\\\":\\\"2829799777\\\"},{\\\"JsonType\\\":7},{\\\"sort_by\\\":\\\"weight\\\"}]}\"}";
340  SphinxSearchTest::testSerializationSearchProcess(json);
341 
342  json = "{\"type\":\"0\",\"data\":\"{\\\"q\\\":\\\"post\\\",\\\"filters\\\":\\\"[]\\\",\\\"max_results\\\":10,\\\"sort_mode\\\":1,\\\"parameters\\\":[]}\"}";
343  SphinxSearchTest::testSerializationSearchProcess(json);
344 
345  json = "{\"type\":\"0\",\"data\":\"{\\\"query\\\":\\\"post\\\",\\\"filters\\\":\\\"[]\\\",\\\"sort_mode\\\":1,\\\"parameters\\\":[]}\"}";
346  SphinxSearchTest::testSerializationSearchProcess(json);
347 
348  json = "{\"type\":\"0\",\"data\":\"{\\\"filters\\\":\\\"[]\\\",\\\"parameters\\\":[]}\"}";
349  SphinxSearchTest::testSerializationSearchProcess(json);
350 }
351 //-----------------------------------------------------------------------------
352 //-----------------------------------------------------------------------------
353 void SphinxFunctionalObjectTest::initFilters(SphinxFilter::FilterType filterType, std::vector<Poco::SharedPtr<SphinxFilter> >& filters)
354 {
355  filters.clear();
356  Poco::SharedPtr<SphinxFilter> pFilter = new SphinxFilter;
357 
358  if (filterType==SphinxFilter::FilterType::ftRangeFloat)
359  {
360  pFilter->setFilterType(filterType);
361  pFilter->setAttributeName("uid");
362  pFilter->setExcludeValue(SphinxFilter::ExcludeType::etFalse);
363  pFilter->addAttributeValue("1.9");
364  pFilter->addAttributeValue("2.1");
365  filters.push_back(pFilter);
366  }
367  else
368  {
369  pFilter->setFilterType(filterType);
370  pFilter->setAttributeName("afflags4");
371  pFilter->setExcludeValue(SphinxFilter::ExcludeType::etFalse);
372  pFilter->addAttributeValue("4");
373  pFilter->addAttributeValue("8");
374  filters.push_back(pFilter);
375 
376  pFilter = new SphinxFilter;
377  pFilter->setFilterType(filterType);
378  pFilter->setAttributeName("afflags4ex");
379  pFilter->setExcludeValue(SphinxFilter::ExcludeType::etFalse);
380  pFilter->addAttributeValue("16");
381  pFilter->addAttributeValue("64");
382  filters.push_back(pFilter);
383  }
384 }
385 //-----------------------------------------------------------------------------
386 void SphinxFunctionalObjectTest::makeSearchFilter(SphinxFunctionalObject& fObj, SphinxFilter::FilterType filterType, size_t repeatCount)
387 {
388  const std::string query = "@bodys ceo";
389  std::vector<Poco::SharedPtr<SphinxFilter> > filters;
390  Poco::SharedPtr<SphinxResultData> resultData[repeatCount];
391  Poco::SharedPtr<SphinxResultDataSerializator> resultSerializator[repeatCount];
392 
393  for (size_t i=0;i<repeatCount;++i)
394  {
395  resultData[i] = new SphinxResultData();
396  resultSerializator[i] = new SphinxResultDataSerializator(*(resultData[i]));
397  }
398  // initialisation of filter early set values
399  initFilters(filterType, filters);
400  // testing search with filters
401  for (size_t i=0;i<repeatCount;++i)
402  ASSERT_TRUE(resultSerializator[i]->unserialize(SphinxSearchTest::searchWithFilter(fObj, query, filters).getData()));
403 
404  // check results of search
405  for (size_t i=1;i<repeatCount;++i)
406  {
407  const size_t matchInfoCountPrev = resultData[i-1]->getMatchInfoCount();
408  const size_t matchInfoCountNext = resultData[i]->getMatchInfoCount();
409 
410  ASSERT_EQ(matchInfoCountPrev, matchInfoCountNext);
411  for (size_t k=0;k<matchInfoCountPrev;++k)
412  {
413  const SphinxMatchInfo& matchInfoPrev = resultData[i-1]->getMatchInfoItem(k);
414  const SphinxMatchInfo& matchInfoNext = resultData[i]->getMatchInfoItem(k);
415 
416  ASSERT_EQ(matchInfoPrev.getDocId(), matchInfoNext.getDocId());
417  ASSERT_EQ(matchInfoPrev.getWeight(), matchInfoNext.getWeight());
418 
419  const size_t attributesCountPrev = matchInfoPrev.getAttributesCount();
420  const size_t attributesCountNext = matchInfoNext.getAttributesCount();
421 
422  ASSERT_EQ(attributesCountPrev, attributesCountNext);
423  for (size_t j=0;j<attributesCountPrev;++j)
424  {
425  const AttrInfo& attrInfoPrev = matchInfoPrev.getAttrItem(j);
426  const AttrInfo& attrInfoNext = matchInfoNext.getAttrItem(j);
427 
428  ASSERT_EQ(attrInfoPrev.name, attrInfoNext.name);
429  ASSERT_EQ(attrInfoPrev.value, attrInfoNext.value);
430  }
431  }
432  }
433 }
434 //-----------------------------------------------------------------------------
435 void SphinxFunctionalObjectTest::differentFiltersTest(SphinxFunctionalObject& fObj)
436 {
437  std::stringstream dataContent;
438  dataContent << "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<sphinx:docset xmlns:sphinx=\"http://sphinxsearch.com\">\n<sphinx:document id=\"9223372490088244475\">"
439  << "<title><![CDATA[Mars One opens applications for one-way ticket to Mars]]></title>\n"
440  << "<h><![CDATA[<a href='http://cdni.wired.co.uk/620x413/k_n/Mars_7.jpg' target=_blank><img src='http://cdni.wired.co.uk/620x413/k_n/Mars_7.jpg' title='size: 0 width: 0 height: 0'></a>one-way ticket\nMars\n∧one∧way∧ticket\n∧Mars http://wired.co.uk/news/archive/2013-04/25/mars-one-applications ]]></h>\n"
441  << "<alt><![CDATA[2,8,32,128]]></alt>\n"
442  << "<bodys><![CDATA[It's time, if you're so inclined, to plunk down a reservation\nfor a one-way ticket to Mars. The privately funded Mars One\n"
443  << "foundation recently opened up applications for astronauts to take a\njourney to the Red Planet in 2022/23. Return trips, the organisers\n"
444  << "said, just aren't feasible with the technology we have.\nFrom a wide field of applicants, organisers said two women and\ntwo men will be selected to make the journey. The technology that\n"
445  << "takes them there has been tested before, and the potential for a\nworldwide audience will be there given four billion people will\nhave the internet by then, said Mars One CEO Bas Lansdorp.\n"
446  << "By: Elizabeth Howell, SEN.com, Edited by: Kadhim ShubberContinue reading...]]></bodys>\n"
447  << "<bodyn><![CDATA[10101010]]></bodyn>\n<meta><![CDATA[1191]]></meta>\n<ref><![CDATA[93]]></ref>\n"
448  << "<url><![CDATA[http://api.snatz.com/index.php?app=hceapi&event=feeds&time=new&feed_id=11250311753507343583&item_id=14946343666313105260-13-04]]></url>\n"
449  << "<cdate>1373619775</cdate>\n<mmedia>0</mmedia>\n<bseo>0</bseo>\n<uid>2</uid>\n<sid>1</sid>\n<lang>255</lang>\n"
450  << "<afflags0>2,8,32,128</afflags0>\n<afflags1>24</afflags1>\n<afflags2>1</afflags2>\n<afflags3>2,8,32,128</afflags3>\n"
451  << "<afflags4>2,8,32,128</afflags4>\n<afflags5>2,8,32,128</afflags5>\n<afflags6>2,8,32,128</afflags6>\n<afflags7>2,8,32,128</afflags7>\n\n"
452  << "<afflags0ex>1,4,16,64</afflags0ex>\n<afflags1ex>1,4,16,64</afflags1ex>\n<afflags2ex>1,4,16,64</afflags2ex>\n<afflags3ex>1,4,16,64</afflags3ex>\n"
453  << "<afflags4ex>1,4,16,64</afflags4ex>\n<afflags5ex>1,4,16,64</afflags5ex>\n<afflags6ex>1,4,16,64</afflags6ex>\n<afflags7ex>1,4,16,64</afflags7ex>\n"
454  << "</sphinx:document>\n</sphinx:docset>\n";
455 
456  const std::string indexName = "i003";
457 
458  // create index
459  SphinxManageTest::testIndexCreate(fObj, indexName);
460  SphinxManageTest::testIndexCheck(fObj, indexName, true);
461  std::vector<std::string> branches = {"branch_0005"};
462  for (size_t i=0;i<branches.size();++i)
463  SphinxManageTest::testIndexStoreDataFile(fObj, indexName, branches[i], dataContent.str());
464  SphinxManageTest::testIndexRebuild(fObj, indexName, branches);
465  SphinxManageTest::testIndexMerge(fObj, indexName, branches);
466  SphinxManageTest::testIndexStart(fObj, indexName);
467 
468 // fObj.setLogStream(std::cout);
469 
470  const size_t testRepeat = 3;
471 
472  SphinxFunctionalObjectTest::makeSearchFilter(fObj, SphinxFilter::FilterType::ftSimple, testRepeat);
473  ASSERT_FALSE(fObj.isError());
474  printSuccess("Search Filter ftSimple");
475 
476  SphinxFunctionalObjectTest::makeSearchFilter(fObj, SphinxFilter::FilterType::ftRangeFloat, testRepeat);
477  ASSERT_FALSE(fObj.isError());
478  printSuccess("Search Filter ftRangeFloat");
479 
480  SphinxFunctionalObjectTest::makeSearchFilter(fObj, SphinxFilter::FilterType::ftRangeInteger, testRepeat);
481  ASSERT_FALSE(fObj.isError());
482  printSuccess("Search Filter ftRangeInteger");
483 
484  SphinxFunctionalObjectTest::makeSearchFilter(fObj, SphinxFilter::FilterType::ftSimpleMulti, testRepeat);
485  ASSERT_FALSE(fObj.isError());
486  printSuccess("Search Filter ftSimpleMulti");
487 
488  SphinxManageTest::testIndexRemove(fObj, indexName);
489  ASSERT_FALSE(fObj.isError());
490  printSuccess("Test search with different filters");
491 }
492 //-----------------------------------------------------------------------------
493 //-----------------------------------------------------------------------------
494 bool SphinxFunctionalObjectTest::isUse64Bit(SphinxFunctionalObject& fObj)
495 {
496  bool result = false;
497  const unsigned long long docId = 9223372490088244475ULL;
498 
499  std::stringstream dataContent;
500  dataContent << "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n<sphinx:docset xmlns:sphinx=\"http://sphinxsearch.com\">\n<sphinx:document id=\""<<docId<<"\">"
501  << "<title><![CDATA[Mars One opens applications for one-way ticket to Mars]]></title>\n"
502  << "<h><![CDATA[<a href='http://cdni.wired.co.uk/620x413/k_n/Mars_7.jpg' target=_blank><img src='http://cdni.wired.co.uk/620x413/k_n/Mars_7.jpg' title='size: 0 width: 0 height: 0'></a>one-way ticket\nMars\n∧one∧way∧ticket\n∧Mars http://wired.co.uk/news/archive/2013-04/25/mars-one-applications ]]></h>\n"
503  << "<alt><![CDATA[2,8,32,128]]></alt>\n"
504  << "<bodys><![CDATA[It's time, if you're so inclined, to plunk down a reservation\nfor a one-way ticket to Mars. The privately funded Mars One\n"
505  << "foundation recently opened up applications for astronauts to take a\njourney to the Red Planet in 2022/23. Return trips, the organisers\n"
506  << "said, just aren't feasible with the technology we have.\nFrom a wide field of applicants, organisers said two women and\ntwo men will be selected to make the journey. The technology that\n"
507  << "takes them there has been tested before, and the potential for a\nworldwide audience will be there given four billion people will\nhave the internet by then, said Mars One CEO Bas Lansdorp.\n"
508  << "By: Elizabeth Howell, SEN.com, Edited by: Kadhim ShubberContinue reading...]]></bodys>\n"
509  << "<bodyn><![CDATA[10101010]]></bodyn>\n<meta><![CDATA[1191]]></meta>\n<ref><![CDATA[93]]></ref>\n"
510  << "<url><![CDATA[http://api.snatz.com/index.php?app=hceapi&event=feeds&time=new&feed_id=11250311753507343583&item_id=14946343666313105260-13-04]]></url>\n"
511  << "<cdate>1373619775</cdate>\n<mmedia>0</mmedia>\n<bseo>0</bseo>\n<uid>2</uid>\n<sid>1</sid>\n<lang>255</lang>\n"
512  << "<afflags0>2,8,32,128</afflags0>\n<afflags1>24</afflags1>\n<afflags2>1</afflags2>\n<afflags3>2,8,32,128</afflags3>\n"
513  << "<afflags4>2,8,32,128</afflags4>\n<afflags5>2,8,32,128</afflags5>\n<afflags6>2,8,32,128</afflags6>\n<afflags7>2,8,32,128</afflags7>\n\n"
514  << "<afflags0ex>1,4,16,64</afflags0ex>\n<afflags1ex>1,4,16,64</afflags1ex>\n<afflags2ex>1,4,16,64</afflags2ex>\n<afflags3ex>1,4,16,64</afflags3ex>\n"
515  << "<afflags4ex>1,4,16,64</afflags4ex>\n<afflags5ex>1,4,16,64</afflags5ex>\n<afflags6ex>1,4,16,64</afflags6ex>\n<afflags7ex>1,4,16,64</afflags7ex>\n"
516  << "</sphinx:document>\n</sphinx:docset>\n";
517 
518  const std::string query = "Mars";
519  const std::string indexName = "i003";
520 
521  // create index
522  SphinxManageTest::testIndexCreate(fObj, indexName);
523  SphinxManageTest::testIndexCheck(fObj, indexName, true);
524 
525  std::vector<std::string> branches = {"branch_"+indexName};
526  for (size_t i=0;i<branches.size();++i)
527  SphinxManageTest::testIndexStoreDataFile(fObj, indexName, branches[i], dataContent.str());
528  SphinxManageTest::testIndexRebuild(fObj, indexName, branches);
529  SphinxManageTest::testIndexMerge(fObj, indexName, branches);
530  SphinxManageTest::testIndexStart(fObj, indexName);
531 
532  // test search document
533  std::string json;
534  SphinxInputJsonMessageSearch inputJsonMessageSearch;
535  inputJsonMessageSearch.setQueryString(query);
536  inputJsonMessageSearch.addQueryParameters("queryId", "1");
537  inputJsonMessageSearch.addQueryParameters("jsonType", "1");
538  inputJsonMessageSearch.addQueryParameters("max_results", "100");
539 
540  bool ret = inputJsonMessageSearch.serialize(json);
541  fObj.log(HCE::LoggerStream::PRIO_INFORMATION) << "SphinxInputJsonMessageSearch serialize => " << std::boolalpha << ret << std::endl;
542  assert(ret);
543  printSuccess("SphinxInputJsonMessageSearch serialize");
544 
545  SphinxInputJsonMessage inputJsonMessage;
546  inputJsonMessage.setType(SphinxInputJsonMessage::MessageType::mtSearch);
547  inputJsonMessage.setData(json);
548 
549  SphinxOutputJsonMessage outputMessage = fObj.Process(inputJsonMessage);
550  if (!fObj.getErrorMsg().empty())
551  fObj.log(HCE::LoggerStream::PRIO_INFORMATION) << "fObj.getErrorMsg() = " << fObj.getErrorMsg() << std::endl;
552  if (fObj.getErrorCode()!=NO_ERROR)
553  fObj.log(HCE::LoggerStream::PRIO_INFORMATION) << "fObj.getErrorCode() = " << fObj.getErrorCode() << std::endl;
554  if (outputMessage.isError())
555  fObj.log(HCE::LoggerStream::PRIO_INFORMATION) << "outputMessage.isError() = " << outputMessage.isError() << std::endl;
556 
557  assert(!fObj.isError());
558  assert(fObj.getErrorCode()==NO_ERROR);
559  assert(fObj.getErrorMsg().empty());
560 
561  SphinxResultData resultData;
562  SphinxResultDataSerializator resultSerializator(resultData);
563 
564  ret = resultSerializator.unserialize(outputMessage.getData());
565  fObj.log(HCE::LoggerStream::PRIO_INFORMATION) << "SphinxResultData unserialize => " << std::boolalpha << ret << std::endl;
566  assert(ret);
567  printSuccess("SphinxResultData unserialize");
568 
569  assert(resultData.getMatchInfoCount()==1);
570  unsigned long long retDocId = resultData.getMatchInfoItem(0).getDocId();
571 
572  result = (retDocId==docId);
573  std::cout << "retDocId = " << retDocId << std::endl;
574  std::cout << "docId = " << docId << std::endl;
575 
576  fObj.log(HCE::LoggerStream::PRIO_INFORMATION) << "Use " << ((result)?"64":"32") << " Bit for document ID" << std::endl;
577 
578  SphinxManageTest::testIndexRemove(fObj, indexName);
579  printSuccess("Test Use 32 or 64 Bit");
580 
581  return result;
582 }
583 //-----------------------------------------------------------------------------
584 float SphinxFunctionalObjectTest::testPerfomance(const std::string& homeDir, size_t indexCount, const std::string& dataFileName, std::vector<std::string>& queries, bool multiIndex)
585 {
586  float fps = 0.0;
587  std::vector<std::string> indexNames;
588  // create indexes
589  SphinxMultiIndexTest::testCreateIndexes(homeDir, indexCount, dataFileName, indexNames);
590  // search use already created indexes
591 // SphinxMultiIndexTest::makeIndexNames(indexCount, indexNames);
592 
593  if (multiIndex)
594  {
595  fps = SphinxMultiIndexTest::testSearchIndexes(indexNames, queries);
596  }
597  else
598  {
599  std::string indexName = indexNames.front();
600  fps = SphinxMultiIndexTest::testSearchIndexes(homeDir, indexNames, queries, indexName);
601  }
602  // remove indexes
603  SphinxMultiIndexTest::testRemoveIndexes(homeDir, indexNames);
604  return fps;
605 }
606 //-----------------------------------------------------------------------------
607 void SphinxFunctionalObjectTest::testPerfomance(const std::string& homeDir, const std::string& dataFileName, const size_t indexCount)
608 {
609  std::vector<std::string> queries=
610  {"seo", "ISO", "with", "LibreOffice", "is", "free", "and", "open", "source", "office", "suite", "for", "Linux", "Mac", "Windows", "computers", "Forest", "Key", "Seattle", "MGM"};
611 
612  float multiIndexFps = SphinxFunctionalObjectTest::testPerfomance(homeDir, indexCount, dataFileName, queries, true);
613  float singleIndexFps = SphinxFunctionalObjectTest::testPerfomance(homeDir, indexCount, dataFileName, queries, false);
614 
615  std::stringstream outMsg;
616  outMsg << "Single index search perfomance: " << singleIndexFps << " fps." << std::endl;
617  const size_t length = outMsg.str().length();
618  std::cout << std::string(length, '=') << "\n";
619  std::cout << "Multi index search perfomance: " << multiIndexFps << " fps." << std::endl;
620  std::cout << outMsg.str() << std::string(length, '=') << std::endl;
621 }
622 //-----------------------------------------------------------------------------
623 //-----------------------------------------------------------------------------
624 void SphinxFunctionalObjectTest::createBigIndex(const std::string& homeDir, const std::string& sourceTestDir, const std::string& indexName)
625 {
626  SphinxFunctionalObject fObj("node", 2005, homeDir, "");
627  std::string errorMsg;
628  std::vector<std::string> dataFiles;
629  bool ret = SphinxStressTest::makeDataFilesList(sourceTestDir, dataFiles, errorMsg);
630  if (!ret)
631  std::cout << "Error: " << errorMsg << std::endl;
632  ASSERT_TRUE(ret);
633 
634  for (const auto& name: dataFiles)
635  std::cout << name << std::endl;
636 
637  SphinxManageTest::testIndexCreate(fObj, indexName);
638  SphinxManageTest::testIndexCheck(fObj, indexName, true);
639 
640  ret = SphinxStressTest::moveDataFiles(sourceTestDir, fObj.getDataDir()+"/"+indexName+"/"+sphinx_admin_command_const::dataDir, dataFiles, errorMsg);
641  if (!ret)
642  std::cout << "Error: " << errorMsg << std::endl;
643  ASSERT_TRUE(ret);
644 
645  std::vector<std::string> branches = {"*"};
646  SphinxManageTest::testIndexRebuild(fObj, indexName, branches);
647  SphinxManageTest::testIndexMerge(fObj, indexName, branches);
648 }
649 //-----------------------------------------------------------------------------
650 void SphinxFunctionalObjectTest::testStressGetBigResult(SphinxFunctionalObject& fObj, const std::string& sourceTestDir,
651  const std::string& indexName, bool needMakeIndex, bool needRemoveIndex)
652 {
653  if (needMakeIndex)
654  SphinxFunctionalObjectTest::createBigIndex(fObj.getHomeDir(), sourceTestDir, indexName);
655 
656  std::string paramValue;
657  SphinxManageTest::testIndexGetConfigVar(fObj, indexName, "", "searchd", "max_matches", paramValue);
658  SphinxManageTest::testIndexSetConfigVar(fObj, indexName, "", "searchd", "max_matches", "100000");
659 
660  const std::string query = "и";
661 
662  SphinxManageTest::testIndexStart(fObj, indexName);
663 
664  size_t maxReturnedDocs = 0;
665  const size_t maxCount = 10000;
666  for (size_t i=1000;i<maxCount;i+=1000)
667  {
668  printMessage("Try get "+std::to_string(i)+" documents");
669  SphinxSearchTest::testSearcherProcess(fObj, query, i);
670  if (fObj.isError())
671  {
672  maxReturnedDocs = i-1000;
673  fObj.resetError();
674  for (size_t j=maxReturnedDocs+100;j<maxCount;j+=100)
675  {
676  printMessage("Try get "+std::to_string(j)+" documents");
677  SphinxSearchTest::testSearcherProcess(fObj, query, j);
678  if (fObj.isError())
679  {
680  fObj.resetError();
681  break;
682  }
683  else
684  maxReturnedDocs = j;
685  }
686  break;
687  }
688  }
689 
690  SphinxManageTest::testIndexStop(fObj, indexName);
691  SphinxManageTest::testIndexSetConfigVar(fObj, indexName, "", "searchd", "max_matches", paramValue);
692 
693  if (needRemoveIndex)
694  SphinxManageTest::testIndexRemove(fObj, indexName);
695 
696  printMessage("MAX RETURNED DOCS: "+std::to_string(maxReturnedDocs));
697  printSuccess("Stress test get max result");
698 }
699 //-----------------------------------------------------------------------------
700 void SphinxFunctionalObjectTest::testGetResultUseOffset(SphinxFunctionalObject& fObj, const std::string& sourceTestDir,
701  const std::string& indexName, bool needMakeIndex, bool needRemoveIndex)
702 {
703  if (needMakeIndex)
704  SphinxFunctionalObjectTest::createBigIndex(fObj.getHomeDir(), sourceTestDir, indexName);
705 
706  std::string paramValue;
707  SphinxManageTest::testIndexGetConfigVar(fObj, indexName, "", "searchd", "max_matches", paramValue);
708  SphinxManageTest::testIndexSetConfigVar(fObj, indexName, "", "searchd", "max_matches", "500000"); // size must be over then documents count in index
709 
710  const std::string query = ""; // must be empty
711  const unsigned int maxResults = 1000;
712  const unsigned int offset = 999;
713  SphinxManageTest::testIndexStart(fObj, indexName);
714  SphinxSearchTest::testGetResultUseOffset(fObj, query, maxResults, offset);
715  SphinxManageTest::testIndexStop(fObj, indexName);
716  SphinxManageTest::testIndexSetConfigVar(fObj, indexName, "", "searchd", "max_matches", paramValue);
717 
718  if (needRemoveIndex)
719  SphinxManageTest::testIndexRemove(fObj, indexName);
720 
721  printSuccess("Stress test get result use offset");
722 }
723 //-----------------------------------------------------------------------------
724 //-----------------------------------------------------------------------------
725 } // namespace tests
726 } // namespace sphinx
727 } // namespace HCE