hce-node application  1.4.3
HCE Hierarchical Cluster Engine node application
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros
SphinxResultDataSerializator.cpp
Go to the documentation of this file.
1 /*
2  * SphinxResultDataSerializator.cpp
3  *
4  * Created on: Oct 30, 2013
5  * Author: alexander
6  */
7 
8 #include <Poco/JSON/Object.h>
9 #include <Poco/JSON/Array.h>
10 #include <Poco/JSON/JSON.h>
11 #include <Poco/JSON/Stringifier.h>
12 #include <Poco/JSON/Parser.h>
13 #include <Poco/JSON/JSONException.h>
14 #include <Poco/Dynamic/Var.h>
15 #include <sstream>
16 #include <iomanip>
17 #include <sys/time.h>
18 
19 #include "SphinxInputJsonMessage.hpp"
20 #include "EncodeDecodeBase64.hpp"
21 #include "SphinxError.hpp"
22 
23 #include "SphinxResultDataSerializator.hpp"
24 
25 namespace HCE
26 {
27 namespace sphinx
28 {
29 //-----------------------------------------------------------------------------
30 std::string SphinxWeightPacker::pack(const std::string& data, unsigned int minNumberFieldsPacking)
31 {
32  std::ostringstream ostr;
33 
34  if (minNumberFieldsPacking<=fieldsCount)
35  {
36  const size_t count = data.length()/fieldWidth;
37 
38  if (!fieldsCount)
39  ++fieldsCount;
40 
41  for (size_t i=0;i<fieldsCount;++i)
42  {
43  std::string hexString;
44  if (i<count)
45  hexString = data.substr(i*fieldWidth, fieldWidth);
46  size_t pos = hexString.find_first_not_of('0');
47  ostr << delimiter;
48  if (pos!=std::string::npos)
49  ostr << hexString.substr(pos);
50  }
51  }
52  else
53  ostr << data;
54 
55  return ostr.str();
56 }
57 //-----------------------------------------------------------------------------
58 std::string SphinxWeightPacker::unpack(const std::string& data)
59 {
60  std::ostringstream ostr;
61  size_t count = 0;
62  for (size_t i=0;i<data.size();++i)
63  if (data[i]==delimiter)
64  ++count;
65 
66  if (count)
67  {
68  std::istringstream istr(data);
69  std::string line;
70  bool begin = false;
71  while(getline(istr, line, delimiter))
72  {
73  if (begin)
74  {
75  ostr << std::setfill('0') << std::setw(fieldWidth) << line;
76  --count;
77  }
78  else
79  begin = true;
80  }
81  if (count)
82  ostr << std::setfill('0') << std::setw(fieldWidth) << "";
83  }
84  else
85  ostr << data;
86 
87  return ostr.str();
88 }
89 //-----------------------------------------------------------------------------
90 //-----------------------------------------------------------------------------
91 SphinxResultDataSerializator::SphinxResultDataSerializator(SphinxResultData& resultData_)
92 : inherited(), resultData(resultData_), jsonType(FILL_MATCH_INFO | FILL_REQUEST_INFO | FILL_ATTRIBUTES | FILL_WORD_INFO)
93 {
94 }
95 //-----------------------------------------------------------------------------
96 SphinxResultDataSerializator::SphinxResultDataSerializator(SphinxResultData& resultData_, const std::string& json)
97 : inherited(), resultData(resultData_), jsonType(FILL_MATCH_INFO | FILL_REQUEST_INFO | FILL_ATTRIBUTES | FILL_WORD_INFO)
98 {
99  unserialize(json);
100 }
101 //-----------------------------------------------------------------------------
102 SphinxResultDataSerializator::SphinxResultDataSerializator(SphinxResultData& resultData_, SphinxResultDataSerializator::JsonType jsonType_)
103 : inherited(), resultData(resultData_), jsonType(jsonType_)
104 {
105 }
106 //-----------------------------------------------------------------------------
107 SphinxResultDataSerializator::~SphinxResultDataSerializator(void)
108 {
109 }
110 //-----------------------------------------------------------------------------
111 bool SphinxResultDataSerializator::serialize(SphinxResultDataSerializator::JsonType jsonType, std::string& json, unsigned int fieldCount, unsigned int minNumberFieldsPacking)
112 {
113  errorMsg.clear();
114  errorCode = NO_ERROR;
115  SphinxWeightPacker packer(fieldCount);
116  try
117  {
118  Poco::JSON::Object::Ptr obj = new Poco::JSON::Object();
119 
120  // make json objects for matches info
121  Poco::JSON::Array::Ptr pMatches = new Poco::JSON::Array();
122  if ((jsonType & FILL_MATCH_INFO)!=0)
123  {
124  const size_t matchInfoCount = resultData.getMatchInfoCount();
125  for (size_t i=0;i<matchInfoCount;++i)
126  {
127  Poco::JSON::Object::Ptr pMatchElem = new Poco::JSON::Object();
128 
129  std::string docId = std::to_string(resultData.getMatchInfoItem(i).getDocId());
130  pMatchElem->set(search_result_json_const::docId, docId);
131  pMatchElem->set(search_result_json_const::weight, packer.pack(resultData.getMatchInfoItem(i).getWeight(), minNumberFieldsPacking));
132 
133  Poco::JSON::Array::Ptr pAttributes = new Poco::JSON::Array();
134  if ((jsonType & FILL_ATTRIBUTES)!=0)
135  {
136  const size_t attributesCount = resultData.getMatchInfoItem(i).getAttributesCount();
137  for (size_t j=0;j<attributesCount;++j)
138  {
139  Poco::JSON::Object::Ptr pAttrElem = new Poco::JSON::Object();
140  pAttrElem->set(resultData.getMatchInfoItem(i).getAttrItem(j).name, resultData.getMatchInfoItem(i).getAttrItem(j).value);
141  pAttributes->add(pAttrElem);
142  }
143  }
144  pMatchElem->set(search_result_json_const::attrArray, pAttributes);
145  pMatches->add(pMatchElem);
146  }
147  }
148  obj->set(search_result_json_const::matchInfo, pMatches);
149 
150  // make json objects for request info
151  Poco::JSON::Array::Ptr pRequests = new Poco::JSON::Array();
152 
153  if ((jsonType & FILL_REQUEST_INFO)!=0)
154  {
155  const size_t requestInfoCount = resultData.getRequestInfoCount();
156  for (size_t i=0;i<requestInfoCount;++i)
157  {
158  Poco::JSON::Object::Ptr pRequestElem = new Poco::JSON::Object();
159 
160  pRequestElem->set(search_result_json_const::nodeName, resultData.getRequestInfoItem(i).getNodeName());
161  pRequestElem->set(search_result_json_const::query, resultData.getRequestInfoItem(i).getQuery());
162  pRequestElem->set(search_result_json_const::queryId, resultData.getRequestInfoItem(i).getQueryId());
163  pRequestElem->set(search_result_json_const::maxResultsNumber, resultData.getRequestInfoItem(i).getMaxResultsNumber());
164  pRequestElem->set(search_result_json_const::orderBy, resultData.getRequestInfoItem(i).getOrderBy());
165  pRequestElem->set(search_result_json_const::total, resultData.getRequestInfoItem(i).getTotal());
166  pRequestElem->set(search_result_json_const::totalFound, resultData.getRequestInfoItem(i).getTotalFound());
167  pRequestElem->set(search_result_json_const::timeMsec, resultData.getRequestInfoItem(i).getTimeMsec());
168 
169  Poco::JSON::Array::Ptr pWords = new Poco::JSON::Array();
170  if ((jsonType & FILL_WORD_INFO)!=0)
171  {
172  const size_t wordsCount = resultData.getRequestInfoItem(i).getWordsCount();
173  for (size_t j=0;j<wordsCount;++j)
174  {
175  Poco::JSON::Object::Ptr pWordElem = new Poco::JSON::Object();
176 
177  pWordElem->set(search_result_json_const::word, resultData.getRequestInfoItem(i).getWordItem(j).word);
178  pWordElem->set(search_result_json_const::hits, resultData.getRequestInfoItem(i).getWordItem(j).hits);
179  pWordElem->set(search_result_json_const::docs, resultData.getRequestInfoItem(i).getWordItem(j).docs);
180  pWords->add(pWordElem);
181  }
182  }
183  pRequestElem->set(search_result_json_const::wordArray, pWords);
184  pRequests->add(pRequestElem);
185  }
186  }
187  obj->set(search_result_json_const::requestInfo, pRequests);
188 
189  std::stringstream ostr;
190  Poco::JSON::Stringifier::stringify(obj, ostr);
191  json = ostr.str();
192  _isError = false;
193  }
194  catch(std::exception& e)
195  {
196  _isError = true;
197  errorMsg = e.what();
198  errorCode = ERROR_SERIALIZE;
199  }
200  return !_isError;
201 }
202 //-----------------------------------------------------------------------------
203 bool SphinxResultDataSerializator::serialize(std::string& json, unsigned int fieldCount, unsigned int minNumberFieldsPacking)
204 {
205  return serialize(jsonType, json, fieldCount, minNumberFieldsPacking);
206 }
207 //-----------------------------------------------------------------------------
208 bool SphinxResultDataSerializator::serialize(std::string& json)
209 {
210  return serialize(json, 0, 0);
211 }
212 //-----------------------------------------------------------------------------
213 bool SphinxResultDataSerializator::unserialize(const std::string& json)
214 {
215  resultData.clear();
216  errorMsg.clear();
217  errorCode = NO_ERROR;
218  SphinxWeightPacker packer;
219  try
220  {
221  Poco::JSON::Parser parser;
222  Poco::Dynamic::Var res = parser.parse(json);
223  Poco::JSON::Object::Ptr obj = res.extract<Poco::JSON::Object::Ptr>();
224  Poco::Dynamic::Var tmp;
225 
226  // extract from json objects of matches info
227  Poco::JSON::Array::Ptr pMatches = obj->getArray(search_result_json_const::matchInfo);
228  if (!pMatches.isNull())
229  {
230  const size_t matchesCount = pMatches->size();
231  for (size_t i=0;i<matchesCount;++i)
232  {
233  Poco::JSON::Object::Ptr pMatchElem = pMatches->getObject(i);
234  if (!pMatchElem.isNull())
235  {
236  SphinxMatchInfo matchInfo;
237  tmp = pMatchElem->get(search_result_json_const::docId);
238  if (tmp.isString())
239  matchInfo.setDocId(std::stoull(tmp.convert<std::string>()));
240 
241  tmp = pMatchElem->get(search_result_json_const::weight);
242  if (tmp.isString())
243  matchInfo.setWeightFromHexString(packer.unpack(tmp.convert<std::string>()));
244 
245  Poco::JSON::Array::Ptr pAttributes = pMatchElem->getArray(search_result_json_const::attrArray);
246  if (!pAttributes.isNull())
247  {
248  const size_t attributesCount = pAttributes->size();
249  for (size_t j=0;j<attributesCount;++j)
250  {
251  Poco::JSON::Object::Ptr pAttrElem = pAttributes->getObject(j);
252  if (!pAttrElem.isNull())
253  {
254  std::vector<std::string> attrNames;
255  pAttrElem->getNames(attrNames);
256  const size_t attrNamesCount = attrNames.size();
257  for (size_t k=0;k<attrNamesCount;++k)
258  {
259  tmp = pAttrElem->get(attrNames[k]);
260  if (tmp.isString())
261  matchInfo.addAttrInfo(attrNames[k], tmp.convert<std::string>());
262  }
263  }
264  }
265  }
266  resultData.addMatchInfo(std::forward<SphinxMatchInfo>(matchInfo));
267  }
268  }
269  }
270  // extract from json objects of request info
271  Poco::JSON::Array::Ptr pRequests = obj->getArray(search_result_json_const::requestInfo);
272  if (!pRequests.isNull())
273  {
274  const size_t requestsCount = pRequests->size();
275  for (size_t i=0;i<requestsCount;++i)
276  {
277  SphinxRequestInfo requestInfo;
278  Poco::JSON::Object::Ptr pRequestElem = pRequests->getObject(i);
279 
280  tmp = pRequestElem->get(search_result_json_const::nodeName);
281  if (tmp.isString())
282  requestInfo.setNodeName(tmp.convert<std::string>());
283 
284  tmp = pRequestElem->get(search_result_json_const::query);
285  if (tmp.isString())
286  requestInfo.setQuery(tmp.convert<std::string>());
287 
288  tmp = pRequestElem->get(search_result_json_const::queryId);
289  requestInfo.setQueryId(convertVarToNumeric<unsigned int>(tmp, 0));
290 
291  tmp = pRequestElem->get(search_result_json_const::maxResultsNumber);
292  requestInfo.setMaxResultsNumber(convertVarToNumeric<unsigned int>(tmp, input_json_message_const::defaultMaxResultsNumber));
293 
294  tmp = pRequestElem->get(search_result_json_const::orderBy);
295  requestInfo.setOrderBy(convertVarToNumeric<unsigned int>(tmp, 0));
296 
297  tmp = pRequestElem->get(search_result_json_const::total);
298  requestInfo.setTotal(convertVarToNumeric<unsigned int>(tmp, 0));
299 
300  tmp = pRequestElem->get(search_result_json_const::totalFound);
301  requestInfo.setTotalFound(convertVarToNumeric<unsigned int>(tmp, 0));
302 
303  tmp = pRequestElem->get(search_result_json_const::timeMsec);
304  requestInfo.setTimeMsec(convertVarToNumeric<unsigned int>(tmp, 0));
305 
306  Poco::JSON::Array::Ptr pWords = pRequestElem->getArray(search_result_json_const::wordArray);
307  if (!pWords.isNull())
308  {
309  const size_t wordsCount = pWords->size();
310  for (size_t j=0;j<wordsCount;++j)
311  {
312  Poco::JSON::Object::Ptr pWordElem = pWords->getObject(j);
313  if (!pWordElem.isNull())
314  {
315  WordInfo wordInfo;
316  tmp = pWordElem->get(search_result_json_const::word);
317  if (tmp.isString())
318  tmp.convert(wordInfo.word);
319 
320  tmp = pWordElem->get(search_result_json_const::hits);
321  wordInfo.hits = convertVarToNumeric<unsigned int>(tmp, 0);
322 
323  tmp = pWordElem->get(search_result_json_const::docs);
324  wordInfo.docs = convertVarToNumeric<unsigned int>(tmp, 0);
325 
326  requestInfo.addWordInfo(std::forward<WordInfo>(wordInfo));
327  }
328  }
329  }
330  resultData.addRequestInfo(std::forward<SphinxRequestInfo>(requestInfo));
331  }
332  }
333  _isError = false;
334  }
335  catch(Poco::JSON::JSONException& e)
336  {
337  _isError = true;
338  errorMsg = e.displayText();
339  errorCode = ERROR_UNSERIALIZE;
340  }
341  catch(std::exception& e)
342  {
343  _isError = true;
344  errorMsg = e.what();
345  errorCode = ERROR_UNSERIALIZE;
346  }
347  return !_isError;
348 }
349 //-----------------------------------------------------------------------------
350 //-----------------------------------------------------------------------------
351 SphinxDefaultJSON::SphinxDefaultJSON(unsigned int matchesCount, unsigned int attributesCount, unsigned int maxResultNumber)
352 : json(""), errorMsg(""), errorCode(NO_ERROR), _isError(false)
353 {
354  makeJSON(matchesCount, attributesCount, maxResultNumber);
355 }
356 //-----------------------------------------------------------------------------
357 bool SphinxDefaultJSON::fillResultData(SphinxResultData& resultData, unsigned int matchesCount, unsigned int attributesCount, unsigned int maxResultNumber)
358 {
359  try
360  {
361  timeval now;
362  gettimeofday(&now, 0);
363 
364  const size_t maxMatchesCount = now.tv_usec + matchesCount;
365 
366  for (size_t i=now.tv_usec;i<maxMatchesCount;++i)
367  {
368  SphinxMatchInfo matchInfo;
369  matchInfo.setDocId(i);
370  matchInfo.setWeightFromNumeric(i);
371 
372  for (size_t j=1;j<=attributesCount;++j)
373  matchInfo.addAttrInfo("attr_"+std::to_string(j), "value_"+std::to_string(j));
374 
375  resultData.addMatchInfo(std::forward<SphinxMatchInfo>(matchInfo));
376  }
377  SphinxRequestInfo requestInfo;
378  requestInfo.setNodeName("node");
379  requestInfo.setQuery("query");
380  requestInfo.setQueryId(matchesCount);
381  requestInfo.setMaxResultsNumber(maxResultNumber);
382  requestInfo.setOrderBy(attributesCount);
383  requestInfo.setTotal(matchesCount);
384  requestInfo.setTotalFound(matchesCount);
385  requestInfo.addWordInfo("word1", 1 , 2);
386  requestInfo.addWordInfo("word2", 2 , 3);
387  requestInfo.addWordInfo("word3", 3 , 4);
388  resultData.addRequestInfo(std::forward<SphinxRequestInfo>(requestInfo));
389 
390  _isError = false;
391  }
392  catch(std::exception& e)
393  {
394  _isError = true;
395  errorMsg = e.what();
396  errorCode = ERROR_MAKE_DEFAULT_JSON;
397  }
398  return !_isError;
399 }
400 //-----------------------------------------------------------------------------
401 bool SphinxDefaultJSON::makeJSON(unsigned int matchesCount, unsigned int attributesCount, unsigned int maxResultNumber)
402 {
403  if (!maxResultNumber)
404  maxResultNumber = input_json_message_const::defaultMaxResultsNumber;
405 
406  json.clear();
407  errorMsg.clear();
408  errorCode = NO_ERROR;
409  _isError = false;
410  SphinxResultData resultData;
411  SphinxResultDataSerializator resultSerializator(resultData);
412 
413  bool success = true;
414  if (!matchesCount)
415  success = resultSerializator.serialize(NO_FILL, json);
416  else
417  {
418  if (fillResultData(resultData, matchesCount, attributesCount, maxResultNumber))
419  success = resultSerializator.serialize((FILL_MATCH_INFO | FILL_REQUEST_INFO | FILL_ATTRIBUTES | FILL_WORD_INFO), json);
420  }
421 
422  if (!success)
423  {
424  _isError = resultSerializator.isError();
425  errorMsg = resultSerializator.getErrorMsg();
426  errorCode = resultSerializator.getErrorCode();
427  }
428  return !_isError;
429 }
430 //-----------------------------------------------------------------------------
431 std::ostream& operator<<(std::ostream& os, const SphinxDefaultJSON& defaultJson)
432 {
433  return os << defaultJson.getJSON();
434 }
435 //-----------------------------------------------------------------------------
436 //-----------------------------------------------------------------------------
437 } // namespace sphinx
438 } // namespace HCE