HighlightingAlgorithmRefine.cpp

Go to the documentation of this file.

#include "Defs.hpp"
#include "HighlightingAlgorithmRefine.hpp"

// cppcheck-suppress unusedFunction  
void HighlightingAlgorithmRefine::setWordPositionMap(std::map<unsigned int, std::pair<unsigned long long, WordPos> > &_wordPositionMap)
{
  wordPositionMap = _wordPositionMap;
}

void HighlightingAlgorithmRefine::fillWordOffsetMap(Poco::SharedPtr<HCE::OutDataRefine> outData, SearchType searchType,
                                  std::map<unsigned long long, std::vector<WordPos> > &wordMapPtr, std::map<unsigned int, WordPos> &wordOffsetMap)
{
  if(!outData.isNull())
  {
    CRC64 crc64;
    typename std::map<unsigned long long, std::vector<WordPos> >::iterator it;
    unsigned long long key = 0llu;
    std::vector<HCE::CWords> cwords = outData->getCWords();
    for(unsigned int i = 0; i < cwords.size(); i++)
    {
      if(cwords[i].getWordType() != HCE::WORD_TYPE::WORD && cwords[i].getWordType() != HCE::WORD_TYPE::NUMBER)
      {
        continue;
      }
      key = crc64.calc(cwords[i].getNormWord().c_str(), cwords[i].getNormWord().length());
      if((it = wordMapPtr.find(key)) != wordMapPtr.end())
      {
        for(unsigned int y = 0; y < (*it).second.size(); y++)
        {
          if(searchType == SearchType::ST_SINGLE)
          {
            incrementalIndex++;
            (*it).second[y].segmentId = incrementalIndex;
            wordOffsetMap.insert(std::pair<unsigned int, WordPos>((*it).second[y].beginOffset, (*it).second[y]));
          }
          else if(searchType == SearchType::ST_STRICKT || searchType == SearchType::ST_NO_STRICKT)
          {
            wordPositionMap.insert(std::pair<unsigned int, std::pair<unsigned long long, WordPos> >
                            ((*it).second[y].position, std::pair<unsigned long long, WordPos>((*it).first, (*it).second[y])));
          }
        }
      }
    }
  }
}

void HighlightingAlgorithmRefine::findChain(Poco::SharedPtr<HCE::OutDataRefine> outData, std::map<unsigned int, WordPos> &wordOffsetMap)
{
  if(!outData.isNull())
  {
    CRC64 crc64;
    NotStricktChain notStricktChain;
    typename std::map<unsigned int, std::pair<unsigned long long, WordPos> >::iterator tempIt;
    std::vector<HCE::CWords> cwords = outData->getCWords();
    for(unsigned int i = 0; i < cwords.size(); i++)
    {
      if(cwords[i].getWordType() != HCE::WORD_TYPE::WORD && cwords[i].getWordType() != HCE::WORD_TYPE::NUMBER)
      {
        continue;
      }
      notStricktChain.addWord(crc64.calc(cwords[i].getNormWord().c_str(), cwords[i].getNormWord().length()));
    }
    if(notStricktChain.wCount() > 0)
    {
      auto prevIt = wordPositionMap.begin();
      auto startChainIt = wordPositionMap.begin();
// cppcheck-suppress variableScope      
      bool jump = false;
      for(auto it = wordPositionMap.begin(); it != wordPositionMap.end(); ++it)
      {
        if((*it).first != ((*prevIt).first + 1) || jump)
        {
          notStricktChain.setDefaultCounts();
          startChainIt = it;
          jump = false;          
        }
        notStricktChain.decrCount((*it).second.first);
        if(notStricktChain.checkOnZerro())
        {
          incrementalIndex++;
          for(tempIt = startChainIt; tempIt != it; ++tempIt)
          {
            (*tempIt).second.second.segmentId = incrementalIndex;
            wordOffsetMap.insert(std::pair<unsigned int, WordPos>((*tempIt).second.second.beginOffset, (*tempIt).second.second));
          }
          (*it).second.second.segmentId = incrementalIndex;
          wordOffsetMap.insert(std::pair<unsigned int, WordPos>((*it).second.second.beginOffset, (*it).second.second));
          jump = true;
          it = startChainIt;
        }
        prevIt = it;
      }
    }
  }
}

void HighlightingAlgorithmRefine::findChainStrict(Poco::SharedPtr<HCE::OutDataRefine> outData, std::map<unsigned int, WordPos> &wordOffsetMap)
{
  if(!outData.isNull())
  {
    CRC64 crc64;
    typename std::map<unsigned int, std::pair<unsigned long long, WordPos> >::iterator localIt;
    std::vector<HCE::CWords> cwords = outData->getCWords();
    std::vector<unsigned long long> crcVec;
    for(unsigned int i = 0; i < cwords.size(); i++)
    {
      if(cwords[i].getWordType() != HCE::WORD_TYPE::WORD && cwords[i].getWordType() != HCE::WORD_TYPE::NUMBER)
      {
        continue;
      }
      crcVec.push_back(crc64.calc(cwords[i].getNormWord().c_str(), cwords[i].getNormWord().length()));
    }
    if(crcVec.size() > 0)
    {
// cppcheck-suppress variableScope      
      unsigned int prevPos = 0;
// cppcheck-suppress variableScope        
      bool isChain = false;      
      for(auto it = wordPositionMap.begin(); it != wordPositionMap.end(); ++it)
      {
        if((*it).second.first == crcVec[0])
        {
          isChain = true;
          prevPos = (*it).first;
          localIt = it;
          localIt++;
          for(unsigned int i = 1; i < crcVec.size(); i++)
          {
            if(localIt == wordPositionMap.end() || ((*localIt).first != prevPos + 1) || ((*localIt).second.first != crcVec[i]))
            {
              isChain = false;
              break;
            }
            prevPos = (*localIt).first;
            localIt++;
          }
          if(isChain)
          {
            localIt = it;
            for(unsigned int i = 0; i < crcVec.size(); i++)
            {
              incrementalIndex++;
              (*localIt).second.second.segmentId = incrementalIndex;
              wordOffsetMap.insert(std::pair<unsigned int, WordPos>((*localIt).second.second.beginOffset, (*localIt).second.second));
              localIt++;
            }
          }
        }
      }
    }
  }
}

void HighlightingAlgorithmRefine::joinBySegmentId(std::map<unsigned int, WordPos> &wordOffsetMap)
{
  unsigned int localSegmentId = 0xFFFFFFFF;
  std::map<unsigned int, WordPos> localWordOffsetMap;
  std::pair<std::map<unsigned int, WordPos>::iterator, bool> insertIt;
  for(auto it = wordOffsetMap.begin(); it != wordOffsetMap.end(); ++it)
  {
    if(it->second.segmentId == localSegmentId)
    {
      insertIt.first->second.endOffset = it->second.endOffset;
    }
    else
    {
      insertIt = localWordOffsetMap.insert(std::pair<unsigned int, WordPos>(it->first, it->second));
    }
    localSegmentId = it->second.segmentId;
  }
  wordOffsetMap = localWordOffsetMap;  
}

void HighlightingAlgorithmRefine::process(const std::vector<std::pair<SearchType, std::string> > &searchStrings, const Poco::SharedPtr<std::string> contentPtr,
                                          ContentsStorageBase::WordPosPtrType wordMapPtr, std::map<unsigned int, WordPos> &wordOffsetMap)
{
  if(wordMapPtr.isNull() || wordMapPtr->empty())
  {
    return;
  }
  wordPositionMap.clear();
  for(unsigned int i = 0; i < searchStrings.size(); i++)
  {
    if(!searchStrings[i].second.empty() && searchStrings[i].second.length() > 0)
    {
      std::map<unsigned long long, std::vector<WordPos> > &localWordPosMap = (searchStrings[i].first == SearchType::ST_SINGLE ? (*wordMapPtr)[1] : (*wordMapPtr)[0]);
      unsigned int normalizationId = (searchStrings[i].first == SearchType::ST_SINGLE ? DEFAULT_NORMALIZATION_WITH : DEFAULT_NORMALIZATION_WITHOUT);
      Poco::SharedPtr<HCE::DataBase> inData = nullptr;
      Poco::SharedPtr<HCE::DataBase> outDataFindStr = nullptr;
      inData = Poco::SharedPtr<HCE::DataBase>(new HCE::InDataRefine(HCE::CT_REFINE));
      inData.cast<HCE::InDataRefine>()->setNormalizationId(normalizationId);
      inData.cast<HCE::InDataRefine>()->setSubjectMask(subjectMask);
      inData.cast<HCE::InDataRefine>()->setContent(searchStrings[i].second);
      outDataFindStr = refine.process(inData);
      fillWordOffsetMap(outDataFindStr.cast<HCE::OutDataRefine>(), searchStrings[i].first, localWordPosMap, wordOffsetMap);
      if(searchStrings[i].first == SearchType::ST_STRICKT)
      {
        findChainStrict(outDataFindStr.cast<HCE::OutDataRefine>(), wordOffsetMap);
      }
      else if(searchStrings[i].first == SearchType::ST_NO_STRICKT)
      {
        findChain(outDataFindStr.cast<HCE::OutDataRefine>(), wordOffsetMap);
      }
    }
  }
  joinBySegmentId(wordOffsetMap);
}