IcuPlugin.cpp

Go to the documentation of this file.

#include <iostream>

#include "IcuPlugin.hpp"

namespace HCE
{

namespace component
{


 #define ICU_DISPLAY(string)    char charBuf[1000]; \
                                UnicodeString s1( (CHAR_TYPE*)(string.c_str()) ); \
                                s1.extract(0, s1.length(), charBuf, sizeof(charBuf)-1, 0); \
                                charBuf[sizeof(charBuf)-1] = 0; \
                                std::cout << charBuf << std::endl;


IcuPlugin::IcuPlugin()
{
    ProcessInfo processInfo;
    SET_PROCESS_INFO_P(processInfo, SPLIT, ICU_BOUNDARIES, LM_ALL, &TPlugin::process, this)
    _exported.push_back( processInfo );
}


IcuPlugin::~IcuPlugin() {}


void IcuPlugin::process( LCoreData& lCData )
{
    _status = U_ZERO_ERROR;
    _boundary = BreakIterator::createWordInstance( Locale::getUS(), _status );
    UnicodeString ustr( (/*UChar* */CHAR_TYPE*)lCData._content.c_str(), lCData._content.size() );
    _boundary->setText( ustr );

     int32_t start = _boundary->first();
      for (int32_t end = _boundary->next();
           end != BreakIterator::DONE;
           start = end, end = _boundary->next())
      {
          //CharacterIterator *strIter = _boundary->getText().clone();
          //UnicodeString  s;
          //strIter->getText(s);
          size_t len = end - start;
          UnicodeString str( ustr/*.getBuffer()*/, start, len );
          SSTRING token;
          str.toUTF8String( token );
          lCData._tokens.push_back( token );

          lCData._mapOffsets[start] = len;
          lCData._mapTokens[start] = token;

 #ifdef _DEBUG_
          //ICU_DISPLAY( str )
          std::cout << token << std::endl;
 #endif
      }
#ifdef _DEBUG_
         std::cout << "Tokens: " << lCData._tokens.size() << std::endl;
#endif
}


} /* namespace component */

} /* namespace HCE */