BoostPlugin.cpp

Go to the documentation of this file.

#include <boost/algorithm/string/split.hpp>
#include <boost/algorithm/string/classification.hpp>
#include <boost/lambda/lambda.hpp>
#include <boost/tokenizer.hpp>
#include <iostream>
#include <cstddef>
#include <string>


#include "BoostPlugin.hpp"

namespace HCE
{

namespace component
{


BoostPlugin::BoostPlugin()
    :  _defaultDelimiters(u8" \n\t,.?;:-+=().,”’“")
{
    //_defaultDelimiters.append(" \n");
    _delimiters.append(_defaultDelimiters);

    ProcessInfo processInfo;
    SET_PROCESS_INFO_P(processInfo, SPLIT, BOOST_SPLIT, LM_ALL, &TPlugin::process, this)
    _exported.push_back( processInfo );
    SET_PROCESS_INFO_P(processInfo, SPLIT, BOOST_TOKENIZER, LM_ALL, &TPlugin::process, this)
    _exported.push_back( processInfo );
}


BoostPlugin::~BoostPlugin() {}


void BoostPlugin::process(LCoreData& lCData)
{
    updateDefaultDelimiters(lCData);
    switch( lCData._processInfo._pluginType )
        {
                case BOOST_SPLIT: split(lCData); break;
                case BOOST_TOKENIZER: tokenizer(lCData); break;
                default: break;
        }
    restoreDefaultDelimiters();
}


void BoostPlugin::split(LCoreData& lCData)
{
        std::vector<SSTRING> tokens;
        std::string buffer(lCData._content);
        boost::split( tokens, lCData._content, boost::is_any_of(_delimiters));
        for( auto token : tokens)
        {
                size_t offset = buffer.find(token);
                if(offset!=std::string::npos) 
                {
                        buffer.replace(offset, token.length(), token.length(), ' ');
                        lCData._mapOffsets[offset] = token.length();
                        lCData._mapTokens[offset] = token;
                }
        }
#ifdef _DEBUG_
        //std::for_each ( tokens.begin(), tokens.end(), std::cout << boost::lambda::_1 << boost::lambda::constant('\n') );
        std::for_each ( tokens.begin(), tokens.end(), _log_ << boost::lambda::_1 << boost::lambda::constant('\n') );
        //_log_ << "Boost split" << _log_end_;
#endif
}


void BoostPlugin::tokenizer(LCoreData& lCData)
{
    //std::string s(" ");
    std::string raw_content = lCData.getContent();
    typedef boost::tokenizer<boost::char_separator<char> > tok_t;
    boost::char_separator<char> sep(_delimiters.c_str()); //(_tags.c_str(), "| ",  boost::keep_empty_tokens);
    tok_t tok( raw_content, sep );
    size_t offset = 0;
        for (tok_t::const_iterator it = tok.begin(), it_end = tok.end();
               it != it_end; ++it)
        {
        /*
        std::ptrdiff_t const offset = it.base() - str.begin() - it->size();
                size_t len = it->length();
*/
        offset = raw_content.find(*it, offset);
        size_t len = it->length();
        lCData._mapOffsets[offset] = len;
        lCData._mapTokens[offset] = *it;
        /*
        std::cout << "<" << *it << "> "
            << offset << "\t::\t" << len << std::endl;
            */
    }
#ifdef _DEBUG_
        std::cout << std::endl << "Boost tokenizer:" << std::endl << std::endl;
        std::map<WORD_CONTENT_OFFSET, WORD_LENGTH>::const_iterator it1;
        std::cout << "token \t offset \t length" << std::endl;
        std::cout << "----- \t ------ \t ------" << std::endl;

    /*
    for ( it=lCData._mapTokens.begin(); it!=lCData._mapTokens.end(); ++it )
    {
        iso639_1 = lCData._mapLanguages[ it->first ];
        stemmer = getStemmer( iso639_1 );
        if(lCData._normalizationId == DEFAULT_NORMALIZATION_WITH && stemmer)
        {
                stemmerString = Poco::UTF8::toLower(it->second);
            const sb_symbol * stemmed = sb_stemmer_stem( stemmer, (const sb_symbol*)stemmerString.c_str(), stemmerString.size() );
            lCData._mapNormalizedForms[ it->first ] = SSTRING( (const char*)stemmed );
        }
        else
        {
            lCData._mapNormalizedForms[ it->first ] = SSTRING( it->second );
        }
    }
     */


    for( it1=lCData._mapOffsets.begin(); it1!=lCData._mapOffsets.end(); ++it1)
        {
                std::cout << lCData._mapTokens[it1->first] << '\t'
                                << it1->first << '\t'
                                << it1->second << std::endl;
        }
        std::cout << "Tokens count: " << lCData._mapOffsets.size() << std::endl << std::endl;
#endif
}


void BoostPlugin::updateDefaultDelimiters(LCoreData& lCData) {
    if(lCData._delimiters.size()) {
        _delimiters.clear();
        for(auto delimiter : lCData._delimiters) {
            _delimiters.append(delimiter);
        }
    }
}


void BoostPlugin::restoreDefaultDelimiters() {
    if(_delimiters != _defaultDelimiters) {
        _delimiters.clear();
        _delimiters.append(_defaultDelimiters);
    }
}


} /* namespace component */

} /* namespace HCE */