SnowballPlugin.cpp

Go to the documentation of this file.

#include <iostream>
#include <Poco/UTF8String.h>

#include "SnowballPlugin.hpp"

namespace HCE
{

namespace component
{


SnowballPlugin::SnowballPlugin()
{
    ProcessInfo processInfo;
    SET_PROCESS_INFO_P(processInfo, NORM, SNOWBALL_NORM, LM_ALL, &TPlugin::process, this)
    _exported.push_back( processInfo );
}


SnowballPlugin::~SnowballPlugin()
{
        for ( _it=_mapStemmer.begin(); _it!=_mapStemmer.end(); ++_it )
                sb_stemmer_delete(_it->second);
}


void SnowballPlugin::process(LCoreData& lCData)
{
        std::map<WORD_CONTENT_OFFSET, SSTRING>::const_iterator it;
        std::string stemmerString;
        SSTRING iso639_1;
        sb_stemmer* stemmer = NULL;
        for ( it=lCData._mapTokens.begin(); it!=lCData._mapTokens.end(); ++it )
        {
                iso639_1 = lCData._mapLanguages[ it->first ];
                stemmer = getStemmer( iso639_1 );         
                if(lCData._normalizationId == DEFAULT_NORMALIZATION_WITH && stemmer)
                {
                        stemmerString = Poco::UTF8::toLower(it->second);
                        const sb_symbol * stemmed = sb_stemmer_stem( stemmer, (const sb_symbol*)stemmerString.c_str(), stemmerString.size() );
                        lCData._mapNormalizedForms[ it->first ] = SSTRING( (const char*)stemmed );          
                }
                else
                {
                        lCData._mapNormalizedForms[ it->first ] = SSTRING( it->second );
                }
        }
#ifdef _DEBUG_
        std::cout << std::endl << "SnowballPlugin normalize" << std::endl << std::endl;
        std::map<WORD_CONTENT_OFFSET, WORD_LENGTH>::const_iterator it1;
        std::cout << "token \t norm" << std::endl;
        std::cout << "----- \t ----" << std::endl;
        for( it1=lCData._mapOffsets.begin(); it1!=lCData._mapOffsets.end(); ++it1)
        {
                std::cout << lCData._mapTokens[it1->first] << '\t'
                                << lCData._mapNormalizedForms[it1->first] << std::endl;
        }
        std::cout << "Tokens count " << lCData._mapOffsets.size() << std::endl << std::endl;
#endif
}


sb_stemmer* SnowballPlugin::getStemmer( const SSTRING& iso639_1 )
{
        _it=_mapStemmer.find( iso639_1 );
        if ( _it==_mapStemmer.end() )
        {
                _stemmer = sb_stemmer_new((char*)iso639_1.c_str(), NULL);
                _mapStemmer[ iso639_1 ] = _stemmer;
        }
        else
        {
                _stemmer = _it->second;
        }
        return _stemmer;
}


} /* namespace component */

} /* namespace HCE */