TurglemPlugin.cpp

Go to the documentation of this file.

#include "TurglemPlugin.hpp"


namespace HCE
{

namespace component
{


TurglemPlugin::TurglemPlugin()
{
    ProcessInfo processInfo;
    SET_PROCESS_INFO_P(processInfo, POS, TURGLEM_POS, LM_ALL, &TPlugin::process, this)
    _exported.push_back( processInfo );

    _lem[LM_ENGLISH].load_lemmatizer("/usr/share/turglem/english/dict_english.auto",
                            "/usr/share/turglem/english/paradigms_english.bin",
                            "/usr/share/turglem/english/prediction_english.auto");
    _lem[LM_RUSSIAN].load_lemmatizer("/usr/share/turglem/russian/dict_russian.auto",
                                "/usr/share/turglem/russian/paradigms_russian.bin",
                                "/usr/share/turglem/russian/prediction_russian.auto");
}


TurglemPlugin::~TurglemPlugin()
{
}


void TurglemPlugin::process(LCoreData& lCData)
{
        std::map<WORD_CONTENT_OFFSET, SSTRING>::const_iterator it;
        for ( it=lCData._mapTokens.begin(); it!=lCData._mapTokens.end(); ++it )
        {
                LanguageMask languageMask = lCData._mapLanguagesMasks[it->first];
                for ( Language lang=ENGLISH; lang!=MAX_NUM_LANG/*THAI*/; ++lang )
                {
                        if ( languageMask&lang ) 
                        {
                                std::map<Language, tl::lemmatizer>::const_iterator lit;
                                lit = _lem.find( lang );
                                if ( lit!=_lem.end() ) 
                                {
                                        size_t sz_lem = 0;
                                        tl::lem_result lr;
                                        switch ( lang )
                                        {
                                        case LM_ENGLISH:
                        sz_lem = lit->second.lemmatize<english_utf8_adapter>(it->second.c_str(), lr);
                                                break;
                                        case LM_RUSSIAN:
                                                sz_lem = lit->second.lemmatize<russian_utf8_adapter>(it->second.c_str(), lr);
                                                break;
                                        default: break;
                                        }
                    if (sz_lem) 
                                        {
                        for (size_t i = 0; i < sz_lem; i++)
                                                {
                                                        //int src = -1;
                                                        POSMask pos;
                                                        //int paradigm = -1;
                                                        std::string nform;
                                                        switch ( lit->first )
                                                        {
                                                        case LM_ENGLISH:
                                                                nform = lit->second.get_text<english_utf8_adapter>(lr, i, 0);
                                                                break;
                                                        case LM_RUSSIAN:
                                                                nform = lit->second.get_text<russian_utf8_adapter>(lr, i, 0);
                                                                break;
                                                        default: break;
                                                        }
                                                        pos = static_cast<POSMask>(lit->second.get_part_of_speech(lr, i, 0));

                                                        //paradigm = lit->second.get_paradigm(lr, i);

                                                        //src = lit->second.get_src_form(lr, i);


                                                        lCData._mapPosMasks[it->first] |= 1 << pos;
                                    POSMaskBitset<POS_NUM> bset;
                                    bset.setMask(pos);
                                    //lCData._mapPosMasks[it->first] = bset;

                                    //lCData._mapNormalizedForms[ it->first ] = nform;
                                                }
                                        }
                                }
                        }
                }
        }
#ifdef _DEBUG_
        std::cout << std::endl << "TurglemPlugin normalize" << std::endl << std::endl;
        std::map<WORD_CONTENT_OFFSET, WORD_LENGTH>::const_iterator it1;
        std::cout << "token \t pos" << std::endl;
        std::cout << "----- \t ---" << std::endl;
        for( it1=lCData._mapOffsets.begin(); it1!=lCData._mapOffsets.end(); ++it1)
        {
                std::cout << lCData._mapTokens[it1->first] << '\t'
                //<< lCData._mapPosMasks[it1->first].getMask()
                << lCData._mapPosMasks[it1->first]
                                << std::endl;
        }
        std::cout << "Tokens count: " << lCData._mapOffsets.size() << std::endl << std::endl;
#endif
}


} /* namespace component */

} /* namespace HCE */