MecabPlugin.cpp

Go to the documentation of this file.

#include <iostream>
#include <map>

#include "MecabPlugin.hpp"

namespace HCE
{

namespace component
{



MecabPlugin::MecabPlugin()
{
    _toPOS["名詞"] = 0; 
    _toPOS["形容詞"] = 1; 
    _toPOS["動詞"] = 2; 
    _toPOS[""] = 3; 
    _toPOS[""] = 4; 
    _toPOS["数"] = 5; 
    _toPOS["接続詞"] = 6; 
    _toPOS["感動詞"] = 7; 
    _toPOS[""] = 8; 
    _toPOS["助詞"] = 9; 
    _toPOS[""] = 10; 
    _toPOS["副詞"] = 11; 
    _toPOS[""] = 12; 
    _toPOS[""] = 13; 
    _toPOS[""] = 14; 
        //toPOS[""] = 15; /// POSS
        //toPOS[""] = 16; /// PN_ADJ
    _toPOS["undefined"] = 15; 


    _toWordType["固有名詞"] = WORD_TYPE::WORD;
    _toWordType["数"] = WORD_TYPE::NUMBER;
    _toWordType["サ変接続"] = WORD_TYPE::PUNCTUATION;
    _toWordType[""] = WORD_TYPE::DELIMITER;
    _toWordType[""] = WORD_TYPE::UNDEFINED;

        _tagger = MeCab::createTagger("");
    /*
    const char *e = _tagger ? _tagger->what() : MeCab::getTaggerError();
    std::cerr << "Exception:" << e << std::endl;
    */
    ProcessInfo processInfo;
    SET_PROCESS_INFO_P(processInfo, NORM, MECAB_NORM, LM_ALL, &TPlugin::process, this)
    _exported.push_back( processInfo );
    SET_PROCESS_INFO_P(processInfo, WTYPE, MECAB_WTYPE, LM_ALL, &TPlugin::process, this)
    _exported.push_back( processInfo );
    SET_PROCESS_INFO_P(processInfo, SPLIT, MECAB_TOKENIZE, LM_ALL, &TPlugin::process, this)
    _exported.push_back( processInfo );
}


MecabPlugin::~MecabPlugin() 
{
        if(_tagger) delete _tagger;
}


void MecabPlugin::process(LCoreData& lCData)
{
    switch( lCData._processInfo._pluginType )
    {
        case MECAB_NORM: normalize(lCData); break;
        case MECAB_WTYPE: wordType(lCData); break;
        case MECAB_TOKENIZE: tokenizer(lCData); break;
        default: break;
    }
}


void MecabPlugin::tokenizer(LCoreData& lCData)
{
        std::string raw_content = lCData.getContent();
        const char *result = _tagger->parse(raw_content.c_str());
        if (result)
        {
            Poco::StringTokenizer token1(result, "\n", Poco::StringTokenizer::TOK_TRIM | Poco::StringTokenizer::TOK_IGNORE_EMPTY );
            Poco::StringTokenizer::Iterator string_iterator;

            for( string_iterator = token1.begin(); string_iterator != token1.end(); ++string_iterator )
            {
                Poco::StringTokenizer token2(*string_iterator, "\t,", Poco::StringTokenizer::TOK_TRIM | Poco::StringTokenizer::TOK_IGNORE_EMPTY );

                size_t offset = 0;
                if ( isTokenValid( token2 ) )
                {
                    offset = raw_content.find(token2[0], offset);
                    size_t len = token2[0].length();
                    lCData._mapOffsets[offset] = len;
                    lCData._mapTokens[offset] = token2[0];
                }
            }
        }
#ifdef _DEBUG_
        std::cout << std::endl << "MeCab plugin: tokenizer" << std::endl << std::endl;
        std::map<WORD_CONTENT_OFFSET, WORD_LENGTH>::const_iterator it1;
        std::cout << "token \t offset \t length" << std::endl;
        std::cout << "----- \t ------ \t ------" << std::endl;
        for( it1=lCData._mapOffsets.begin(); it1!=lCData._mapOffsets.end(); ++it1)
        {
            std::cout << lCData._mapTokens[it1->first] << '\t'
                    << it1->first << '\t'
                    << it1->second << std::endl;
        }
#endif
}


void MecabPlugin::wordType(LCoreData& lCData)
{
    std::map<WORD_CONTENT_OFFSET, WORD_LENGTH>::const_iterator it;
    for ( it=lCData._mapOffsets.begin(); it!=lCData._mapOffsets.end(); ++it )
    {
        const char *result = _tagger->parse(lCData._mapTokens[it->first].c_str());
        if (result)
        {
            Poco::StringTokenizer token1(result, "\n", Poco::StringTokenizer::TOK_TRIM | Poco::StringTokenizer::TOK_IGNORE_EMPTY );
            Poco::StringTokenizer::Iterator string_iterator;

            for( string_iterator = token1.begin(); string_iterator != token1.end(); ++string_iterator )
            {
                Poco::StringTokenizer token2(*string_iterator, "\t,", Poco::StringTokenizer::TOK_TRIM | Poco::StringTokenizer::TOK_IGNORE_EMPTY );

                if ( isTokenValid( token2 ) )
                {
                    lCData._mapWordType[it->first] = getWordType(token2[2]);
                }
            }
        }
    }
#ifdef _DEBUG_
        std::cout << std::endl << "MeCab plugin: word type" << std::endl << std::endl;
        std::map<WORD_CONTENT_OFFSET, WORD_LENGTH>::const_iterator it1;
        std::cout << "token \t WordType" << std::endl;
        std::cout << "----- \t ------ \t ------" << std::endl;
        for( it1=lCData._mapOffsets.begin(); it1!=lCData._mapOffsets.end(); ++it1)
        {
            //std::cout << lCData._mapTokens[it1->first] << '\t'
                    //<< lCData._mapWordType[it1->first] << std::endl;
        }
#endif
}


void MecabPlugin::normalize(LCoreData& lCData)
{
        const char* char_src = lCData._content.c_str();

        if ( char_src )
        {
                const char *result = _tagger->parse(char_src);

        if (result)
        {
            Poco::StringTokenizer token1(result, "\n", Poco::StringTokenizer::TOK_TRIM | Poco::StringTokenizer::TOK_IGNORE_EMPTY );

            long len = 0;

            Poco::StringTokenizer::Iterator string_iterator;

                        for( string_iterator = token1.begin(); string_iterator != token1.end(); ++string_iterator )
            {
                                Poco::StringTokenizer token2(*string_iterator, "\t,", Poco::StringTokenizer::TOK_TRIM | Poco::StringTokenizer::TOK_IGNORE_EMPTY );

                if ( isTokenValid( token2 ) )
                {
                        std::string word = token2[0];

                    //std::string pos = "undefined";
                    std::string pos = token2[2];

                        std::string norm = token2[0];

                        if ( token2.count() == 10 )
                        {
                                if ( token2[9] != "*" )
                                {
                                        pos = token2[1];

                                        norm = token2[7]; //( token2[7]=="*" ) ? token2[0] : token2[7];
                                }
                        }
                    lCData._mapOffsets[len] = word.length();
                    lCData._mapTokens[len] = word;
                    lCData._mapPosMasks[len] |= _toPOS[pos];
                    POSMaskBitset<POS_NUM> bset;
                    bset.setMask( _toPOS[pos] );
                    //lCData._mapPosMasks[len] = bset;

                                        lCData._mapNormalizedForms[ len ] = norm;

                    lCData._mapWordType[len] = getWordType(token2[2]);

                                        len += word.length();
                }
            }
        }
        }
#ifdef _DEBUG_
                std::cout << std::endl << "MeCab plugin:" << std::endl << std::endl;
                std::map<WORD_CONTENT_OFFSET, WORD_LENGTH>::const_iterator it1;
                std::cout << "token \t offset \t length" << std::endl;
                std::cout << "----- \t ------ \t ------" << std::endl;
                for( it1=lCData._mapOffsets.begin(); it1!=lCData._mapOffsets.end(); ++it1)
                {
                        std::cout << lCData._mapTokens[it1->first] << '\t'
                                        << it1->first << '\t'
                                        << it1->second << std::endl;
                }
                std::cout << "Tokens count: " << lCData._mapOffsets.size() << std::endl << std::endl;
#endif
}


} /* namespace component */

} /* namespace HCE */