Refine.cpp

Go to the documentation of this file.

#include <iostream>
#include <cld/compact_lang_det.h>

#include "Refine.hpp"
#include "OutDataRefine.hpp"
#include "RWords.hpp"
#include "CWords.hpp"
#include "LCoreData.hpp"
#include "HCETimer.hpp"


namespace HCE
{

namespace component
{


Refine::Refine(ComponentType inType) : _lCore()
{
        ProcessInfo processInfo;




        SET_PROCESS_INFO_L(processInfo, TAGS, TAG_REDUCE, LM_ALL )
        _processInfoTable.push_back( processInfo );

        //SET_PROCESS_INFO_L(processInfo, SPLIT, ICU_BOUNDARIES, LM_ALL)
    //_processInfoTable.push_back( processInfo );
        //SET_PROCESS_INFO_L(processInfo, SPLIT, BOOST_SPLIT, LM_ALL)
    //_processInfoTable.push_back( processInfo );
        SET_PROCESS_INFO_L(processInfo, SPLIT, BOOST_TOKENIZER, LM_ALL)
    _processInfoTable.push_back( processInfo );
        //SET_PROCESS_INFO_L(processInfo, NORM, MECAB_NORM, LM_ALL )
        //_processInfoTable.push_back( processInfo );

        SET_PROCESS_INFO_L(processInfo, LDET, CHROMIUM_LDET, LM_ALL )
        _processInfoTable.push_back( processInfo );

    //SET_PROCESS_INFO_L(processInfo, NORM, MECAB_NORM, LM_JAPANESE )
    //SET_PROCESS_INFO_L(processInfo, NORM, MECAB_NORM, LM_ALL )
    //_processInfoTable.push_back( processInfo );

        SET_PROCESS_INFO_L(processInfo, NORM, SNOWBALL_NORM, LM_ALL )
        _processInfoTable.push_back( processInfo );

        SET_PROCESS_INFO_L(processInfo, POS, TURGLEM_POS, LM_ALL )
        _processInfoTable.push_back( processInfo );

        SET_PROCESS_INFO_L(processInfo, HCRC, FAKE_HCRC, LM_ALL )
        _processInfoTable.push_back( processInfo );
}


Refine::~Refine() {}


Poco::SharedPtr<DataBase> Refine::process(const Poco::SharedPtr<DataBase> inData)
{
#ifdef _DEBUG_
        HCE::HCETimer hceTimer("Parsing content time: ");
#endif

    LCoreData lCData( inData.cast<InDataRefine>()->getContent() );
    lCData._normalizationId = inData.cast<InDataRefine>()->getNormalizationId();
    lCData._delimiters = inData.cast<InDataRefine>()->getDelimiters();
    lCData._granularityLevel= inData.cast<InDataRefine>()->getGranularityLevel();
    std::vector<ProcessInfo> ProcessInfoTable = \
                        ( inData.cast<InDataRefine>()->getProcessInfoTable().size() ) ? \
                                        inData.cast<InDataRefine>()->getProcessInfoTable() :\
                                        this->_processInfoTable;

        //BOOST_FOREACH( ProcessInfo& processInfo, ProcessInfoTable )
        std::vector<ProcessInfo>::iterator PI_iter;
        for ( PI_iter = ProcessInfoTable.begin(); PI_iter != ProcessInfoTable.end(); ++PI_iter )
        {
#ifdef _DEBUG_
         std::cout << "------------------------" << std::endl;
#endif
        lCData._processInfo = *PI_iter; //processInfo;
                _lCore.selectMethod(lCData);
                /*processInfo*/PI_iter->_status = lCData._processInfo._status;
        }
        //_log_ << "Tokens count" << lCData._mapOffsets.size() << _log_end_;

        Poco::SharedPtr<HCE::DataBase> outDataRefine(new HCE::OutDataRefine(HCE::CT_REFINE) );
        std::vector < CWords > cWords;
        std::vector < RWords > rWords;
        std::map<WORD_CONTENT_OFFSET, WORD_LENGTH>::const_iterator it;
#ifdef _DEBUG_
    std::cout << "token \t norm \t offset \t length \t pos \t hCrc64" << std::endl;
    std::cout << "----- \t ---- \t ------ \t ------ \t --- \t ------" << std::endl;
#endif

        for ( it=lCData._mapOffsets.begin(); it!=lCData._mapOffsets.end(); ++it )
        {
                CWords cword;



                cword.setBlack( lCData._mapBlack[it->first] );

                cword.setSimClass( lCData._mapMorphology[it->first].getMask() );

                //cword.setHCrc( lCData._mapHCRC64[ it->first ] );

                cword.setOffset( it->first );

                cword.setSentenceNumber( lCData._mapSentences[ it->first ] );

                cword.setLingIntegrity( lCData._mapLingIntegrity[ it->first ] );

                cword.setInitWordLen( it->second );

        cword.setOriginWord( lCData._mapTokens[it->first] );

        cword.setNormWord( lCData._mapNormalizedForms[it->first] );

        //cword.setPosMask ( lCData._mapPosMasks[ it->first ].getMask() );
        cword.setPosMask ( lCData._mapPosMasks[ it->first ]);

        cword.setWordType(lCData._mapWordType[ it->first ]);

                cWords.push_back( cword );

                RWords rword;



                rword.setNormWord ( lCData._mapNormalizedForms[it->first] );
                rword.setCrc64 ( lCData._mapHCRC64[ it->first ] );
                //rword.setPosMask ( lCData._mapPosMasks[ it->first ].getMask() );
        rword.setPosMask ( lCData._mapPosMasks[ it->first ] );
                rword.setMorphChangeGradMask ( lCData._mapMorphology[ it->first ].getMask() );
                rWords.push_back(rword);

#ifdef _DEBUG_
        std::cout << lCData._mapTokens[it->first] << '\t'
                        << lCData._mapNormalizedForms[it->first] << '\t'
                        << it->first << '\t'
                                << lCData._mapOffsets[it->first] << '\t'
                //<< lCData._mapPosMasks[it->first].getMask() << '\t'
                << lCData._mapPosMasks[it->first]<< '\t'
                                << lCData._mapHCRC64[it->first] << std::endl;
#endif
        }
        unsigned int langMask = 0;
        unsigned char err = 0;
        outDataRefine.cast<OutDataRefine>()->setCWords(cWords);
        outDataRefine.cast<OutDataRefine>()->setRWords(rWords);
        outDataRefine.cast<OutDataRefine>()->setLangMask(langMask);
        outDataRefine.cast<OutDataRefine>()->setLocalErrorCode(err);

        return outDataRefine;
}


} /* namespace component */

} /* namespace HCE */