CldPlugin.cpp

Go to the documentation of this file.

#include <cld/ext_lang_enc.h>
#include <iostream>
#include <cld/encodings/proto/encodings.pb.h>

#include "CldPlugin.hpp"
#include "CWords.hpp"

namespace HCE
{

namespace component
{


CldPlugin::CldPlugin()
{
        for ( unsigned i=0; i<MAX_NUM_LANG; ++i)
        {
        toLang[ static_cast<Language>(i) ] = LM_NONE;
        }

        toLang[ static_cast<Language>(0) ] = LM_ENGLISH;
        toLang[ static_cast<Language>(7) ] = LM_ENGLISH;
        toLang[ static_cast<Language>(31) ] = LM_ENGLISH;
        toLang[ static_cast<Language>(32) ] = LM_ENGLISH;
        toLang[ static_cast<Language>(40) ] = LM_ENGLISH;
        toLang[ static_cast<Language>(8) ] = LM_JAPANESE;
    toLang[ static_cast<Language>(13) ] = LM_RUSSIAN;
    toLang[ static_cast<Language>(27) ] = LM_RUSSIAN;

    _adjLang[ static_cast<Language>(13) ] = ::RUSSIAN;
    _adjLang[ static_cast<Language>(27) ] = ::RUSSIAN;
    _adjLang[ static_cast<Language>(29) ] = ::RUSSIAN;
    _adjLang[ static_cast<Language>(34) ] = ::RUSSIAN;
    _adjLang[ static_cast<Language>(36) ] = ::RUSSIAN;
    _adjLang[ static_cast<Language>(37) ] = ::RUSSIAN;
    _adjLang[ static_cast<Language>(47) ] = ::RUSSIAN;
    _adjLang[ static_cast<Language>(50) ] = ::RUSSIAN;

        toLang[ static_cast<Language>(CATALAN) ] = LM_ENGLISH;

        toLangByString["en"] = "en";
        toLangByString["ms"] = "en";
        toLangByString["gl"] = "en";
        toLangByString["it"] = "en";
        toLangByString["fil"] = "en";
        toLangByString["sv"] = "en";
        toLangByString["ja"] = "ja";
        toLangByString["zh"] = "ja";
        toLangByString["zh-TW"] = "ja";
        toLangByString["ru"] = "ru";
    toLangByString["be"] = "ru";
    toLangByString["bg"] = "ru";
    toLangByString["mk"] = "ru";
    toLangByString["sr"] = "ru";
    toLangByString["uk"] = "ru";
    toLangByString["ur"] = "ru";
        toLangByString["ca"] = "en";

    ProcessInfo processInfo;
    SET_PROCESS_INFO_P(processInfo, LDET, CHROMIUM_LDET, LM_ALL, &TPlugin::process, this)
    _exported.push_back( processInfo );
}


CldPlugin::~CldPlugin() {}


template <class T>
inline
const Language detectLanguage(const T& token)
{
        const char* src = token.c_str();
    bool is_plain_text = true;
    bool do_allow_extended_languages = true;
    bool do_pick_summary_language = false;
    bool do_remove_weak_matches = false;
    bool is_reliable;
    const char* tld_hint = NULL;
    int encoding_hint = UNKNOWN_ENCODING;
    Language language_hint = UNKNOWN_LANGUAGE;

    double normalized_score3[3];
    Language language3[3];
    int percent3[3];
    int text_bytes;

    Language lang;
    lang = CompactLangDet::DetectLanguage(0,
                                          src, strlen(src),
                                          is_plain_text,
                                          do_allow_extended_languages,
                                          do_pick_summary_language,
                                          do_remove_weak_matches,
                                          tld_hint,
                                          encoding_hint,
                                          language_hint,
                                          language3,
                                          percent3,
                                          normalized_score3,
                                          &text_bytes,
                                          &is_reliable);
  return lang;
}


void CldPlugin::process(LCoreData& lCData)
{
    _basis_lang_mask = detectLanguage(lCData._content);
#ifdef _DEBUG_
    std::cout << "basic language: " << _basis_lang_mask<<std::endl;
#endif
    std::map<WORD_CONTENT_OFFSET, SSTRING>::const_iterator it;
    for ( it=lCData._mapTokens.begin(); it!=lCData._mapTokens.end(); ++it )
    {
        Language lang = detectLanguage( it->second );
        adjustLanguage(lang);
        //SSTRING iso639_1 = LanguageCode(lang);
        //std::string lstr = toLangByString[iso639_1];
        //lCData._mapLanguages[ it->first ] = (lstr=="") ? "en" : lstr;

        if(lang==UNKNOWN_LANGUAGE) {
            lCData._mapWordType[it->first ] = WORD_TYPE::DELIMITER;
            lang = langAdditionDetection(it->second, lCData._mapWordType[it->first ]);
        } else {
            lCData._mapWordType[it->first ] = WORD_TYPE::WORD;
        }
        lCData._mapLanguages[ it->first ] = correctLanguage(lang);
        lCData._mapLanguagesMasks[ it->first ] |= toLang[lang] | _basis_lang_mask<<1;
        lCData._mapVectorsOfLanguages[ static_cast<Language>( toLang[lang] ) ].push_back( it->first );


    }
#ifdef _DEBUG_
    std::cout << std::endl << "CLD plugin:" << std::endl << std::endl;
    std::map<WORD_CONTENT_OFFSET, WORD_LENGTH>::const_iterator it1;
    std::cout << "token \t language" << std::endl;
    std::cout << "----- \t ----" << std::endl;
    for( it1=lCData._mapOffsets.begin(); it1!=lCData._mapOffsets.end(); ++it1)
    {
        std::cout << lCData._mapTokens[it1->first] << '\t'
                << lCData._mapLanguages[it1->first] << std::endl;
    }
    std::cout << "Tokens count: " << lCData._mapOffsets.size() << std::endl << std::endl;
#endif
}

Language CldPlugin::langAdditionDetection(const std::string &str, WORD_TYPE &wType)
{
  Language ret = UNKNOWN_LANGUAGE;
  for(unsigned int i = 0; i < str.size(); i++)
  {
    if(str[i] >= '0' && str[i] <= '9')
    {
      ret = ::ENGLISH;
      wType = (i == 0) ? WORD_TYPE::NUMBER : WORD_TYPE::WORD;
      break;
    }
  }  
  return ret;
}


} /* namespace component */

} /* namespace HCE */