dc_processor.ScraperLangDetector.ScraperLangDetector Class Reference

Inheritance diagram for dc_processor.ScraperLangDetector.ScraperLangDetector:

Collaboration diagram for dc_processor.ScraperLangDetector.ScraperLangDetector:

Public Member Functions
def	__init__ (self, scraperLangDetectProperty)
def	process (self, response, log=None)
def	getLangTags (self)
def	getLangTagsNames (self)
def	getSummaryLang (self, response, log=None)

Static Public Member Functions
def	langDetect (incomeBuf, convertToFullName=True, log=None)

Public Attributes
	prefix
	suffix
	tagsList
	maps
	size
	detectedLangs

Static Public Attributes
string	MSG_ERROR_LANG_DETECT = "Language detection failed. Error: %s"
string	PROPERTY_OPTION_PREFIX = "prefix"
string	PROPERTY_OPTION_SUFFIX = "suffix"
string	PROPERTY_OPTION_TAGS = "tags"
string	PROPERTY_OPTION_MAPS = "maps"
string	PROPERTY_OPTION_SIZE = "size"
string	DEFAULT_VALUE_OPTION_PREFIX = ""
string	DEFAULT_VALUE_OPTION_SUFFIX = "_lang"
list	DEFAULT_VALUE_OPTION_TAGS = []
dictionary	DEFAULT_VALUE_OPTION_MAPS
int	DEFAULT_VALUE_OPTION_SIZE = 1024
string	DEFAULT_VALUE_SUMMARY_LANG = "en"
string	DEFAULT_VALUE_LANG_MAPPING = '*'
list	DEFAULT_VALUE_TAGS_NAMES
string	TAGS_EXTENDED_VALUE_ALL = "*"
string	TAGS_EXTENDED_VALUE_SUMMARY = "&"
string	SCRAPER_RESULT_TAG_OPTION_DATA = "data"
string	SCRAPER_RESULT_TAG_OPTION_LANG = "lang"
string	SCRAPER_RESULT_TAG_OPTION_SUMMARY_LANG = "summary_lang"

Private Member Functions
def	__makeTagName (self, tagName)
def	__retTagsText (self, tagName, response)
def	__truncateBuffer (self, text, log=None)
def	__setLangField (self, text, tagName, fieldName, response, log=None)
def	__langMapping (self, lang)
def	__isExistValue (self, src, val)

Detailed Description

Definition at line 20 of file ScraperLangDetector.py.

Constructor & Destructor Documentation

__init__()

def dc_processor.ScraperLangDetector.ScraperLangDetector.__init__	(		self,
			scraperLangDetectProperty
	)

Definition at line 56 of file ScraperLangDetector.py.

  def __init__(self, scraperLangDetectProperty):
    self.prefix = self.DEFAULT_VALUE_OPTION_PREFIX
    self.suffix = self.DEFAULT_VALUE_OPTION_SUFFIX
    self.tagsList = self.DEFAULT_VALUE_OPTION_TAGS
    self.maps = self.DEFAULT_VALUE_OPTION_MAPS
    self.size = self.DEFAULT_VALUE_OPTION_SIZE
    self.detectedLangs = {}

    if isinstance(scraperLangDetectProperty, dict):
      # set prefix options
      if self.PROPERTY_OPTION_PREFIX in scraperLangDetectProperty:
        self.prefix = scraperLangDetectProperty[self.PROPERTY_OPTION_PREFIX]

      # set suffix options
      if self.PROPERTY_OPTION_SUFFIX in scraperLangDetectProperty:
        self.suffix = scraperLangDetectProperty[self.PROPERTY_OPTION_SUFFIX]

      # set tags options
      if self.PROPERTY_OPTION_TAGS in scraperLangDetectProperty:
        self.tagsList = scraperLangDetectProperty[self.PROPERTY_OPTION_TAGS]

      # set maps options
      if self.PROPERTY_OPTION_MAPS in scraperLangDetectProperty:
        self.maps = scraperLangDetectProperty[self.PROPERTY_OPTION_MAPS]

      # set size options
      if self.PROPERTY_OPTION_SIZE in scraperLangDetectProperty:
        self.size = int(scraperLangDetectProperty[self.PROPERTY_OPTION_SIZE])

Member Function Documentation

__isExistValue()

def dc_processor.ScraperLangDetector.ScraperLangDetector.__isExistValue (   self,   src,   val  )

private

Definition at line 222 of file ScraperLangDetector.py.

  def __isExistValue(self, src, val):
    return len([s for s in src if val in s]) > 0

Here is the caller graph for this function:

__langMapping()

def dc_processor.ScraperLangDetector.ScraperLangDetector.__langMapping (   self,   lang  )

private

Definition at line 194 of file ScraperLangDetector.py.

  def __langMapping(self, lang):
    # variable for result
    ret = lang

    isExist = False
    for key, value in self.maps.items():
      isExist = self.__isExistValue(value, lang)
      if isExist:
        ret = key
        break

    if not isExist:
      default = None
      for key, value in self.maps.items():
        if self.__isExistValue(value, self.DEFAULT_VALUE_LANG_MAPPING):
          default = key

      if default is not None:
        ret = default

    return ret

Here is the call graph for this function:

Here is the caller graph for this function:

__makeTagName()

def dc_processor.ScraperLangDetector.ScraperLangDetector.__makeTagName (   self,   tagName  )

private

Definition at line 90 of file ScraperLangDetector.py.

  def __makeTagName(self, tagName):
    return self.prefix + tagName + self.suffix

Here is the caller graph for this function:

__retTagsText()

def dc_processor.ScraperLangDetector.ScraperLangDetector.__retTagsText (   self,   tagName,   response  )

private

Definition at line 121 of file ScraperLangDetector.py.

  def __retTagsText(self, tagName, response):
    # variable for result
    ret = None

    if response is not None and tagName in response.tags:
      if isinstance(response.tags[tagName], basestring):
        ret = response.tags[tagName]

      elif isinstance(response.tags[tagName], dict) and \
        ScraperLangDetector.SCRAPER_RESULT_TAG_OPTION_DATA in response.tags[tagName]:
        if isinstance(response.tags[tagName][ScraperLangDetector.SCRAPER_RESULT_TAG_OPTION_DATA], basestring):
          ret = response.tags[tagName][ScraperLangDetector.SCRAPER_RESULT_TAG_OPTION_DATA]

        elif isinstance(response.tags[tagName][ScraperLangDetector.SCRAPER_RESULT_TAG_OPTION_DATA], list):
          ret = ""
          for elem in response.tags[tagName][ScraperLangDetector.SCRAPER_RESULT_TAG_OPTION_DATA]:
            ret += elem
            ret += ' '
          ret = ret.strip()

    return ret

Here is the caller graph for this function:

__setLangField()

def dc_processor.ScraperLangDetector.ScraperLangDetector.__setLangField (   self,   text,   tagName,   fieldName,   response,   log = None  )

private

Definition at line 173 of file ScraperLangDetector.py.

  def __setLangField(self, text, tagName, fieldName, response, log=None):

    if text is not None:
      # truncate buffer by limit size
      text = self.__truncateBuffer(text, log)
      
      # detect language
      lang = ScraperLangDetector.langDetect(text, False, log)
      if log is not None:
        log.debug("for '%s' was detected '%s'", str(tagName), str(lang))

      if lang is not None and isinstance(response.tags[tagName], dict):
        lang = self.__langMapping(lang)
        response.tags[tagName][fieldName] = lang
        self.detectedLangs[tagName] = lang

Here is the call graph for this function:

Here is the caller graph for this function:

__truncateBuffer()

def dc_processor.ScraperLangDetector.ScraperLangDetector.__truncateBuffer (   self,   text,   log = None  )

private

Definition at line 149 of file ScraperLangDetector.py.

  def __truncateBuffer(self, text, log=None):
    # variable for result
    buff = text if len(text) <= self.size else text[:self.size]
    while len(buff) > 0:
      try:
        buff.decode('utf-8')
        break
      except Exception:
        buff = buff[:-1]

    if log is not None:
      log.debug("buffer len = %s was trancated to len = %s used limit = %s", str(len(text)), str(len(buff)), str(self.size))

    return buff

Here is the caller graph for this function:

getLangTags()

def dc_processor.ScraperLangDetector.ScraperLangDetector.getLangTags

(

self

)

Definition at line 266 of file ScraperLangDetector.py.

  def getLangTags(self):
    # variable for result
    langTagsDict = {}

    for tagName, lang in self.detectedLangs.items():
      langTagsDict[self.__makeTagName(tagName)] = lang

    return langTagsDict

Here is the call graph for this function:

getLangTagsNames()

def dc_processor.ScraperLangDetector.ScraperLangDetector.getLangTagsNames

(

self

)

Definition at line 280 of file ScraperLangDetector.py.

  def getLangTagsNames(self):
    # variable for result
    langTagsNames = []

    tagsList = []

    if isinstance(self.tagsList, basestring) and self.tagsList == self.TAGS_EXTENDED_VALUE_ALL or \
      isinstance(self.tagsList, list) and self.TAGS_EXTENDED_VALUE_ALL in self.tagsList or \
      isinstance(self.tagsList, basestring) and self.tagsList == self.TAGS_EXTENDED_VALUE_SUMMARY or \
      isinstance(self.tagsList, list) and self.TAGS_EXTENDED_VALUE_SUMMARY in self.tagsList:
      tagsList = self.DEFAULT_VALUE_TAGS_NAMES

    elif isinstance(self.tagsList, list):
      tagsList = self.tagsList

    for tagName in tagsList:
      langTagsNames.append(self.__makeTagName(tagName))

    return langTagsNames

Here is the call graph for this function:

getSummaryLang()

def dc_processor.ScraperLangDetector.ScraperLangDetector.getSummaryLang	(		self,
			response,
			log = None
	)

Definition at line 306 of file ScraperLangDetector.py.

  def getSummaryLang(self, response, log=None):
    #variable for result
    summaryLang = self.DEFAULT_VALUE_SUMMARY_LANG

    if response is not None:
      for tagName, tagValue in response.tags.items():
        if isinstance(tagValue, dict) and self.SCRAPER_RESULT_TAG_OPTION_SUMMARY_LANG in tagValue:
          summaryLang = tagValue[self.SCRAPER_RESULT_TAG_OPTION_SUMMARY_LANG]
          if log is not None:
            log.debug("Summary lang '%s' was extracted from field '%s'", str(summaryLang), str(tagName))
          break

    return summaryLang

langDetect()

def dc_processor.ScraperLangDetector.ScraperLangDetector.langDetect (   incomeBuf,   convertToFullName = True,   log = None  )

static

Definition at line 101 of file ScraperLangDetector.py.

  def langDetect(incomeBuf, convertToFullName=True, log=None):
    ret = None

    if incomeBuf is not None and incomeBuf != "":
      try:
        from langdetect import detect
        ret = detect(incomeBuf.decode('utf-8')).replace('-', ',')
      except Exception, err:
        if log is not None:
          log.error(ScraperLangDetector.MSG_ERROR_LANG_DETECT, str(err))
          log.debug(Utils.getTracebackInfo())

    return ret

process()

def dc_processor.ScraperLangDetector.ScraperLangDetector.process	(		self,
			response,
			log = None
	)

Definition at line 231 of file ScraperLangDetector.py.

  def process(self, response, log=None):
    if response is not None:
      # use all tags
      if isinstance(self.tagsList, basestring) and self.tagsList == self.TAGS_EXTENDED_VALUE_ALL or \
        isinstance(self.tagsList, list) and self.TAGS_EXTENDED_VALUE_ALL in self.tagsList:
        for tagName in response.tags:
          localTextValue = self.__retTagsText(tagName, response)
          self.__setLangField(localTextValue, tagName, self.SCRAPER_RESULT_TAG_OPTION_LANG, response, log)

      # use summary tags
      elif isinstance(self.tagsList, basestring) and self.tagsList == self.TAGS_EXTENDED_VALUE_SUMMARY or \
        isinstance(self.tagsList, list) and self.TAGS_EXTENDED_VALUE_SUMMARY in self.tagsList:
        localTextResult = None
        for tagName in response.tags:
          localTextResult = ""
          localTextValue = self.__retTagsText(tagName, response)
          if localTextValue is not None:
            localTextResult += localTextValue
            localTextResult += ' '
          localTextResult = localTextResult.strip()

        for tagName in response.tags:
          self.__setLangField(localTextResult, tagName, self.SCRAPER_RESULT_TAG_OPTION_SUMMARY_LANG, response, log)

      # use list tags
      elif isinstance(self.tagsList, list):    
        for tagName in self.tagsList:
          localTextValue = self.__retTagsText(tagName, response)
          self.__setLangField(localTextValue, tagName, self.SCRAPER_RESULT_TAG_OPTION_LANG, response, log)
          
          

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

DEFAULT_VALUE_LANG_MAPPING

string dc_processor.ScraperLangDetector.ScraperLangDetector.DEFAULT_VALUE_LANG_MAPPING = '*'

static

Definition at line 43 of file ScraperLangDetector.py.

DEFAULT_VALUE_OPTION_MAPS

dictionary dc_processor.ScraperLangDetector.ScraperLangDetector.DEFAULT_VALUE_OPTION_MAPS

static

Initial value:

=  { "en": [ "fr", "nl", "ro", "af", "ca", "it", "da", "tl", "et", "cy", "sv", "id", "es", "*" ], \
                                "ja": [ "ja", "zh", "za" ], \
                                "ru": [ "ru", "uk" ], \
                                "pl": [ "pl" ], \
                                "de": [ "de" ] }

Definition at line 35 of file ScraperLangDetector.py.

DEFAULT_VALUE_OPTION_PREFIX

string dc_processor.ScraperLangDetector.ScraperLangDetector.DEFAULT_VALUE_OPTION_PREFIX = ""

static

Definition at line 32 of file ScraperLangDetector.py.

DEFAULT_VALUE_OPTION_SIZE

int dc_processor.ScraperLangDetector.ScraperLangDetector.DEFAULT_VALUE_OPTION_SIZE = 1024

static

Definition at line 40 of file ScraperLangDetector.py.

DEFAULT_VALUE_OPTION_SUFFIX

string dc_processor.ScraperLangDetector.ScraperLangDetector.DEFAULT_VALUE_OPTION_SUFFIX = "_lang"

static

Definition at line 33 of file ScraperLangDetector.py.

DEFAULT_VALUE_OPTION_TAGS

list dc_processor.ScraperLangDetector.ScraperLangDetector.DEFAULT_VALUE_OPTION_TAGS = []

static

Definition at line 34 of file ScraperLangDetector.py.

DEFAULT_VALUE_SUMMARY_LANG

string dc_processor.ScraperLangDetector.ScraperLangDetector.DEFAULT_VALUE_SUMMARY_LANG = "en"

static

Definition at line 41 of file ScraperLangDetector.py.

DEFAULT_VALUE_TAGS_NAMES

list dc_processor.ScraperLangDetector.ScraperLangDetector.DEFAULT_VALUE_TAGS_NAMES

static

Initial value:

=  [CONSTS.TAG_MEDIA, CONSTS.TAG_TITLE, CONSTS.TAG_LINK, CONSTS.TAG_DESCRIPTION, CONSTS.TAG_PUB_DATE, CONSTS.TAG_DC_DATE, \
                              CONSTS.TAG_AUTHOR, CONSTS.TAG_CONTENT_UTF8_ENCODED, CONSTS.TAG_KEYWORDS]

Definition at line 45 of file ScraperLangDetector.py.

detectedLangs

dc_processor.ScraperLangDetector.ScraperLangDetector.detectedLangs

Definition at line 62 of file ScraperLangDetector.py.

maps

dc_processor.ScraperLangDetector.ScraperLangDetector.maps

Definition at line 60 of file ScraperLangDetector.py.

MSG_ERROR_LANG_DETECT

string dc_processor.ScraperLangDetector.ScraperLangDetector.MSG_ERROR_LANG_DETECT = "Language detection failed. Error: %s"

static

Definition at line 22 of file ScraperLangDetector.py.

prefix

dc_processor.ScraperLangDetector.ScraperLangDetector.prefix

Definition at line 57 of file ScraperLangDetector.py.

PROPERTY_OPTION_MAPS

string dc_processor.ScraperLangDetector.ScraperLangDetector.PROPERTY_OPTION_MAPS = "maps"

static

Definition at line 29 of file ScraperLangDetector.py.

PROPERTY_OPTION_PREFIX

string dc_processor.ScraperLangDetector.ScraperLangDetector.PROPERTY_OPTION_PREFIX = "prefix"

static

Definition at line 26 of file ScraperLangDetector.py.

PROPERTY_OPTION_SIZE

string dc_processor.ScraperLangDetector.ScraperLangDetector.PROPERTY_OPTION_SIZE = "size"

static

Definition at line 30 of file ScraperLangDetector.py.

PROPERTY_OPTION_SUFFIX

string dc_processor.ScraperLangDetector.ScraperLangDetector.PROPERTY_OPTION_SUFFIX = "suffix"

static

Definition at line 27 of file ScraperLangDetector.py.

PROPERTY_OPTION_TAGS

string dc_processor.ScraperLangDetector.ScraperLangDetector.PROPERTY_OPTION_TAGS = "tags"

static

Definition at line 28 of file ScraperLangDetector.py.

SCRAPER_RESULT_TAG_OPTION_DATA

string dc_processor.ScraperLangDetector.ScraperLangDetector.SCRAPER_RESULT_TAG_OPTION_DATA = "data"

static

Definition at line 51 of file ScraperLangDetector.py.

SCRAPER_RESULT_TAG_OPTION_LANG

string dc_processor.ScraperLangDetector.ScraperLangDetector.SCRAPER_RESULT_TAG_OPTION_LANG = "lang"

static

Definition at line 52 of file ScraperLangDetector.py.

SCRAPER_RESULT_TAG_OPTION_SUMMARY_LANG

string dc_processor.ScraperLangDetector.ScraperLangDetector.SCRAPER_RESULT_TAG_OPTION_SUMMARY_LANG = "summary_lang"

static

Definition at line 53 of file ScraperLangDetector.py.

size

dc_processor.ScraperLangDetector.ScraperLangDetector.size

Definition at line 61 of file ScraperLangDetector.py.

suffix

dc_processor.ScraperLangDetector.ScraperLangDetector.suffix

Definition at line 58 of file ScraperLangDetector.py.

TAGS_EXTENDED_VALUE_ALL

string dc_processor.ScraperLangDetector.ScraperLangDetector.TAGS_EXTENDED_VALUE_ALL = "*"

static

Definition at line 48 of file ScraperLangDetector.py.

TAGS_EXTENDED_VALUE_SUMMARY

string dc_processor.ScraperLangDetector.ScraperLangDetector.TAGS_EXTENDED_VALUE_SUMMARY = "&"

static

Definition at line 49 of file ScraperLangDetector.py.

tagsList

dc_processor.ScraperLangDetector.ScraperLangDetector.tagsList

Definition at line 59 of file ScraperLangDetector.py.

---

The documentation for this class was generated from the following file:

• sources/hce/dc_processor/ScraperLangDetector.py