dc_processor.base_extractor Namespace Reference

Classes
class	BaseExtractor

Functions
def	signal_handler (signum, frame)
def	adjustPubDate (dates)
def	adjustMedia (medias)
def	adjustContentUTF8Encoded (data)
def	adjustLink (data)
def	adjustNone (data)

Variables
	logger = Utils.MPLogger().getLogger()
string	ERR_MSG_ADJUST_PUB_DATE = "Error in adjustPubDate: "
string	ERR_MSG_ADJUST_MEDIA = "Error in adjustMedia: "
string	ERR_MSG_ADJUST_CONTENT_UTF8_ENCODED = "Error in adjustContentUTF8Encoded: "
string	ERR_MSG_OK = ""
string	EMPTY_DATE = ""

Detailed Description

@package docstring
 @file base_extractor.py
 @author Alexey, bgv <developers.hce@gmail.com>
 @link http://hierarchical-cluster-engine.com/
 @copyright Copyright &copy; 2013 IOIX Ukraine
 @license http://hierarchical-cluster-engine.com/license/
 @package HCE project node API
 @since 0.1

Function Documentation

adjustContentUTF8Encoded()

def dc_processor.base_extractor.adjustContentUTF8Encoded

(

data

)

Definition at line 81 of file base_extractor.py.

def adjustContentUTF8Encoded(data):
  return data


# Adjust data in content_encoded tag
# If content are non-meaningfull adjust it
# @param data content extracted from content

adjustLink()

def dc_processor.base_extractor.adjustLink

(

data

)

Definition at line 88 of file base_extractor.py.

def adjustLink(data):
  if isinstance(data, list) and len(data) > 1:
    data = data[0]
  return data

adjustMedia()

def dc_processor.base_extractor.adjustMedia

(

medias

)

Definition at line 64 of file base_extractor.py.

def adjustMedia(medias):
  return medias
  # valid_http_url = HttpUrl()
  # res = []
  # try:
  #  if isinstance(medias, list):
  #    for media in medias:
  #      if valid_http_url(media):
  #        res.append(media)
  # except Exception as err:
  #  logger.error(ERR_MSG_ADJUST_MEDIA + err.message)
  # return res


# Adjust data in content_encoded tag
# If content are non-meaningfull adjust it
# @param data content extracted from content

adjustNone()

def dc_processor.base_extractor.adjustNone

(

data

)

Definition at line 94 of file base_extractor.py.

def adjustNone(data):
  return data


# #The BaseExtractor class
# This is the base class for custom extractors
# Provide basic functionality such as add tag, etc.

adjustPubDate()

def dc_processor.base_extractor.adjustPubDate

(

dates

)

Definition at line 42 of file base_extractor.py.

def adjustPubDate(dates):
  # logger.debug("dates: %s", dates)
  pub_date = EMPTY_DATE
  try:
    # TODO: improve to return most appropriate
    # if dates and any(i.isdigit() for i in dates):
    if isinstance(dates, list) and len(dates):
      # pub_date = dates[0]
      pub_date = " ".join(dates)
    else:
      pub_date = dates
    if pub_date and len(dates) and not re.search(r'\d+', pub_date):
      pub_date = EMPTY_DATE
  except Exception as err:
    ExceptionLog.handler(logger, err, ERR_MSG_ADJUST_PUB_DATE)

  return pub_date


# Adjust data in media tag
# If media are PR (partial reference) adjust path
# @param medias media extracted from content

signal_handler()

def dc_processor.base_extractor.signal_handler	(		signum,
			frame
	)

Definition at line 24 of file base_extractor.py.

def signal_handler(signum, frame):
  del signum, frame
  logger.debug("Time execution limit was reached: %s seconds.", str(CONSTS.TIME_EXECUTION_LIMIT))
  raise Exception("Timed out!")


# Local class constants

Variable Documentation

EMPTY_DATE

string dc_processor.base_extractor.EMPTY_DATE = ""

Definition at line 37 of file base_extractor.py.

ERR_MSG_ADJUST_CONTENT_UTF8_ENCODED

string dc_processor.base_extractor.ERR_MSG_ADJUST_CONTENT_UTF8_ENCODED = "Error in adjustContentUTF8Encoded: "

Definition at line 33 of file base_extractor.py.

ERR_MSG_ADJUST_MEDIA

string dc_processor.base_extractor.ERR_MSG_ADJUST_MEDIA = "Error in adjustMedia: "

Definition at line 32 of file base_extractor.py.

ERR_MSG_ADJUST_PUB_DATE

string dc_processor.base_extractor.ERR_MSG_ADJUST_PUB_DATE = "Error in adjustPubDate: "

Definition at line 31 of file base_extractor.py.

ERR_MSG_OK

string dc_processor.base_extractor.ERR_MSG_OK = ""

Definition at line 35 of file base_extractor.py.

logger

dc_processor.base_extractor.logger = Utils.MPLogger().getLogger()

Definition at line 20 of file base_extractor.py.