app.Utils.UrlNormalizator Class Reference

Inheritance diagram for app.Utils.UrlNormalizator:

Collaboration diagram for app.Utils.UrlNormalizator:

Public Member Functions
def	__init__ (self)

Static Public Member Functions
def	normalize (url, supportProtocols=None, normMask=NORM_DEFAULT)
def	isNormalUrl (url)
def	entitiesEncode (url, entities=None)

Static Public Attributes
int	NORM_NONE = 0
int	NORM_SKIP_WWW = 1
int	NORM_USE_VALIDATOR = 2
int	NORM_MAIN = 4
int	NORM_DEFAULT = NORM_MAIN
string	BAD_URL_PREFIX = "normalization-error://?"

Detailed Description

Definition at line 659 of file Utils.py.

Constructor & Destructor Documentation

__init__()

def app.Utils.UrlNormalizator.__init__

(

self

)

Definition at line 669 of file Utils.py.

  def __init__(self):
    pass

Member Function Documentation

entitiesEncode()

def app.Utils.UrlNormalizator.entitiesEncode (   url,   entities = None  )

static

Definition at line 728 of file Utils.py.

  def entitiesEncode(url, entities=None):
    ret = url
    if entities is None:
      entities = {"&": "&"}

    for k in entities:
      le = len(entities[k])
      p = -1
      while True:
        l = len(ret)
        p = ret.find(k, p + 1)
        if p == -1:
          break
        else:
          if (p + le - 1 > l) or ((p + le - 1 <= l) and (ret[p:p + le] != entities[k])):
            ret = ret[:p] + entities[k] + ret[p + 1:]
          else:
            continue

    return ret



# ENV_SCRAPER_STORE_PATH = "ENV_SCRAPER_STORE_PATH"
# # storePickleOnDisk
#

isNormalUrl()

def app.Utils.UrlNormalizator.isNormalUrl (   url )

static

Definition at line 718 of file Utils.py.

  def isNormalUrl(url):
    return False if url.find(UrlNormalizator.BAD_URL_PREFIX) == 0 else True

normalize()

def app.Utils.UrlNormalizator.normalize (   url,   supportProtocols = None,   normMask = NORM_DEFAULT  )

static

Definition at line 678 of file Utils.py.

  def normalize(url, supportProtocols=None, normMask=NORM_DEFAULT):
    norm_url = url.strip()
    if normMask != 0:
      logger.debug("None zero normMask: %s", str(normMask))
    # TODO: need to be replaced with default filter for collect URLs protocols check stage
    if supportProtocols is not None and isinstance(supportProtocols, list):
      colonPos = norm_url.find(':')
      slashPos = norm_url.find('/')
      if colonPos != -1 and (slashPos == -1 or slashPos > colonPos):
        if len(norm_url.split(':')) > 1:
          protocol = norm_url.split(':')[0]
          if protocol not in supportProtocols:
            try:
              norm_url = UrlNormalizator.BAD_URL_PREFIX + urllib.quote(norm_url)
            except Exception as err:
              logger.debug(">>> urllib.quote error = " + str(err))
              norm_url = UrlNormalizator.BAD_URL_PREFIX + norm_url

    if norm_url == url:
      try:
        stripWWW = True if normMask & UrlNormalizator.NORM_SKIP_WWW else False
        useValidator = True if normMask & UrlNormalizator.NORM_USE_VALIDATOR else False
        enableAdditionNormalize = True if normMask & UrlNormalizator.NORM_MAIN else False
        norm_url = str(urinormpath(url.strip(), stripWWW, useValidator, enableAdditionNormalize))
        # norm_url = str(canonicalize_url(url.strip()))
        # logger.debug("norm_url: <%s>", norm_url)
        # except urlnorm.InvalidUrl:
        #  logger.error("Normalization InvalidUrl")
        #  norm_url = ""
      except Exception as e:
        logger.error("Normalization error: " + str(e) + "\nURL: [" + url + "]\n" + str(getTracebackInfo()))

    return norm_url

Here is the call graph for this function:

Member Data Documentation

BAD_URL_PREFIX

string app.Utils.UrlNormalizator.BAD_URL_PREFIX = "normalization-error://?"

static

Definition at line 666 of file Utils.py.

NORM_DEFAULT

int app.Utils.UrlNormalizator.NORM_DEFAULT = NORM_MAIN

static

Definition at line 665 of file Utils.py.

NORM_MAIN

int app.Utils.UrlNormalizator.NORM_MAIN = 4

static

Definition at line 664 of file Utils.py.

NORM_NONE

int app.Utils.UrlNormalizator.NORM_NONE = 0

static

Definition at line 661 of file Utils.py.

NORM_SKIP_WWW

int app.Utils.UrlNormalizator.NORM_SKIP_WWW = 1

static

Definition at line 662 of file Utils.py.

NORM_USE_VALIDATOR

int app.Utils.UrlNormalizator.NORM_USE_VALIDATOR = 2

static

Definition at line 663 of file Utils.py.

---

The documentation for this class was generated from the following file:

• sources/hce/app/Utils.py