dc_crawler.UrlSchema.UrlSchema Class Reference

Inheritance diagram for dc_crawler.UrlSchema.UrlSchema:

Collaboration diagram for dc_crawler.UrlSchema.UrlSchema:

Public Member Functions
def	__init__ (self, schema=None, siteId=None, urlSchemaDataDir=None)
def	readJsonFile (self, fileName)
def	schemaPredefined (self, inUrl, parametrs)
def	schemaIncrementalInt (self, inUrl, parameters, maxItems)
def	replaceSchemaIncrementalInt (self, inUrl, macroName, minPos, maxPos, step)
def	schemaRandomInt (self, inUrl, parametrs)
def	schemaRandomStr (self, inUrl, parametrs)
def	saveJsonInFile (self, fileName)
def	resolveParametersByHTTP (self, urls, defaultValue=None)
def	resolveParametersByFormat (self, parameters, delimiter=' ', formatValue='json', defaultValue=None)
def	urlEncodeToParameters (self, parameters, urlEncode)
def	getMaxCountParameters (self, parameters)
def	resolveParametersFromFile (self, fileName, defaultValue=None)
def	generateUrlSchema (self, inUrl)

Public Attributes
	batchInsert
	externalError
	indexFileName
	indexStruct
	schema

Static Public Attributes
int	SCHEMA_DISABLE = 0
int	SCHEMA_PREDEFINED = 1
int	SCHEMA_INCREMENTAL_INT = 2
int	SCHEMA_RANDOM_INT = 3
int	SCHEMA_RANDOM_STR = 4
int	CHAR_ASCII_LATIN = 0
int	CHAR_HEXADECIMAL = 1
int	CHAR_LOWER = 0
int	CHAR_UPPER = 1
int	MODE_ONE_URL = 0
int	MODE_LIST_URLS = 1
int	BATCH_INSERT_NO_ONE_ITEMS = 0
int	BATCH_INSERT_ALL_NEW_ITEMS = 1
int	BATCH_INSERT_ONLY_FIRST_ITEM = 2
int	BATCH_INSERT_DEFAULT = BATCH_INSERT_NO_ONE_ITEMS
int	BATCH_INSERT_MIN_ALLOWED_VALUE = BATCH_INSERT_NO_ONE_ITEMS
int	BATCH_INSERT_MAX_ALLOWED_VALUE = BATCH_INSERT_ONLY_FIRST_ITEM
string	JSON_SUFF = ".json"
string	URL_SCHEMA_DATA_FILE_NAME_PREFIX = "url_schema_data_"

Detailed Description

Definition at line 32 of file UrlSchema.py.

Constructor & Destructor Documentation

__init__()

def dc_crawler.UrlSchema.UrlSchema.__init__	(		self,
			schema = None,
			siteId = None,
			urlSchemaDataDir = None
	)

Definition at line 62 of file UrlSchema.py.

  def __init__(self, schema=None, siteId=None, urlSchemaDataDir=None):
    self.batchInsert = self.BATCH_INSERT_DEFAULT
    self.externalError = APP_CONSTS.ERROR_OK
    self.indexFileName = None
    self.indexStruct = None
    try:
      # self.globalindex = None
      self.schema = json.loads(schema)

      if isinstance(urlSchemaDataDir, basestring):
        if not os.path.isdir(urlSchemaDataDir):
          logger.debug("Create urlSchemaDataDir: %s", str(urlSchemaDataDir))
          try:
            os.makedirs(urlSchemaDataDir)
          except OSError, err:
            logger.debug("Creation of %s return error: %s", str(urlSchemaDataDir), str(err))

        if urlSchemaDataDir[-1] != '/':
          urlSchemaDataDir += '/'
        self.indexFileName = urlSchemaDataDir + self.URL_SCHEMA_DATA_FILE_NAME_PREFIX + str(siteId) + self.JSON_SUFF
        if os.path.isfile(self.indexFileName):
          self.indexStruct = self.readJsonFile(self.indexFileName)
          logger.debug(">>> readJsonFile '" + str(self.indexFileName) + "' - SUCCESS")
        else:
          self.indexStruct = None

    except Exception as excp:
      ExceptionLog.handler(logger, excp, ">>> UrlSchema wrong json loads")
      self.schema = None

Member Function Documentation

generateUrlSchema()

def dc_crawler.UrlSchema.UrlSchema.generateUrlSchema	(		self,
			inUrl
	)

Definition at line 486 of file UrlSchema.py.

  def generateUrlSchema(self, inUrl):
    ret = []
    itemsLen = 1
    if self.schema is not None:
      try:
        if "urls" in self.schema:
          self.schema["parameters"] = self.resolveParametersByHTTP(self.schema["urls"])

        if "file_path" in self.schema:
          self.schema["parameters"] = self.resolveParametersFromFile(self.schema["file_path"], \
                                                                     self.schema["parameters"])

        # logger.debug('self.schema["parameters"]: ' + str(self.schema["parameters"]))

        if "format" in self.schema:
          delimiter = ' '
          if "delimiter" in self.schema:
            delimiter = self.schema["delimiter"]
          self.schema["parameters"] = self.resolveParametersByFormat(self.schema["parameters"], delimiter, \
                                                                         self.schema["format"], \
                                                                         self.schema["parameters"])

        if "url_encode" in self.schema:
          self.schema["parameters"] = self.urlEncodeToParameters(self.schema["parameters"], self.schema["url_encode"])

        if "batch_insert" in self.schema and \
        int(self.schema["batch_insert"]) >= self.BATCH_INSERT_MIN_ALLOWED_VALUE and \
        int(self.schema["batch_insert"]) <= self.BATCH_INSERT_MAX_ALLOWED_VALUE:
          self.batchInsert = int(self.schema["batch_insert"])

        # get max count parameters
        maxCountParameters = self.getMaxCountParameters(self.schema["parameters"])
        logger.debug('maxCountParameters: ' + str(maxCountParameters))

        if self.schema["mode"] == self.MODE_LIST_URLS:
          itemsLen = int(self.schema["max_items"])
        for _ in xrange(0, itemsLen):
          localRet = inUrl
          if self.schema["type"] == self.SCHEMA_PREDEFINED:
            localRet = self.schemaPredefined(inUrl, self.schema["parameters"])

          elif self.schema["type"] == self.SCHEMA_INCREMENTAL_INT:
            # localRet = self.schemaIncrementalInt(inUrl, self.schema["parameters"])
            # get full urls list
            ret = self.schemaIncrementalInt(inUrl, self.schema["parameters"], itemsLen)

          elif self.schema["type"] == self.SCHEMA_RANDOM_INT:
            localRet = self.schemaRandomInt(inUrl, self.schema["parameters"])

          elif self.schema["type"] == self.SCHEMA_RANDOM_STR:
            localRet = self.schemaRandomStr(inUrl, self.schema["parameters"])

          if localRet != inUrl and localRet not in ret:
            ret.append(localRet)
            if len(ret) >= int(maxCountParameters):
              logger.debug('>>> break   len(ret) = ' + str(len(ret)))
              break
          else:
            break
      except Exception as excp:
        ExceptionLog.handler(logger, excp, ">>> generateUrlSchema has some error")
      self.saveJsonInFile(self.indexFileName)
    logger.debug(">>> urlSchema len = " + str(len(ret)))
    return ret

getMaxCountParameters()

def dc_crawler.UrlSchema.UrlSchema.getMaxCountParameters	(		self,
			parameters
	)

Definition at line 435 of file UrlSchema.py.

  def getMaxCountParameters(self, parameters):
    countsList = [0]
    for values in parameters.values():
      # logger.debug('>>> values: ' + str(values))
      if isinstance(values, list):
        countsList.append(len(values))

    return max(countsList)

Here is the caller graph for this function:

readJsonFile()

def dc_crawler.UrlSchema.UrlSchema.readJsonFile	(		self,
			fileName
	)

Definition at line 97 of file UrlSchema.py.

  def readJsonFile(self, fileName):
    ret = {}
    fd = None
    try:
      fd = open(fileName, "r")      if fd is not None:
        ret = json.loads(fd.read())  # #.decode('utf-8').encode('latin-1', errors='ignore'))
        fd.close()
    except Exception, err:
      logger.debug(">>> readJsonFile error, file name = " + str(fileName) + " | " + str(err))
      if fd is not None:
        fd.close()
    return ret

replaceSchemaIncrementalInt()

def dc_crawler.UrlSchema.UrlSchema.replaceSchemaIncrementalInt	(		self,
			inUrl,
			macroName,
			minPos,
			maxPos,
			step
	)

Definition at line 256 of file UrlSchema.py.

  def replaceSchemaIncrementalInt(self, inUrl, macroName, minPos, maxPos, step):
    # variable for result
    ret = []
    localRet = inUrl
    for x in range(minPos, maxPos, step):
      localUrl = copy.copy(inUrl)
      logger.debug("Before replace inUrl = " + str(localUrl))
      localUrl = localUrl.replace(macroName, str(x))
      logger.debug("After replace inUrl = " + str(localUrl))

      if localRet != localUrl and localUrl not in ret:
        ret.append(localUrl)

    return ret

Here is the caller graph for this function:

resolveParametersByFormat()

def dc_crawler.UrlSchema.UrlSchema.resolveParametersByFormat	(		self,
			parameters,
			delimiter = ' ',
			formatValue = 'json',
			defaultValue = None
	)

Definition at line 366 of file UrlSchema.py.

  def resolveParametersByFormat(self, parameters, delimiter=' ', formatValue='json', defaultValue=None):
    # variable for result
    ret = defaultValue
    logger.debug('!!! parameters: ' + str(parameters))

    if formatValue == 'plain-text':
      for paramName in parameters:
        logger.debug("paramName: '" + str(paramName) + "' type: " + str(type(paramName)))
        logger.debug("paramValue: '" + str(parameters[paramName]) + "' type: " + str(type(parameters[paramName])))

        if isinstance(parameters[paramName], basestring):
          if delimiter == "":
            # split don't use delimiter
            parameters[paramName] = unicode(parameters[paramName]).splitlines()
          else:
            # split use delimiter
            parameters[paramName] = unicode(parameters[paramName]).split(delimiter)

          # remove empty strings from list
          parameters[paramName] = [elem for elem in parameters[paramName] if elem]

      ret = parameters
      logger.debug('!!! ret: ' + str(ret))

    elif formatValue == 'json':
      ret = parameters
    else:
      logger.error("Unsupported format value: '" + str(formatValue) + "'")

    return ret

Here is the caller graph for this function:

resolveParametersByHTTP()

def dc_crawler.UrlSchema.UrlSchema.resolveParametersByHTTP	(		self,
			urls,
			defaultValue = None
	)

Definition at line 333 of file UrlSchema.py.

  def resolveParametersByHTTP(self, urls, defaultValue=None):
    if defaultValue is None:
      ret = {}
    else:
      ret = defaultValue
    newParams = None
    for url in urls:
      result = None
      try:
        result = requests.get(url)
      except Exception as excp:
        self.externalError = APP_CONSTS.ERROR_URLS_SCHEMA_EXTERNAL
        logger.debug(">>> bad url request; url=" + url + ";err= " + str(excp))
      if result is not None and result.status_code == 200 and result.text is not None:
        try:
          newParams = json.loads(result.text)
        except Exception as excp:
          self.externalError = APP_CONSTS.ERROR_URLS_SCHEMA_EXTERNAL
          logger.debug(">>> bad external parameters json" + str(excp))
      if newParams is not None:
        ret = newParams
        self.externalError = APP_CONSTS.ERROR_OK
        break
    return ret

Here is the caller graph for this function:

resolveParametersFromFile()

def dc_crawler.UrlSchema.UrlSchema.resolveParametersFromFile	(		self,
			fileName,
			defaultValue = None
	)

Definition at line 450 of file UrlSchema.py.

  def resolveParametersFromFile(self, fileName, defaultValue=None):
    logger.debug(">>> resolveParametersFromFile enter  fileName: " + str(fileName))
    # variable for result
    ret = defaultValue
    parameters = {}

    if fileName.find(self.JSON_SUFF) == len(fileName) - len(self.JSON_SUFF):

      fd = None
      try:
        fd = open(fileName, "r")
        if fd is not None:
          buff = fd.read()
          if len(buff) > 0 and buff[0] == '{':  # maybe 'json'
            parameters = json.loads(buff)
          else:
            parameters = {"":buff}  # maybe 'plain-text'

      except Exception, err:
        logger.debug(">>> resolveParametersFromFile error, file name = " + str(fileName) + " | " + str(err))
      finally:
        if fd is not None:
          fd.close()

      if len(parameters) > 0:
        ret = parameters
    else:
      logger.debug("Wrong file name: '" + str(fileName) + "', expected  '<file_name>.json'")

    return ret

Here is the caller graph for this function:

saveJsonInFile()

def dc_crawler.UrlSchema.UrlSchema.saveJsonInFile	(		self,
			fileName
	)

Definition at line 314 of file UrlSchema.py.

  def saveJsonInFile(self, fileName):
    if self.indexStruct is not None and len(self.indexStruct) > 0 and fileName is not None:
      fd = None
      try:
        fd = open(fileName, "w")
        if fd is not None:
          fd.write(json.dumps(self.indexStruct, ensure_ascii=False))
          fd.close()
      except Exception, err:
        ExceptionLog.handler(logger, err, ">>> saveJsonInFile error, file name = " + str(fileName))
        if fd is not None:
          fd.close()

Here is the caller graph for this function:

schemaIncrementalInt()

def dc_crawler.UrlSchema.UrlSchema.schemaIncrementalInt	(		self,
			inUrl,
			parameters,
			maxItems
	)

Definition at line 176 of file UrlSchema.py.

  def schemaIncrementalInt(self, inUrl, parameters, maxItems):
    logger.debug("schemaIncrementalInt() enter ... parameters: " + str(parameters) + "\ninUrl: " + str(inUrl) + \
                 "\nmaxItems: " + str(maxItems))
    # variable for result
    ret = []
    # localRet = inUrl
    for paramKey in parameters:
      macroName = '%' + paramKey + '%'
      if inUrl.find(macroName) >= 0:
#        index = parameters[paramKey]["min"]
#        logger.debug("index = " + str(index))
#        if self.indexStruct is not None and paramKey in self.indexStruct:
#          index = self.indexStruct[paramKey]
#          logger.debug("index = " + str(index))
#
#        if index >= parameters[paramKey]["min"] and index <= parameters[paramKey]["max"]:
#          logger.debug("Before replace inUrl = " + str(inUrl))
#          inUrl = inUrl.replace(macroName, str(index))
#          logger.debug("After replace inUrl = " + str(inUrl))
#        else:
#          logger.debug("!!! continue !!!")
#          continue
#
#        logger.debug("self.indexStruct: " + varDump(self.indexStruct))
#        if self.indexStruct is not None:
#          logger.debug("Old index = " + str(index))
#          index += parameters[paramKey]["step"]
#          if index > parameters[paramKey]["max"]:
#            index = parameters[paramKey]["min"]
#          self.indexStruct[paramKey] = index
#          logger.debug("New index = " + str(index))

        if maxItems > int(parameters[paramKey]["max"]):
          ret = self.replaceSchemaIncrementalInt(inUrl,
                                                 macroName,
                                                 int(parameters[paramKey]["min"]),
                                                 int(parameters[paramKey]["max"]),
                                                 int(parameters[paramKey]["step"]))
        else:
          logger.debug("Start self.indexStruct: %s", varDump(self.indexStruct))
          minPos = 0
          if self.indexStruct is not None and paramKey in self.indexStruct:
            minPos = int(self.indexStruct[paramKey])
            logger.debug("minPos from structure = " + str(minPos))
          else:
            minPos = int(parameters[paramKey]["min"])

          nextPos = maxItems * int(parameters[paramKey]["step"]) + minPos
          if nextPos >= int(parameters[paramKey]["max"]):
            nextPos = int(parameters[paramKey]["max"])

          ret = self.replaceSchemaIncrementalInt(inUrl,
                                                 macroName,
                                                 minPos,
                                                 nextPos,
                                                 int(parameters[paramKey]["step"]))

          if self.indexStruct is None:
            self.indexStruct = {}

          logger.debug("nextPos = " + str(nextPos))
          if nextPos >= int(parameters[paramKey]["max"]):
            nextPos = 0

          logger.debug("nextIndex after truncate = " + str(nextPos))
          self.indexStruct[paramKey] = nextPos

        logger.debug("Finish self.indexStruct: %s", varDump(self.indexStruct))

    return ret

Here is the call graph for this function:

Here is the caller graph for this function:

schemaPredefined()

def dc_crawler.UrlSchema.UrlSchema.schemaPredefined	(		self,
			inUrl,
			parametrs
	)

Definition at line 117 of file UrlSchema.py.

  def schemaPredefined(self, inUrl, parametrs):
    # logger.debug('schemaPredefined enter  parametrs: ' + str(parametrs))
    for paramKey in parametrs:
      macroName = '%' + paramKey + '%'
      if inUrl.find(macroName) >= 0:

        paramList = []
        frequencyList = []
        timeList = []
        elements = {}

        if self.indexStruct is not None and paramKey in self.indexStruct:
          elements.update(self.indexStruct[paramKey])
          # logger.debug('elements1: ' + str(elements))
          for val in parametrs[paramKey]:
            if val not in self.indexStruct[paramKey]:
              elements.update({val:{"frequency":0, "time":0}})
              self.indexStruct[paramKey].update(elements)
          # logger.debug('elements3: ' + str(elements))
        else:
          for val in parametrs[paramKey]:
            elements.update({val:{"frequency":0, "time":0}})
          # logger.debug('elements2: ' + str(elements))
          self.indexStruct = {paramKey:elements}

        for key, element in elements.items():
          if "frequency" in element and "time" in element:
            paramList.append(key)
            frequencyList.append(int(element["frequency"]))
            timeList.append(int(element["time"]))

          # logger.debug('key: ' + str(key))
          # logger.debug('element: ' + str(element))

        mixIndex = frequencyList.index(min(frequencyList))
        logger.debug('mixIndex: ' + str(mixIndex))
        logger.debug('paramList[mixIndex]: ' + str(paramList[mixIndex]))

        logger.debug('>>> inUrl 1: ' + str(inUrl))
        inUrl = unicode(inUrl.replace(macroName, paramList[mixIndex]))
        logger.debug('>>> inUrl 2: ' + str(inUrl))
        frequencyList[mixIndex] += 1
        timeList[mixIndex] = int((datetime.now() - datetime.fromtimestamp(0)).total_seconds())

        # logger.debug('>>> self.indexStruct: ' + str(self.indexStruct))
        # logger.debug('self.indexStruct[paramKey] 1: ' + str(self.indexStruct[paramKey]))
        self.indexStruct[paramKey].update({paramList[mixIndex]:{"frequency":frequencyList[mixIndex], \
                                                              "time":timeList[mixIndex]}})
        # logger.debug('self.indexStruct[paramKey] 2: ' + str(self.indexStruct[paramKey]))

    return inUrl

Here is the caller graph for this function:

schemaRandomInt()

def dc_crawler.UrlSchema.UrlSchema.schemaRandomInt	(		self,
			inUrl,
			parametrs
	)

Definition at line 277 of file UrlSchema.py.

  def schemaRandomInt(self, inUrl, parametrs):
    for paramKey in parametrs:
      macroName = '%' + paramKey + '%'
      if inUrl.find(macroName) >= 0:
        inUrl = inUrl.replace(macroName, str(random.randint(parametrs[paramKey]["min"], parametrs[paramKey]["max"])))
    return inUrl

Here is the caller graph for this function:

schemaRandomStr()

def dc_crawler.UrlSchema.UrlSchema.schemaRandomStr	(		self,
			inUrl,
			parametrs
	)

Definition at line 290 of file UrlSchema.py.

  def schemaRandomStr(self, inUrl, parametrs):
    lowAsciiSet = string.ascii_lowercase
    hexdigitsSet = ''.join([ch for ch in string.hexdigits if not ch.isupper()])
    for paramKey in parametrs:
      macroName = '%' + paramKey + '%'
      if inUrl.find(macroName) >= 0:
        valueLen = random.randint(parametrs[paramKey]["min"], parametrs[paramKey]["max"])
        valueStr = ''
        for _ in xrange(0, valueLen):
          if parametrs[paramKey]["chars"] == self.CHAR_ASCII_LATIN:
            valueStr += lowAsciiSet[random.randint(0, len(lowAsciiSet) - 1)]
          elif parametrs[paramKey]["chars"] == self.CHAR_HEXADECIMAL:
            valueStr += hexdigitsSet[random.randint(0, len(hexdigitsSet) - 1)]
        if parametrs[paramKey]["case"] == self.CHAR_LOWER:
          valueStr = valueStr.lower()
        elif parametrs[paramKey]["case"] == self.CHAR_UPPER:
          valueStr = valueStr.upper()
        inUrl = inUrl.replace(macroName, valueStr)
    return inUrl

Here is the caller graph for this function:

urlEncodeToParameters()

def dc_crawler.UrlSchema.UrlSchema.urlEncodeToParameters	(		self,
			parameters,
			urlEncode
	)

Definition at line 403 of file UrlSchema.py.

  def urlEncodeToParameters(self, parameters, urlEncode):
    # variable for result
    ret = parameters
    # logger.debug('>>>>> parameters: ' + str(parameters))

    if urlEncode is not None and int(urlEncode) > 0:
      for paramName in parameters:
        if isinstance(parameters[paramName], list) or isinstance(parameters[paramName], unicode):
          paramsList = []
          for elem in parameters[paramName]:
            if isinstance(elem, str) or isinstance(elem, unicode):
              try:
                encodedStr = urllib.urlencode({'':elem})
                if len(encodedStr) > 0 and encodedStr[0] == '=':
                  encodedStr = encodedStr[1:]
                paramsList.append(encodedStr)
              except Exception, err:
                logger.debug("urlencode '" + str(elem) + "' has error: " + str(err))
                paramsList.append(unicode(elem))

          parameters[paramName] = paramsList

      ret = parameters
      # logger.debug('>>>>> ret: ' + str(ret))

    return ret

Here is the caller graph for this function:

Member Data Documentation

BATCH_INSERT_ALL_NEW_ITEMS

int dc_crawler.UrlSchema.UrlSchema.BATCH_INSERT_ALL_NEW_ITEMS = 1

static

Definition at line 49 of file UrlSchema.py.

BATCH_INSERT_DEFAULT

int dc_crawler.UrlSchema.UrlSchema.BATCH_INSERT_DEFAULT = BATCH_INSERT_NO_ONE_ITEMS

static

Definition at line 51 of file UrlSchema.py.

BATCH_INSERT_MAX_ALLOWED_VALUE

int dc_crawler.UrlSchema.UrlSchema.BATCH_INSERT_MAX_ALLOWED_VALUE = BATCH_INSERT_ONLY_FIRST_ITEM

static

Definition at line 53 of file UrlSchema.py.

BATCH_INSERT_MIN_ALLOWED_VALUE

int dc_crawler.UrlSchema.UrlSchema.BATCH_INSERT_MIN_ALLOWED_VALUE = BATCH_INSERT_NO_ONE_ITEMS

static

Definition at line 52 of file UrlSchema.py.

BATCH_INSERT_NO_ONE_ITEMS

int dc_crawler.UrlSchema.UrlSchema.BATCH_INSERT_NO_ONE_ITEMS = 0

static

Definition at line 48 of file UrlSchema.py.

BATCH_INSERT_ONLY_FIRST_ITEM

int dc_crawler.UrlSchema.UrlSchema.BATCH_INSERT_ONLY_FIRST_ITEM = 2

static

Definition at line 50 of file UrlSchema.py.

batchInsert

dc_crawler.UrlSchema.UrlSchema.batchInsert

Definition at line 63 of file UrlSchema.py.

CHAR_ASCII_LATIN

int dc_crawler.UrlSchema.UrlSchema.CHAR_ASCII_LATIN = 0

static

Definition at line 40 of file UrlSchema.py.

CHAR_HEXADECIMAL

int dc_crawler.UrlSchema.UrlSchema.CHAR_HEXADECIMAL = 1

static

Definition at line 41 of file UrlSchema.py.

CHAR_LOWER

int dc_crawler.UrlSchema.UrlSchema.CHAR_LOWER = 0

static

Definition at line 42 of file UrlSchema.py.

CHAR_UPPER

int dc_crawler.UrlSchema.UrlSchema.CHAR_UPPER = 1

static

Definition at line 43 of file UrlSchema.py.

externalError

dc_crawler.UrlSchema.UrlSchema.externalError

Definition at line 64 of file UrlSchema.py.

indexFileName

dc_crawler.UrlSchema.UrlSchema.indexFileName

Definition at line 65 of file UrlSchema.py.

indexStruct

dc_crawler.UrlSchema.UrlSchema.indexStruct

Definition at line 66 of file UrlSchema.py.

JSON_SUFF

string dc_crawler.UrlSchema.UrlSchema.JSON_SUFF = ".json"

static

Definition at line 55 of file UrlSchema.py.

MODE_LIST_URLS

int dc_crawler.UrlSchema.UrlSchema.MODE_LIST_URLS = 1

static

Definition at line 46 of file UrlSchema.py.

MODE_ONE_URL

int dc_crawler.UrlSchema.UrlSchema.MODE_ONE_URL = 0

static

Definition at line 45 of file UrlSchema.py.

schema

dc_crawler.UrlSchema.UrlSchema.schema

Definition at line 69 of file UrlSchema.py.

SCHEMA_DISABLE

int dc_crawler.UrlSchema.UrlSchema.SCHEMA_DISABLE = 0

static

Definition at line 34 of file UrlSchema.py.

SCHEMA_INCREMENTAL_INT

int dc_crawler.UrlSchema.UrlSchema.SCHEMA_INCREMENTAL_INT = 2

static

Definition at line 36 of file UrlSchema.py.

SCHEMA_PREDEFINED

int dc_crawler.UrlSchema.UrlSchema.SCHEMA_PREDEFINED = 1

static

Definition at line 35 of file UrlSchema.py.

SCHEMA_RANDOM_INT

int dc_crawler.UrlSchema.UrlSchema.SCHEMA_RANDOM_INT = 3

static

Definition at line 37 of file UrlSchema.py.

SCHEMA_RANDOM_STR

int dc_crawler.UrlSchema.UrlSchema.SCHEMA_RANDOM_STR = 4

static

Definition at line 38 of file UrlSchema.py.

URL_SCHEMA_DATA_FILE_NAME_PREFIX

string dc_crawler.UrlSchema.UrlSchema.URL_SCHEMA_DATA_FILE_NAME_PREFIX = "url_schema_data_"

static

Definition at line 56 of file UrlSchema.py.

---

The documentation for this class was generated from the following file:

• sources/hce/dc_crawler/UrlSchema.py

• dc_crawler
• UrlSchema
• UrlSchema
• Generated on Fri Nov 24 2017 18:54:35 for HCE Project Python language Distributed Tasks Manager Application, Distributed Crawler Application and client API bindings. by 1.8.13