dc_crawler.Fetcher.SeleniumFetcher Class Reference

Inheritance diagram for dc_crawler.Fetcher.SeleniumFetcher:

Collaboration diagram for dc_crawler.Fetcher.SeleniumFetcher:

Public Member Functions
def	__init__ (self, tmpDirOptions=None, log=None)
def	__del__ (self)
def	open (self, url, method='get', headers=None, timeout=DEFAUIL_TIMEOUT, allow_redirects=True, proxies=None, auth=None, data=None, log=None, allowed_content_types=None, max_resource_size=None, max_redirects=1, filters=None, executable_path=None, depth=None, macro=None)
def	openT (self, url, headers, timeout, proxies, executable_path, macro)
def	cleanup (self, state=0, headers=None)
def	chromeProcessesCleanup (self, headers)
def	killProcess (self, pid, dirsTemplate=CHROME_DIRS_TEMPLATE, dirDeleteBeforeTimeout=DELAY_TERMINATE_AND_QUIT)
def	should_have_meta_res (self)
def	getProcessDirs (self, process, dirsTemplate)
def	getOptions (self, webdriver, headers, proxies, url)
def	initializeTmpDirs (self, headers)
def	removeTmpDirs (self, delay=DELAY_TERMINATE_AND_QUIT, tries=3)
def	execMacroSimple (self, macro)
def	getAllLogsAsString (self)
def	execMacroExtended (self, macro)
Public Member Functions inherited from dc_crawler.Fetcher.BaseFetcher
def	__init__ (self)
def	open (self, url, method='get', headers=None, timeout=100, allow_redirects=True, proxies=None, auth=None, data=None, log=None, allowed_content_types=None, max_resource_size=None, max_redirects=CONSTS.MAX_HTTP_REDIRECTS_LIMIT, filters=None, executable_path=None, depth=None, macro=None)
def	should_have_meta_res (self)
def	getDomainNameFromURL (self, url, default='')

Public Attributes
	logger
	tmpDirPath
	tmpDirPrefix
	tmpDirSuffix
	tmpDirType
	tmpDirRemoveBeforeCreate
	tmpDir
	driver
	driverPid
	inlineURLMacroDelimiter
	sessionId
	userDataDirUsed
Public Attributes inherited from dc_crawler.Fetcher.BaseFetcher
	connectionTimeout
	logger

Static Public Attributes
int	DEFAUIL_TIMEOUT = 5
string	CONTENT_TYPE_JSON = 'text/json'
string	CONTENT_TYPE_HTML = 'text/html'
float	DELAY_TERMINATE_AND_QUIT = 0.5
int	ERROR_FATAL = 1
int	ERROR_GENERAL = 2
int	ERROR_CONTENT_OR_COOKIE = 3
int	ERROR_NAME_NOT_RESOLVED = 400
int	ERROR_TOO_MANY_REDIRECTS = 11
int	ERROR_MACRO_RETURN_VALUE = 12
int	ERROR_PROXY_CONNECTION_FAILED = 504
int	ERROR_CONNECTION_TIMED_OUT = 505
int	ERROR_TUNNEL_CONNECTION_FAILED = 403
int	ERROR_SERVICE_UNAVAILABLE = 503
int	ERROR_CONFLICT = 409
int	ERROR_EMPTY_RESPONSE = 13
string	LOG_MESSAGE_RENDERRER_TIMEOUT = 'Timed out receiving message from renderer'
string	LOG_MESSAGE_SERVER_RESPONSE_503 = 'server responded with a status of 503'
string	LOG_MESSAGE_SERVER_RESPONSE_409 = 'server responded with a status of 409 (Conflict)'
list	CHROME_PROCESS_NAMES = ['chrome', 'BrowserBlocking']
string	CHROME_DIRS_TEMPLATE = '.google.Chrome.'
string	CHROME_DEBUG_LOG_NAME = 'chrome_debug.log'
int	MACRO_RESULT_TYPE_DEFAULT = 0
int	MACRO_RESULT_TYPE_URLS_LIST = 1
int	MACRO_RESULT_TYPE_CONTENT = 2
int	MACRO_RESULT_TYPE_AUTO = 3
int	TMP_DIR_TYPE_OPEN = 0
int	TMP_DIR_TYPE_INSTANTIATE = 1
Static Public Attributes inherited from dc_crawler.Fetcher.BaseFetcher
	fetchers = None
int	TYP_NORMAL = 1
int	TYP_DYNAMIC = 2
int	TYP_URLLIB = 5
int	TYP_CONTENT = 6
int	TYP_AUTO = 7
float	CONNECTION_TIMEOUT = 1.0

Additional Inherited Members
Static Public Member Functions inherited from dc_crawler.Fetcher.BaseFetcher
def	init (dbWrapper=None, siteId=None)
def	get_fetcher (typ, dbWrapper=None, siteId=None)

Detailed Description

Definition at line 397 of file Fetcher.py.

Constructor & Destructor Documentation

__init__()

def dc_crawler.Fetcher.SeleniumFetcher.__init__	(		self,
			tmpDirOptions = None,
			log = None
	)

Definition at line 437 of file Fetcher.py.

  def __init__(self, tmpDirOptions=None, log=None):
    super(SeleniumFetcher, self).__init__()

    if log is not None:
      self.logger = log

    if self.logger:
      self.logger.debug("Initialization of instance, tmpDirOptions: %s", str(tmpDirOptions))

    self.tmpDirPath = '/tmp'
    self.tmpDirPrefix = 'dfetcher_tmp_%PID%'
    self.tmpDirSuffix = ''
    self.tmpDirType = self.TMP_DIR_TYPE_OPEN
    self.tmpDirRemoveBeforeCreate = True
    if tmpDirOptions is not None:
      if 'path' in tmpDirOptions:
        self.tmpDirPath = tmpDirOptions['path']
      if 'prefix' in tmpDirOptions:
        self.tmpDirPrefix = tmpDirOptions['prefix']
      if 'suffix' in tmpDirOptions:
        self.tmpDirSuffix = tmpDirOptions['suffix']
      if 'type' in tmpDirOptions:
        self.tmpDirType = int(tmpDirOptions['type'])
      if 'remove_before_create' in tmpDirOptions:
        self.tmpDirRemoveBeforeCreate = bool(int(tmpDirOptions['remove_before_create']))
    pid = str(os.getpid()).strip()

    if self.tmpDirPath == '' and self.tmpDirPrefix == '' and self.tmpDirSuffix == '':
      self.tmpDir = ''
    else:
      self.tmpDir = self.tmpDirPath + '/' + self.tmpDirPrefix.replace('%PID%', pid) + \
                    self.tmpDirSuffix.replace('%PID%', pid)
    if self.tmpDirType == self.TMP_DIR_TYPE_INSTANTIATE:
      if not self.initializeTmpDirs(None):
        msg = 'Temporary directory type INSTANTIATE `%s` initialization error!', self.tmpDir
        if self.logger is not None:
          self.logger.error(msg)
        raise SeleniumFetcherException(msg)
      else:
        if self.logger is not None:
          self.logger.debug("Temporary directory type INSTANTIATE `%s` initialized!", self.tmpDir)
    self.driver = None
    self.driverPid = 0
    self.inlineURLMacroDelimiter = '###'
    self.sessionId = '--sessionId=' + str(pid)
    self.userDataDirUsed = ''

__del__()

def dc_crawler.Fetcher.SeleniumFetcher.__del__

(

self

)

Definition at line 487 of file Fetcher.py.

  def __del__(self):
    if self.logger:
      self.logger.debug("Delete instance, temporary dir type: %s", str(self.tmpDirType))

    if self.tmpDirType == self.TMP_DIR_TYPE_INSTANTIATE:
      self.removeTmpDirs()

Here is the call graph for this function:

Member Function Documentation

chromeProcessesCleanup()

def dc_crawler.Fetcher.SeleniumFetcher.chromeProcessesCleanup	(		self,
			headers
	)

Definition at line 960 of file Fetcher.py.

  def chromeProcessesCleanup(self, headers):
    if self.logger:
      self.logger.debug("Chrome processes cleanup started")

    if self.sessionId != '':
      key = self.sessionId
    else:
      if self.tmpDir == '':
        if '--disk-cache-dir' in headers:
          key = '--disk-cache-dir=' + headers['--disk-cache-dir']
        if '--profile-directory' in headers:
          key = '--profile-directory=' + headers['--profile-directory']
        if '--user-data-dir' in headers:
          key = '--user-data-dir=' + headers['--user-data-dir']
      else:
        key = self.tmpDir

    try:
      for proc in psutil.process_iter():
        try:
          # if self.logger:
          #  self.logger.debug("Candidate, pid:%s, name: %s cmdline: %s", str(proc.pid), str(proc.name()),
          #                    str(proc.cmdline()))
          for name in self.CHROME_PROCESS_NAMES:
            if name in proc.name():
              found = False
              for item in proc.cmdline():
                if key in item:
                  found = True
                  break
              if found:
                if self.logger:
                  self.logger.debug("Chrome process killing, pid:%s, cmdline: %s", str(proc.pid), str(proc.cmdline()))
                self.killProcess(proc.pid, self.CHROME_DIRS_TEMPLATE, self.DELAY_TERMINATE_AND_QUIT)
        except Exception, err:
          if self.logger:
            self.logger.debug("Chrome process kill error: %s", str(err))
    except Exception, err:
      if self.logger:
        self.logger.debug("Chrome process kill error: %s", str(err))

Here is the call graph for this function:

Here is the caller graph for this function:

cleanup()

def dc_crawler.Fetcher.SeleniumFetcher.cleanup	(		self,
			state = 0,
			headers = None
	)

Definition at line 915 of file Fetcher.py.

  def cleanup(self, state=0, headers=None):
    if self.logger is not None and '--log-chrome-debug-log' in headers:
      logFile = self.userDataDirUsed + '/' + self.CHROME_DEBUG_LOG_NAME
      try:
        with open(logFile, 'r') as f:
          logData = f.read()
        self.logger.debug("Chrome debug log file `%s`:\n%s", logFile, logData)
      except Exception, err:
        self.logger.debug("Error read chrome debug log file `%s`: %s", logFile, str(err))

    if self.logger is not None:
      self.logger.debug("Cleanup type: %s, driver: %s", str(state), str(self.driver))

    try:
      if self.driver is not None:
        self.driver.quit()
        time.sleep(self.DELAY_TERMINATE_AND_QUIT)
    except Exception:
      pass

    if state == 1:
      time.sleep(self.DELAY_TERMINATE_AND_QUIT)
      try:
        if self.driver is not None:
          self.driver.quit()
      except Exception:
        pass
      time.sleep(self.DELAY_TERMINATE_AND_QUIT)
      try:
        if self.logger:
          self.logger.debug("Driver pid: " + str(self.driverPid))
        self.killProcess(self.driverPid)
      except Exception:
        if self.logger:
          self.logger.debug("Error kill driver pid: %s", str(self.driverPid))

    self.chromeProcessesCleanup(headers)

    if self.tmpDirType == self.TMP_DIR_TYPE_OPEN:
      self.removeTmpDirs(self.DELAY_TERMINATE_AND_QUIT)

Here is the call graph for this function:

Here is the caller graph for this function:

execMacroExtended()

def dc_crawler.Fetcher.SeleniumFetcher.execMacroExtended	(		self,
			macro
	)

Definition at line 1446 of file Fetcher.py.

  def execMacroExtended(self, macro):
    macroResults = []
    error_code = 0
    error_msg = ''
    content_type = None
    result_type = self.MACRO_RESULT_TYPE_DEFAULT

    for mset in macro['sets']:
      if 'name' not in mset:
        mset['name'] = ''
      if 'repeat' not in mset:
        mset['repeat'] = '1'
      if 'delay' not in mset:
        mset['delay'] = '0'
      if self.logger is not None:
        self.logger.debug("Set:\n%s", str(mset))
      for i in xrange(0, int(mset['repeat'])):
        if int(mset['delay']) > 0:
          time.sleep(int(mset['delay']))
        if self.logger is not None:
          self.logger.debug("Macro %s in set", str(i))
        r, error_code, error_msg = self.execMacroSimple(mset['items'])
        if error_code > 0:
          break
        macroResults += r
      if error_code > 0:
        break

    if 'result_type' in macro:
      result_type = int(macro['result_type'])
    self.logger.debug("Macro results type: %s", str(result_type))
    if 'result_content_type' in macro:
      content_type = str(macro['result_content_type'])
    self.logger.debug("Macro results content type: %s", str(content_type))

    if result_type == self.MACRO_RESULT_TYPE_AUTO:
      self.logger.debug("Macro results before autodetect type: %s", str(macroResults))
      for r in macroResults:
        if isinstance(r, basestring):
          result_type = self.MACRO_RESULT_TYPE_CONTENT
          self.logger.debug("Macro results type autodetected as string content")
          break
        elif isinstance(r, list):
          for ri in r:
            if isinstance(ri, basestring):
              result_type = self.MACRO_RESULT_TYPE_URLS_LIST
              self.logger.debug("Macro results type autodetected as URLs list")
              break
      if result_type == self.MACRO_RESULT_TYPE_CONTENT:
        macroResults = ''.join(macroResults)  # pylint: disable=R0204
      if result_type == self.MACRO_RESULT_TYPE_URLS_LIST:
        macroResults = [item for sublist in macroResults for item in sublist]
      self.logger.debug("Macro results after autodetect type: %s", str(macroResults))

    return macroResults, error_code, error_msg, content_type, result_type



# # external Fetcher
#
#

# # urllib Fetcher
#
#

Here is the call graph for this function:

Here is the caller graph for this function:

execMacroSimple()

def dc_crawler.Fetcher.SeleniumFetcher.execMacroSimple	(		self,
			macro
	)

Definition at line 1336 of file Fetcher.py.

  def execMacroSimple(self, macro):
    macroResults = []
    error_code = 0
    error_msg = ''
    macroCounter = 0
    maxLenToLog = 512

    for m in macro:
      if self.logger is not None:
        self.logger.debug("Macro #%s in set of %s items:\n%s...",
                          str(macroCounter), str(len(macro)), str(m)[:maxLenToLog])
      macroCounter += 1
      if m.isdigit():
        if self.logger is not None:
          self.logger.debug("Macro sleep: %s sec", str(m))
        time.sleep(int(m))
      else:
        iType = 0
        iDelay = 0
        iMaxIterations = 1
        if m.startswith('!'):
          m = m[1:]
          iType = 1
          params = m.split(':')
          if len(params) > 2:
            iDelay = int(params[0])
            iMaxIterations = int(params[1])
            m = params[2]
          elif len(params) > 1:
            iDelay = int(params[0])
            m = params[1]
          elif len(params) == 1:
            m = params[0]
          if self.logger is not None:
            self.logger.debug("Macro blocking iterative, delay: %s, max ierations: %s",
                              str(iDelay), str(iMaxIterations))
        for i in xrange(0, iMaxIterations):
          if iType == 1:
            if self.logger is not None:
              self.logger.debug("Macro blocking iteration: %s of: %s", str(i + 1), str(iMaxIterations))
          if m.startswith('http://') or m.startswith('https://') or m.startswith('file://'):
            try:
              if m.startswith('file://'):
                with open(m[7:].replace('%PID%', str(os.getpid())), 'r') as f:
                  m = f.read()
              else:
                r = requests.get(m.replace('%PID%', str(os.getpid())))
                m = r.text
              if self.logger is not None:
                self.logger.debug("Macro %s bytes loaded:\n%s...", str(len(str(m))), str(m)[:maxLenToLog])
            except Exception, err:
              error_msg = 'Error load macro code, URL: `' + str(m) + '` : ' + str(err)
              error_code = self.ERROR_MACRO_RETURN_VALUE
              if self.logger is not None:
                self.logger.debug(error_msg)
              r = None
              break
          try:
            r = self.driver.execute_script(m)
            if self.logger is not None:
              self.logger.debug("Macro returned: %s", json.dumps(r))
          except Exception, err:
            error_msg = 'Error macro execution: ' + str(err) + '; logs: ' + self.getAllLogsAsString()
            error_code = self.ERROR_MACRO_RETURN_VALUE
            if self.logger is not None:
              self.logger.debug(error_msg)
            r = None
            break
          if iType == 0 and r is not None:
            if isinstance(r, (basestring, list, dict)):
              macroResults.append(r)
              if isinstance(r, (list, dict)):
                if self.logger is not None:
                  self.logger.debug("Macro items returned: %s", str(len(r)))
            else:
              error_msg = 'Error macro result value, type is: ' + str(type(r))
              error_code = self.ERROR_MACRO_RETURN_VALUE
              if self.logger is not None:
                self.logger.debug(error_msg)
              break
          elif iType == 1:
            if r is True:
              if self.logger is not None:
                self.logger.debug("Macro blocking got `True` on iteration: %s, sleeped: %s sec",
                                  str(i + 1), str(int(iDelay) * i))
              break
            elif r is not True and iDelay > 0:
              if self.logger is not None:
                self.logger.debug("Macro blocking iteration: %s sleep on: %s sec", str(i + 1), str(iDelay))
              time.sleep(int(iDelay))
        if iType == 1 and r is not True:
          if self.logger is not None:
            self.logger.debug("Macro blocking finished, but no `True` value returned!")
        if error_code > 0:
          break

    return macroResults, error_code, error_msg

Here is the call graph for this function:

Here is the caller graph for this function:

getAllLogsAsString()

def dc_crawler.Fetcher.SeleniumFetcher.getAllLogsAsString

(

self

)

Definition at line 1438 of file Fetcher.py.

  def getAllLogsAsString(self):
    return 'browser: ' + str(self.driver.get_log('browser')) + '; driver: ' + str(self.driver.get_log('driver'))

Here is the caller graph for this function:

getOptions()

def dc_crawler.Fetcher.SeleniumFetcher.getOptions	(		self,
			webdriver,
			headers,
			proxies,
			url
	)

Definition at line 1076 of file Fetcher.py.

  def getOptions(self, webdriver, headers, proxies, url):
    chrome_option = webdriver.ChromeOptions()

    arg_disable_http_cache = "--disable-http-cache"
    arg_clear_data_reduction_proxy_data_savings = '--clear-data-reduction-proxy-data-savings'
    arg_host_resolver_retry_attempts = '--host-resolver-retry-attempts=0'
    arg_start_maximized = '--start-maximized'
    if headers is not None and '--use-mobile-user-agent' in headers:
      use_mobile_user_agent = '--use-mobile-user-agent'
    else:
      use_mobile_user_agent = None

    if headers is not None and '--disable-web-security' in headers:
      disable_web_security = '--disable-web-security'
    else:
      disable_web_security = None
    # disable_web_security = '--disable-web-security'

    if headers is not None and '--allow-running-insecure-content' in headers:
      allow_running_insecure_content = '--allow-running-insecure-content'
    else:
      allow_running_insecure_content = None
    # allow_running_insecure_content = '--allow-running-insecure-content'

    if headers is not None and '--allow-file-access-from-files' in headers:
      allow_file_access_from_files = '--allow-file-access-from-files'
    else:
      allow_file_access_from_files = None
    # allow_file_access_from_files = '--allow-file-access-from-files'

    if headers is not None and '--proxy-bypass-list' in headers:
      arg_proxy_bypass_list = '--proxy-bypass-list=' + headers['--proxy-bypass-list']
    else:
      arg_proxy_bypass_list = None

    # if headers is not None and 'User-Agent' in headers and '--user-agent' in headers and\
    if headers is not None and 'User-Agent' in headers and\
      '--use-mobile-user-agent' not in headers:
      arg_user_agent = '--user-agent=' + headers['User-Agent']
    else:
      arg_user_agent = None
    if '--disk-cache-dir' in headers:
      if os.path.isdir(headers['--disk-cache-dir']):
        arg_disk_cache_dir = '--disk-cache-dir=' + headers['--disk-cache-dir']
      else:
        if self.logger:
          self.logger.debug("Header `--disk-cache-dir` directory: `%s` not found!", headers['--disk-cache-dir'])
    else:
      arg_disk_cache_dir = None
    if '--profile-directory' in headers:
      if os.path.isdir(headers['--profile-directory']):
        arg_profile_directory = '--profile-directory=' + headers['--profile-directory']
      else:
        if self.logger:
          self.logger.debug("Header `--profile-directory` directory: `%s` not found!", headers['--profile-directory'])
    else:
      arg_profile_directory = None
    if '--user-data-dir' in headers:
      if os.path.isdir(headers['--user-data-dir']):
        arg_user_data_dir = '--user-data-dir=' + headers['--user-data-dir']
        self.userDataDirUsed = headers['--user-data-dir']
      else:
        if self.logger:
          self.logger.debug("Header `--user-data-dir` directory: `%s` not found!", headers['--user-data-dir'])
    else:
      if self.tmpDir != '':
        arg_user_data_dir = '--user-data-dir=' + self.tmpDir
        self.userDataDirUsed = self.tmpDir
      else:
        arg_user_data_dir = None
        if self.logger:
          self.logger.error("Empty tmp dir configured!")

    if self.userDataDirUsed != '' and not os.path.isdir(self.userDataDirUsed):
      if self.logger:
        self.logger.debug("Profile archive user data dir `%s` not found, trying to create...",
                          str(self.userDataDirUsed))
      try:
        os.makedirs(self.userDataDirUsed)
      except Exception, err:
        if self.logger:
          self.logger.debug("Profile archive user data dir creation error: %s", str(err))
      if os.path.isdir(self.userDataDirUsed):
        if self.logger:
          self.logger.debug("Profile archive user data dir `%s` created", str(self.userDataDirUsed))

    if '--user-data-dir-zip' in headers and self.userDataDirUsed != '' and os.path.isdir(self.userDataDirUsed):
      try:
        profiles = [p.strip() for p in headers['--user-data-dir-zip'].split(',') if p.strip() != '']
        if '--user-data-dir-zip-rotation' in headers and headers['--user-data-dir-zip-rotation'] is not None and\
          headers['--user-data-dir-zip-rotation'] != '':
          rotationType = int(headers['--user-data-dir-zip'])
        else:
          rotationType = 0
        profileIndex = 0
        if len(profiles) > 1:
          if rotationType == 0:
            r = [randint(0, len(profiles) - 1) for p in range(0, len(profiles) - 1)]
            profileIndex = r[0]
          elif rotationType == 1:
            pass
          elif rotationType == 2:
            pass
#         os.system('unzip -qq ' + profiles[profileIndex] + ' -d ' + self.userDataDirUsed)
#         os.system('mv ' + self.userDataDirUsed + '/' + \
#                   os.path.splitext(os.path.basename(profiles[profileIndex]))[0] + \
#                   '/* ' + self.userDataDirUsed)

        res = Utils.executeCommand('unzip -qq ' + profiles[profileIndex] + ' -d ' + self.userDataDirUsed)
        if res.exitCode != APP_CONSTS.EXIT_SUCCESS:
          raise Exception(str(res.stderr))

        res = Utils.executeCommand('mv ' + self.userDataDirUsed + '/' + \
                  os.path.splitext(os.path.basename(profiles[profileIndex]))[0] + \
                  '/* ' + self.userDataDirUsed)
        if res.exitCode != APP_CONSTS.EXIT_SUCCESS:
          raise Exception(str(res.stderr))

        if self.logger:
          self.logger.debug("Profile archive `%s` extracted to `%s` directory, rotation: %s",
                            profiles[profileIndex], self.userDataDirUsed, str(rotationType))
      except Exception, err:
#         if self.logger:
#           self.logger.error("Profile archive extraction error: %s", str(err))
        raise Exception("Profile archive extraction error: %s" % str(err))

    else:
      d = {'--user-data-dir-zip in headers':str('--user-data-dir-zip' in headers),
           'self.userDataDirUsed':self.userDataDirUsed,
           'os.path.isdir(self.userDataDirUsed)':str(os.path.isdir(self.userDataDirUsed))}
      if self.logger:
        self.logger.debug("Profile archive not used, condition data:\n%s", str(d))

    arg_dns_prefetch_disable = '--dns-prefetch-disable'
    # --disk-cache-size=1
    # --media-cache-size=1
    # --safe-plugins

    if headers is not None and '--disable-setuid-sandbox' in headers:
      chrome_option.add_argument('--disable-setuid-sandbox')

    if headers is not None and '--no-sandbox' in headers:
      chrome_option.add_argument('--no-sandbox')

    if headers is not None and '--incognito' in headers:
      chrome_option.add_argument('--incognito')

    # chrome_option.add_argument('--enable-logging')
    # chrome_option.add_argument('--v=1')
    # chrome_option.add_argument('--log-level=0')

    if arg_user_agent is not None:
      chrome_option.add_argument(arg_user_agent)
    if use_mobile_user_agent is not None:
      chrome_option.add_argument(use_mobile_user_agent)
    if disable_web_security is not None:
      chrome_option.add_argument(disable_web_security)
    if allow_running_insecure_content is not None:
      chrome_option.add_argument(allow_running_insecure_content)
    if allow_file_access_from_files is not None:
      chrome_option.add_argument(allow_file_access_from_files)
    if arg_proxy_bypass_list is not None:
      chrome_option.add_argument(arg_proxy_bypass_list)
    chrome_option.add_argument(arg_disable_http_cache)
    chrome_option.add_argument(arg_clear_data_reduction_proxy_data_savings)
    chrome_option.add_argument(arg_host_resolver_retry_attempts)
    chrome_option.add_argument(arg_start_maximized)
    if arg_disk_cache_dir is not None and arg_disk_cache_dir != '':
      chrome_option.add_argument(arg_disk_cache_dir)
    if arg_profile_directory is not None and arg_profile_directory != '':
      chrome_option.add_argument(arg_profile_directory)
    if arg_user_data_dir is not None and arg_user_data_dir != '':
      chrome_option.add_argument(arg_user_data_dir)
    chrome_option.add_argument(arg_dns_prefetch_disable)
    # chrome_option.add_argument(arg_incognito)

    if self.sessionId != '':
      chrome_option.add_argument(self.sessionId)

    # Proxy options
    if proxies is not None:
      proxy_type, proxy_host, proxy_port, proxy_user, proxy_passwd = proxies
      if self.logger:
        self.logger.debug("Proxy used from argument tuple: %s", str(proxies))
      if proxy_user:
        proxies = proxy_type + "://%s:%s@%s:%s" % (proxy_user, proxy_passwd, proxy_host, proxy_port)
      else:
        proxies = proxy_type + "://%s:%s" % (proxy_host, proxy_port)
      chrome_option.add_argument("--proxy-server=" + proxies)
    else:
      if '--proxy-http' in headers and headers['--proxy-http'] is not None and headers['--proxy-http'] != '':
        if '--proxy-http-domains' in headers and headers['--proxy-http-domains'] is not None and\
          headers['--proxy-http-domains'] != '':
          dn = self.getDomainNameFromURL(url)
          domain = bool(dn in headers['--proxy-http-domains'].split(','))
          if self.logger and domain is False:
            self.logger.debug("Proxy not used because domain `%s` not listed in `--proxy-http-domains`", str(dn))
        else:
          domain = True
        if domain:
          p = headers['--proxy-http'].replace('%3A', ':')
          if self.logger:
            self.logger.debug("Proxy used from header: %s", str(p))
          chrome_option.add_argument("--proxy-server=" + p)

    return chrome_option

Here is the call graph for this function:

Here is the caller graph for this function:

getProcessDirs()

def dc_crawler.Fetcher.SeleniumFetcher.getProcessDirs	(		self,
			process,
			dirsTemplate
	)

Definition at line 1050 of file Fetcher.py.

  def getProcessDirs(self, process, dirsTemplate):
    ret = []

    for f in process.open_files():
      # if self.logger:
      #  self.logger.debug("Path candidate: %s", str(f.path))
      fp = f.path.split('/')
      fpr = ''
      templateFound = False
      for item in fp:
        fpr += '/' + item
        if dirsTemplate is not None and dirsTemplate != '' and dirsTemplate in item:
          templateFound = True
          break
      if templateFound or dirsTemplate is None or dirsTemplate == '':
        ret.append(fpr)

    return ret

initializeTmpDirs()

def dc_crawler.Fetcher.SeleniumFetcher.initializeTmpDirs	(		self,
			headers
	)

Definition at line 1289 of file Fetcher.py.

  def initializeTmpDirs(self, headers):
    ret = True

    if self.tmpDir != '':
      try:
        if headers is not None and 'tmp-dir' in headers:
          self.tmpDir = headers['tmp-dir']
        if self.tmpDirRemoveBeforeCreate:
          self.removeTmpDirs()
        if not os.path.isdir(self.tmpDir):
          if logger is not None:
            self.logger.debug("Create temporary directory: %s", str(self.tmpDir))
          os.makedirs(self.tmpDir)
      except Exception, err:
        if self.logger is not None:
          ret = False
          if logger is not None:
            self.logger.debug("Error temporary directories initialization: %s", str(err))

      if os.path.isdir(self.tmpDir):
        ret = True

    return ret

Here is the call graph for this function:

Here is the caller graph for this function:

killProcess()

def dc_crawler.Fetcher.SeleniumFetcher.killProcess	(		self,
			pid,
			dirsTemplate = CHROME_DIRS_TEMPLATE,
			dirDeleteBeforeTimeout = DELAY_TERMINATE_AND_QUIT
	)

Definition at line 1002 of file Fetcher.py.

  def killProcess(self, pid, dirsTemplate=CHROME_DIRS_TEMPLATE, dirDeleteBeforeTimeout=DELAY_TERMINATE_AND_QUIT):
    del dirsTemplate, dirDeleteBeforeTimeout
    try:
      if self.logger:
        self.logger.debug("Try to Kill process pid: %s", str(pid))
        process = psutil.Process(pid)
        for proc in process.children(recursive=True):
          if self.logger:
            self.logger.debug("Killing child process pid: %s", str(proc.pid))
          try:
            # dirs = self.getProcessDirs(proc, dirsTemplate)
            proc.kill()
            # if self.logger:
            #  self.logger.debug("Dirs to remove: %s", str(dirs))
            # for d in dirs:
            #  time.sleep(dirDeleteBeforeTimeout)
              # self.removeTmpDirs(d)
            # os.kill(pid, signal.SIGKILL)
          except Exception, err:
            if self.logger:
              self.logger.debug("Child process pid: %s kill error: ", str(pid), str(err))
        if self.logger:
          self.logger.debug("Killing main process pid: %s", str(process.pid))
        # dirs = self.getProcessDirs(process, dirsTemplate)
        process.kill()
        # if self.logger:
        #  self.logger.debug("Dirs to remove: %s", str(dirs))
        # for d in dirs:
        #  time.sleep(dirDeleteBeforeTimeout)
          # self.removeTmpDirs(d)
        # os.kill(pid, signal.SIGKILL)
    except Exception, err:
      if self.logger:
        self.logger.debug("Process pid: %s kill error: %s", str(pid), str(err))

Here is the caller graph for this function:

open()

def dc_crawler.Fetcher.SeleniumFetcher.open	(		self,
			url,
			method = 'get',
			headers = None,
			timeout = DEFAUIL_TIMEOUT,
			allow_redirects = True,
			proxies = None,
			auth = None,
			data = None,
			log = None,
			allowed_content_types = None,
			max_resource_size = None,
			max_redirects = 1,
			filters = None,
			executable_path = None,
			depth = None,
			macro = None
	)

Definition at line 527 of file Fetcher.py.

           macro=None):

    if log is not None:
      self.logger = log

    if self.logger is not None:
      self.logger.debug("Dynamic fetcher call:\nurl:" + str(url) + \
                        "\nmethod:" + str(method) + "\nheaders:" + str(headers) + "\ntimeout:" + str(timeout) + \
                        "\nallow_redirects:" + str(allow_redirects) + "\nproxies:" + str(proxies) + "\nauth:" + \
                        str(auth) + "\ndata:" + str(data) + "\nlogger:" + str(self.logger) + \
                        "\nallowed_content_types:" + str(allowed_content_types) + "\nmax_resource_size:" + \
                        str(max_resource_size) + "\nmax_redirects:" + str(max_redirects) + "\nexecutable_path:" + \
                        str(executable_path) + "\ncur_dir:" + str(os.getcwd()) + "\nmacro:" + str(macro))

    t1 = 0
    if isinstance(timeout, tuple):
      t = int(timeout[0])
      if isinstance(timeout[0], float):
        t1 = int(str(timeout[0]).strip()[str(timeout[0]).strip().find('.') + 1:])
    else:
      t = int(timeout)
      if isinstance(timeout, float):
        t1 = int(str(timeout).strip()[str(timeout).strip().find('.') + 1:])
    if self.logger is not None:
      self.logger.debug("Execution timeout: %s, damping timeout: %s", str(t), str(t1))
    if t1 >= t:
      msg = "Execution timeout: %s less or equal than damping timeout: %s, aborted" % (str(t), str(t1))
      if self.logger is not None:
        self.logger.error(msg)
      raise SeleniumFetcherException(msg)

    if self.tmpDirType == self.TMP_DIR_TYPE_OPEN:
      if not self.initializeTmpDirs(headers):
        msg = 'Temporary directory type OPEN `%s` initialization error!' % self.tmpDir
        if self.logger is not None:
          self.logger.error(msg)
        raise SeleniumFetcherException(msg)
      else:
        if self.logger is not None:
          self.logger.debug('Temporary directory type OPEN `%s` initialized', self.tmpDir)

    from app.Utils import executeWithTimeout
    try:
      ret = executeWithTimeout(func=self.openT, args=(url, headers, t1, proxies, executable_path, macro,),
                               timeout=t, log=self.logger)
      if ret is None:
        if self.logger is not None:
          msg = 'Execution timeout: ' + str(t) + ' reached!'
          self.logger.error(msg)
        raise SeleniumFetcherException(msg, APP_CONSTS.ERROR_FETCH_TIMEOUT)
    except SeleniumFetcherException, err:
      if self.logger is not None:
        self.logger.error("Error SeleniumFetcherException: %s", str(err))
      self.cleanup(1, headers)
      raise err
    except Exception, err:
      if self.logger is not None:
        msg = 'Execution with timeout error:' + str(err)
        self.logger.error(msg)
      self.cleanup(1, headers)
      raise SeleniumFetcherException(msg)
    finally:
      self.cleanup(0, headers)

    if self.logger is not None:
      self.logger.debug("Dynamic fetcher call finished normally.")

    return ret

Here is the call graph for this function:

Here is the caller graph for this function:

openT()

def dc_crawler.Fetcher.SeleniumFetcher.openT	(		self,
			url,
			headers,
			timeout,
			proxies,
			executable_path,
			macro
	)

Definition at line 607 of file Fetcher.py.

  def openT(self, url, headers, timeout, proxies, executable_path, macro):
    startTime = time.time()
    inlineMacro = ''

    try:
      # Prepare inline macro
      if self.inlineURLMacroDelimiter in url:
        t = url.split(self.inlineURLMacroDelimiter)
        url = t[0]
        inlineMacro = t[1]
      # Dependent import
      try:
        from selenium import webdriver
        import selenium.webdriver.support.ui  # pylint: disable=W0611
      except Exception, err:
        msg = 'Selenium module import error: ' + str(err)
        if self.logger is not None:
          self.logger.error(msg)
        raise SeleniumFetcherException(msg)

      if self.logger is not None:
        # One way
        from selenium.webdriver.remote.remote_connection import LOGGER as seleniumLogger
        seleniumLogger.setLevel(self.logger.getEffectiveLevel())
        # Second way
        selenium_logger = logging.getLogger('selenium.webdriver.remote.remote_connection')
        # Only display possible problems
        # selenium_logger.setLevel(logging.WARNING)
        selenium_logger.setLevel(self.logger.getEffectiveLevel())

      # Initialize defaults
      exec_path = "./"
      driver_name = "chromedriver"
      error_msg = ""
      error_code = 0
      error_code_macro = 0
      page_source_macro = None
      content_type_macro = None
      result_type_macro = self.MACRO_RESULT_TYPE_DEFAULT
      fatalErrors = [self.ERROR_FATAL, self.ERROR_GENERAL, self.ERROR_NAME_NOT_RESOLVED, self.ERROR_TOO_MANY_REDIRECTS,
                     self.ERROR_PROXY_CONNECTION_FAILED, self.ERROR_CONNECTION_TIMED_OUT, self.ERROR_CONFLICT,
                     self.ERROR_TUNNEL_CONNECTION_FAILED, self.ERROR_EMPTY_RESPONSE, self.ERROR_SERVICE_UNAVAILABLE]

      # Check environment
      # TODO: add dependecy argument pass, now reduced and hardcoded
      checkEnv = True
      if checkEnv:
        # envVars = {"DISPLAY": "", "LC_ALL":"en_US.UTF-8", "LANG":"en_US.UTF-8", "LANGUAGE":"en_US.UTF-8"}
        envVars = {"DISPLAY": "", "LANG":"en_US.UTF-8"}
        for varName in envVars:
          v = os.getenv(varName, "")
          if varName == "DISPLAY":
            if v == "":
              raise SeleniumFetcherException("Environment variable 'DISPLAY' is not set!")
          else:
            if v != envVars[varName]:
              raise SeleniumFetcherException("Environment variable '" + varName + "' value expected:'" + \
                                             envVars[varName] + "', got from os: '" + v + "'; all env: " + \
                                             str(os.environ))

      # Create driver instance
      try:
        # get chrome options
        chrome_option = self.getOptions(webdriver, headers, proxies, url)

        # The platform-dependent path to the driver executable
        if executable_path is None:
          path = exec_path + driver_name + str(ctypes.sizeof(ctypes.c_voidp) * 8)
        else:
          path = executable_path
        if self.logger is not None:
          self.logger.debug("Chrome driver executable path: %s, options: %s", str(path), str(chrome_option.arguments))
        from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
        # enable browser logging
        d = DesiredCapabilities.CHROME
        d['loggingPrefs'] = {'browser':'ALL'}
        # Get driver
        self.driver = webdriver.Chrome(executable_path=path, chrome_options=chrome_option, desired_capabilities=d)
        self.driverPid = self.driver.service.process.pid
        if self.logger:
          self.logger.debug("Driver pid: " + str(self.driverPid))
      except Exception, err:
        error_msg = 'Driver initialization error: ' + str(err)
        error_code = self.ERROR_FATAL
      except:  # pylint: disable=W0702
        error_msg = 'General driver initialization!'
        error_code = self.ERROR_GENERAL

      if error_code > 0:
        if self.logger is not None:
          self.logger.error('Fatal error: ' + error_msg)
        raise SeleniumFetcherException(error_msg)

      # Make request
      try:
        # driver.set_page_load_timeout(timeout * 10)
        # driver.set_script_timeout(timeout * 10)
        # driver.implicitly_wait(timeout * 10)

        if self.logger is not None:
          self.logger.debug("Chrome driver get url: `%s`", str(url))
        self.driver.get(url)
        # Get logs
        log_types = self.driver.log_types
        if 'browser' in log_types:
          log_list = self.driver.get_log('browser')
          if self.logger is not None:
            self.logger.debug("Driver logs: " + str(log_list))
          for item_dict in log_list:
            if self.logger is not None:
              self.logger.debug("Driver message: `%s`", str(item_dict["message"]))
            if "message" in item_dict and ((url + ' ') in item_dict["message"] or (url + '/ ') in item_dict["message"]):
              error_msg += item_dict["message"] + " | "
        else:
          if self.logger is not None:
            self.logger.error("No driver logs!")
        if error_msg != "":
          entrances = [
              (r"(.*)net::ERR_NAME_NOT_RESOLVED(.*)", self.ERROR_NAME_NOT_RESOLVED),
              (r"(.*)net::ERR_TOO_MANY_REDIRECTS(.*)", self.ERROR_TOO_MANY_REDIRECTS),
              (r"(.*)ERR_PROXY_CONNECTION_FAILED(.*)", self.ERROR_PROXY_CONNECTION_FAILED),
              (r"(.*)net::ERR_CONNECTION_TIMED_OUT(.*)", self.ERROR_CONNECTION_TIMED_OUT),
              (r"(.*)net::ERR_TUNNEL_CONNECTION_FAILED(.*)", self.ERROR_TUNNEL_CONNECTION_FAILED),
              (r"(.*)net::ERR_CONNECTION_RESET(.*)", self.ERROR_TUNNEL_CONNECTION_FAILED),
              (r"(.*)net::ERR_INVALID_URL(.*)", self.ERROR_TUNNEL_CONNECTION_FAILED),
              (r"(.*)net::ERR_EMPTY_RESPONSE(.*)", self.ERROR_EMPTY_RESPONSE),
              (r"(.*)" + self.LOG_MESSAGE_RENDERRER_TIMEOUT + r"(.*)", self.ERROR_CONNECTION_TIMED_OUT),
              (r"(.*)" + self.LOG_MESSAGE_SERVER_RESPONSE_503 + r"(.*)", self.ERROR_SERVICE_UNAVAILABLE),
              (r"(.*)" + self.LOG_MESSAGE_SERVER_RESPONSE_409 + r"(.*)", self.ERROR_CONFLICT),
              (r"(.*)403 \(Forbidden\)(.*)", 403),
              (r"(.*)404 \(Not Found\)(.*)", 404),
              (r"(.*)500 \(Internal Server Error\)(.*)", 500),
              (r"(.*)net::(.*)", 520)]
          for item in entrances:
            regex = re.compile(item[0])
            r = regex.search(error_msg)
            if r:
              error_code = item[1]
              if self.logger is not None:
                self.logger.debug("Page error: " + error_msg)
              break
        if error_code not in fatalErrors and inlineMacro != '':
          if self.logger is not None:
            self.logger.debug("Execute inline macro: %s", str(inlineMacro))
          macroResults, errorCode, errorMsg = self.execMacroSimple([inlineMacro])
        if error_code not in fatalErrors and macro is not None:
          if self.logger is not None:
            self.logger.debug("Execute macro: %s", str(macro))
          if isinstance(macro, list):
            macroResults, errorCode, errorMsg = self.execMacroSimple(macro)
          else:
            macroResults, errorCode, errorMsg, content_type_macro, result_type_macro = self.execMacroExtended(macro)
          if errorCode > 0:
            error_code_macro |= APP_CONSTS.ERROR_MACRO
            error_msg = errorMsg
          if len(macroResults) > 0:
            if result_type_macro == self.MACRO_RESULT_TYPE_CONTENT:
              page_source_macro = macroResults
            else:
              page_source_macro = json.dumps(macroResults, ensure_ascii=False)  # pylint: disable=R0204
      except Exception, err:
        error_msg = 'Driver error: ' + str(err) + '; logs: ' + self.getAllLogsAsString()
        error_code = self.ERROR_FATAL
      except:  # pylint: disable=W0702
        error_msg = "General driver usage error!"
        error_code = self.ERROR_GENERAL

      if error_code == 0:
        if timeout > 0:
          if self.logger is not None:
            self.logger.debug("Wait on damping timeout to load all dynamic parts of the page: %s sec", str(timeout))
          # Wait fixed time to load all dynamic parts of the page
          time.sleep(timeout)
      elif error_code in fatalErrors:
        if self.logger is not None:
          self.logger.debug("Fatal error, code: %s, msg: %s", str(error_code), error_msg)
        if error_code == self.ERROR_NAME_NOT_RESOLVED:
          code = APP_CONSTS.ERROR_FETCH_INVALID_URL
        elif error_code == self.ERROR_TOO_MANY_REDIRECTS:
          code = APP_CONSTS.ERROR_FETCH_TOO_MANY_REDIRECTS
        elif error_code == self.ERROR_PROXY_CONNECTION_FAILED:
          code = APP_CONSTS.ERROR_FETCH_CONNECTION_ERROR
        elif error_code == self.ERROR_CONNECTION_TIMED_OUT:
          code = APP_CONSTS.ERROR_FETCH_CONNECTION_TIMEOUT
        elif error_code == self.ERROR_TUNNEL_CONNECTION_FAILED:
          code = APP_CONSTS.ERROR_FETCH_FORBIDDEN
        elif error_code == self.ERROR_EMPTY_RESPONSE:
          code = APP_CONSTS.ERROR_EMPTY_RESPONSE
        elif error_code == self.ERROR_SERVICE_UNAVAILABLE:
          code = APP_CONSTS.ERROR_FETCH_FORBIDDEN
        elif error_code == self.ERROR_CONFLICT:
          code = APP_CONSTS.ERROR_FETCH_HTTP_ERROR
        else:
          code = APP_CONSTS.ERROR_FETCHER_INTERNAL
        # self.cleanup(driver)
        raise SeleniumFetcherException(error_msg, code)

      page_source = ""
      cookies = {}
      try:
        page_source = self.driver.page_source
        cookies = self.driver.get_cookies()
      except Exception, err:
        error_msg = str(err)
        error_code = self.ERROR_CONTENT_OR_COOKIE
      except:  # pylint: disable=W0702
        error_msg = "Content and cookies get error!"
        error_code = self.ERROR_CONTENT_OR_COOKIE

      content_type = None
      charset = None
      try:
        attr = self.driver.find_element_by_xpath(".//meta[translate(@http-equiv,'ABCDEFGHIJKLMNOPQRSTUVWXYZ','abcdefghijklmnopqrstuvwxyz')='content-type']").get_attribute("content")  # pylint: disable=C0301
        regex = re.compile(r"(.*); charset=(.*)", re.IGNORECASE)
        items = regex.search(attr)
        if items is not None:
          items = items.groups()
          if len(items) > 1:
            content_type = items[0]
            charset = items[1]
      except Exception, err:
        pass
      if content_type is None:
        try:
          attr = self.driver.find_element_by_xpath('//html')
          content_type = self.CONTENT_TYPE_HTML
        except Exception, err:
          pass
      if content_type is not None and charset is None:
        try:
          charset = self.driver.find_element_by_xpath('//meta[@charset]').get_attribute("charset")
        except Exception, err:
          pass
      if charset is None:
        try:
          charset = self.driver.execute_script("return document.characterSet;")
        except Exception, err:
          if self.logger is not None:
            self.logger.debug("Charset detection error: %s", str(err))

      try:
        current_url = self.driver.current_url
      except Exception, err:
        current_url = url
        if self.logger is not None:
          self.logger.debug("Get 'current_url' error: %s, input url assumed: %s", str(err), str(url))

      # if self.LOG_MESSAGE_RENDERRER_TIMEOUT in error_msg:
      #  self.cleanup(driver)
      #  raise SeleniumFetcherException(error_msg, APP_CONSTS.ERROR_FETCH_CONNECTION_TIMEOUT)

      try:
        res = Response()
        res.url = current_url
        if error_code > 100 or error_code == self.ERROR_FATAL:
          res.status_code = error_code
        else:
          res.status_code = 200
        res.redirects = []
        if page_source_macro is None:
          res.unicode_content = page_source
        else:
          res.unicode_content = page_source_macro
        res.str_content = res.unicode_content
        res.rendered_unicode_content = res.unicode_content
        res.content_size = len(res.unicode_content)
        res.encoding = charset
        res.headers = {'content-length': res.content_size}
        if page_source_macro is not None:
          if content_type_macro is not None:
            content_type = content_type_macro
          else:
            content_type = self.CONTENT_TYPE_JSON
        if content_type is not None:
          res.headers['content-type'] = content_type
        if current_url != url:
          res.headers['location'] = current_url
        res.meta_res = res.unicode_content
        res.cookies = cookies
        res.dynamic_fetcher_type = driver_name
        res.dynamic_fetcher_result_type = result_type_macro
        if error_code_macro != APP_CONSTS.ERROR_OK:
          res.error_mask |= error_code_macro
        res.time = time.time() - startTime
        res.request = {'headers':headers}  # # alexv
        res.error_msg = error_msg
      except Exception, err:
        msg = 'Response fill error: ' + str(err)
        if self.logger is not None:
          self.logger.error(msg)
        raise SeleniumFetcherException(msg)

      if self.logger is not None and error_msg != "":
        self.logger.debug("Dynamic fetcher none fatal error: " + error_msg)

      return res

    except Exception, err:
      msg = 'Unrecognized dynamic fetcher error: ' + str(err)
      if self.logger is not None:
        self.logger.error(msg)
      raise SeleniumFetcherException(msg)

Here is the call graph for this function:

Here is the caller graph for this function:

removeTmpDirs()

def dc_crawler.Fetcher.SeleniumFetcher.removeTmpDirs	(		self,
			delay = DELAY_TERMINATE_AND_QUIT,
			tries = 3
	)

Definition at line 1316 of file Fetcher.py.

  def removeTmpDirs(self, delay=DELAY_TERMINATE_AND_QUIT, tries=3):
    if self.tmpDir != '':
      for i in xrange(1, tries):
        try:
          time.sleep(delay)
          if os.path.isdir(self.tmpDir):
            if self.logger is not None:
              self.logger.debug("Removing tmp dir: %s, try: %s", self.tmpDir, str(i))
            shutil.rmtree(self.tmpDir)
          else:
            break
        except Exception, err:
          if self.logger is not None:
            self.logger.debug("Remove tmp dir: %s, try: %s, error: %s", self.tmpDir, str(i), str(err))

Here is the caller graph for this function:

should_have_meta_res()

def dc_crawler.Fetcher.SeleniumFetcher.should_have_meta_res

(

self

)

Definition at line 1041 of file Fetcher.py.

  def should_have_meta_res(self):
    return True

Member Data Documentation

CHROME_DEBUG_LOG_NAME

string dc_crawler.Fetcher.SeleniumFetcher.CHROME_DEBUG_LOG_NAME = 'chrome_debug.log'

static

Definition at line 423 of file Fetcher.py.

CHROME_DIRS_TEMPLATE

string dc_crawler.Fetcher.SeleniumFetcher.CHROME_DIRS_TEMPLATE = '.google.Chrome.'

static

Definition at line 422 of file Fetcher.py.

CHROME_PROCESS_NAMES

list dc_crawler.Fetcher.SeleniumFetcher.CHROME_PROCESS_NAMES = ['chrome', 'BrowserBlocking']

static

Definition at line 421 of file Fetcher.py.

CONTENT_TYPE_HTML

string dc_crawler.Fetcher.SeleniumFetcher.CONTENT_TYPE_HTML = 'text/html'

static

Definition at line 401 of file Fetcher.py.

CONTENT_TYPE_JSON

string dc_crawler.Fetcher.SeleniumFetcher.CONTENT_TYPE_JSON = 'text/json'

static

Definition at line 400 of file Fetcher.py.

DEFAUIL_TIMEOUT

int dc_crawler.Fetcher.SeleniumFetcher.DEFAUIL_TIMEOUT = 5

static

Definition at line 399 of file Fetcher.py.

DELAY_TERMINATE_AND_QUIT

float dc_crawler.Fetcher.SeleniumFetcher.DELAY_TERMINATE_AND_QUIT = 0.5

static

Definition at line 402 of file Fetcher.py.

driver

dc_crawler.Fetcher.SeleniumFetcher.driver

Definition at line 478 of file Fetcher.py.

driverPid

dc_crawler.Fetcher.SeleniumFetcher.driverPid

Definition at line 479 of file Fetcher.py.

ERROR_CONFLICT

int dc_crawler.Fetcher.SeleniumFetcher.ERROR_CONFLICT = 409

static

Definition at line 414 of file Fetcher.py.

ERROR_CONNECTION_TIMED_OUT

int dc_crawler.Fetcher.SeleniumFetcher.ERROR_CONNECTION_TIMED_OUT = 505

static

Definition at line 411 of file Fetcher.py.

ERROR_CONTENT_OR_COOKIE

int dc_crawler.Fetcher.SeleniumFetcher.ERROR_CONTENT_OR_COOKIE = 3

static

Definition at line 406 of file Fetcher.py.

ERROR_EMPTY_RESPONSE

int dc_crawler.Fetcher.SeleniumFetcher.ERROR_EMPTY_RESPONSE = 13

static

Definition at line 415 of file Fetcher.py.

ERROR_FATAL

int dc_crawler.Fetcher.SeleniumFetcher.ERROR_FATAL = 1

static

Definition at line 404 of file Fetcher.py.

ERROR_GENERAL

int dc_crawler.Fetcher.SeleniumFetcher.ERROR_GENERAL = 2

static

Definition at line 405 of file Fetcher.py.

ERROR_MACRO_RETURN_VALUE

int dc_crawler.Fetcher.SeleniumFetcher.ERROR_MACRO_RETURN_VALUE = 12

static

Definition at line 409 of file Fetcher.py.

ERROR_NAME_NOT_RESOLVED

int dc_crawler.Fetcher.SeleniumFetcher.ERROR_NAME_NOT_RESOLVED = 400

static

Definition at line 407 of file Fetcher.py.

ERROR_PROXY_CONNECTION_FAILED

int dc_crawler.Fetcher.SeleniumFetcher.ERROR_PROXY_CONNECTION_FAILED = 504

static

Definition at line 410 of file Fetcher.py.

ERROR_SERVICE_UNAVAILABLE

int dc_crawler.Fetcher.SeleniumFetcher.ERROR_SERVICE_UNAVAILABLE = 503

static

Definition at line 413 of file Fetcher.py.

ERROR_TOO_MANY_REDIRECTS

int dc_crawler.Fetcher.SeleniumFetcher.ERROR_TOO_MANY_REDIRECTS = 11

static

Definition at line 408 of file Fetcher.py.

ERROR_TUNNEL_CONNECTION_FAILED

int dc_crawler.Fetcher.SeleniumFetcher.ERROR_TUNNEL_CONNECTION_FAILED = 403

static

Definition at line 412 of file Fetcher.py.

inlineURLMacroDelimiter

dc_crawler.Fetcher.SeleniumFetcher.inlineURLMacroDelimiter

Definition at line 480 of file Fetcher.py.

LOG_MESSAGE_RENDERRER_TIMEOUT

string dc_crawler.Fetcher.SeleniumFetcher.LOG_MESSAGE_RENDERRER_TIMEOUT = 'Timed out receiving message from renderer'

static

Definition at line 417 of file Fetcher.py.

LOG_MESSAGE_SERVER_RESPONSE_409

string dc_crawler.Fetcher.SeleniumFetcher.LOG_MESSAGE_SERVER_RESPONSE_409 = 'server responded with a status of 409 (Conflict)'

static

Definition at line 419 of file Fetcher.py.

LOG_MESSAGE_SERVER_RESPONSE_503

string dc_crawler.Fetcher.SeleniumFetcher.LOG_MESSAGE_SERVER_RESPONSE_503 = 'server responded with a status of 503'

static

Definition at line 418 of file Fetcher.py.

logger

dc_crawler.Fetcher.SeleniumFetcher.logger

Definition at line 441 of file Fetcher.py.

MACRO_RESULT_TYPE_AUTO

int dc_crawler.Fetcher.SeleniumFetcher.MACRO_RESULT_TYPE_AUTO = 3

static

Definition at line 428 of file Fetcher.py.

MACRO_RESULT_TYPE_CONTENT

int dc_crawler.Fetcher.SeleniumFetcher.MACRO_RESULT_TYPE_CONTENT = 2

static

Definition at line 427 of file Fetcher.py.

MACRO_RESULT_TYPE_DEFAULT

int dc_crawler.Fetcher.SeleniumFetcher.MACRO_RESULT_TYPE_DEFAULT = 0

static

Definition at line 425 of file Fetcher.py.

MACRO_RESULT_TYPE_URLS_LIST

int dc_crawler.Fetcher.SeleniumFetcher.MACRO_RESULT_TYPE_URLS_LIST = 1

static

Definition at line 426 of file Fetcher.py.

sessionId

dc_crawler.Fetcher.SeleniumFetcher.sessionId

Definition at line 481 of file Fetcher.py.

TMP_DIR_TYPE_INSTANTIATE

int dc_crawler.Fetcher.SeleniumFetcher.TMP_DIR_TYPE_INSTANTIATE = 1

static

Definition at line 431 of file Fetcher.py.

TMP_DIR_TYPE_OPEN

int dc_crawler.Fetcher.SeleniumFetcher.TMP_DIR_TYPE_OPEN = 0

static

Definition at line 430 of file Fetcher.py.

tmpDir

dc_crawler.Fetcher.SeleniumFetcher.tmpDir

Definition at line 465 of file Fetcher.py.

tmpDirPath

dc_crawler.Fetcher.SeleniumFetcher.tmpDirPath

Definition at line 446 of file Fetcher.py.

tmpDirPrefix

dc_crawler.Fetcher.SeleniumFetcher.tmpDirPrefix

Definition at line 447 of file Fetcher.py.

tmpDirRemoveBeforeCreate

dc_crawler.Fetcher.SeleniumFetcher.tmpDirRemoveBeforeCreate

Definition at line 450 of file Fetcher.py.

tmpDirSuffix

dc_crawler.Fetcher.SeleniumFetcher.tmpDirSuffix

Definition at line 448 of file Fetcher.py.

tmpDirType

dc_crawler.Fetcher.SeleniumFetcher.tmpDirType

Definition at line 449 of file Fetcher.py.

userDataDirUsed

dc_crawler.Fetcher.SeleniumFetcher.userDataDirUsed

Definition at line 482 of file Fetcher.py.

---

The documentation for this class was generated from the following file:

• sources/hce/dc_crawler/Fetcher.py