ftest_dc_fetchers_gplus.py

Go to the documentation of this file.

"""
HCE project, Python bindings, Crawler application.
Dynamic fetcher module tests of FB scrapping. Account: hcegroup2016@gmail.com/testtest2016

@package: dc
@file ftest_dc_dynamic_fetcher.py
@author bgv <bgv.hce@gmail.com>
@link: http://hierarchical-cluster-engine.com/
@copyright: Copyright &copy; 2016 IOIX Ukraine
@license: http://hierarchical-cluster-engine.com/license/
@since: 1.4.3
"""

import logging
import os
import json
import jsonpickle

from app.Utils import varDump
from dc_crawler.Fetcher import BaseFetcher


# create logger
logger = logging.getLogger('ftest_dc_dynamic_fetcher')
logger.setLevel(logging.DEBUG)
# create console handler and set level to debug
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
# create formatter
formatter = logging.Formatter('%(asctime)s - %(thread)ld - %(threadName)s - %(module)s - %(funcName)s - %(levelname)s - %(message)s')
# add formatter to ch
ch.setFormatter(formatter)
# add ch to logger
logger.addHandler(ch)


if __name__ == "__main__":
  res = None

  print "CurDir:\n", os.path.dirname(os.path.realpath(__file__))
  loadHeaders = False
  headersDict = {}
  userDataDir = None
  userCacheDir = None
  userProfileZip = None
  # userDataDir = '/tmp/custom_profile_fb'
  # userDataDir = '/tmp/custom_profile_fb_noimg'
  # userDataDir = '/tmp/custom_profile'
  # userCacheDir = '/tmp/custom_profile_cache'

  # userProfileZip = '/tmp/custom_profile_fb_noimg50.zip'
  # userProfileZip = '/tmp/custom_profile_fb_noimg50b.zip'
  # userProfileZip = '/tmp/custom_profile_fb_noimg50c.zip'
  # userProfileZip = '/tmp/custom_profile_fb_noimg50d.zip'
  # userProfileZip = '/tmp/custom_profile_fb_noimg50e.zip'
  # userProfileZip = '/tmp/custom_profile_fb_noimg50f.zip'
  # userProfileZip = '/tmp/custom_profile_fb_noimg50g.zip'
  # userProfileZip = '/tmp/custom_profile_fb_noimg50h.zip'
  # userProfileZip = '/tmp/custom_profile_fb_noimg50i.zip'
  # userProfileZip = '/tmp/custom_profile_fb_noimg50j.zip'
  # userProfileZip = '/tmp/custom_profile_fb_noimg50k.zip'
  # userProfileZip = '/tmp/custom_profile_fb_noimg50l.zip'
  # userProfileZip = '/tmp/custom_profile_fb_noimg50m.zip'
  # userProfileZip = '/tmp/custom_profile_fb_noimg50.zip,/tmp/custom_profile_fb_noimg50b.zip,/tmp/custom_profile_fb_noimg50c.zip'

  if loadHeaders:
    hdrs = None
    with open("../../ini/crawler-task_headers.txt", 'r') as f:
      hdrs = ''.join(f.readlines())
    for header in hdrs.splitlines():
      if not header:
        continue
      try:
        key, value = header[:header.index(':')].strip(), header[header.index(':') + len(':'):].strip()
      except Exception:
        print "header error:%s", header
        os.abort()
      headersDict[key] = value

  allowRedirects = 1
  proxies = None
  # proxies = (proxy_type, proxy_host, proxy_port, proxy_user, proxy_passwd)
  # proxies = ('http', 'dev.hce-project.com', '3129', None, None)
  authorization = None
  postData = None
  process_content_types = ["text/html"]
  maxResourceSize = 1024 * 1024
  maxHttpRedirects = 3
  fetchType = BaseFetcher.TYP_NORMAL
  localFilters = None
  urlObjDepth = 0
  inlineMacro = "window.IFRAME_QUERY_URL=\"https://plus.google.com/s/%25QUERY_STRING%25\";" + \
                "window.IFRAME_CSCROLL_COUNT=100;" + \
                "window.IFRAME_MAX_TIME=350;" + \
                "window.IFRAME_SFIELD=\"title\";"
  url = 'https://plus.google.com/' + '###' + inlineMacro
  httpTimeout = 120000
  tm = int(httpTimeout) / 1000.0
  if isinstance(httpTimeout, float):
    tm += float('0' + str(httpTimeout).strip()[str(httpTimeout).strip().find('.'):])
  headersDict.update({'--disable-web-security':'', '--allow-running-insecure-content':''})
  macroCode = {"name":"tests",
               "sets":[{"name":"set1", "items":['', '', '', '', ''], "repeat":1, "delay":0}],
               "result_type":2,
               "result_content_type":"text/json",
               "result_fetcher_type":1}
  macroCode['sets'][0]['items'][0] = '5'
  macroCode['sets'][0]['items'][1] = "file:///tmp/ftest_dc_fetchers_gplus_social_macro_data.js"
  macroCode['sets'][0]['items'][2] = 'file:///tmp/social.js'
  macroCode['sets'][0]['items'][3] = '!5:20:return window.IFRAME_DATA_READY;'
  macroCode['sets'][0]['items'][4] = "return window.MACRO_COLLECT;"

  if userDataDir is not None:
    headersDict['--user-data-dir'] = userDataDir
  if userCacheDir is not None:
    headersDict['--disk-cache-dir'] = userCacheDir
  if userProfileZip is not None:
    headersDict['--user-data-dir-zip'] = userProfileZip
    # Rotation type 0 - random, 1 - cyclic, 2 - next not used now
    # headersDict['--user-data-dir-zip-rotation'] = 0
  headersDict['--log-chrome-debug-log'] = ''
  # headersDict['--proxy-http'] = 'http://proxy1.dev.hce-project.com%3A3180'
  # headersDict['--proxy-http'] = 'http://proxy2.dev.hce-project.com%3A3280'
  # headersDict['--proxy-http'] = 'http://proxy3.dev.hce-project.com%3A3380'
  # headersDict['--proxy-http'] = 'http://proxy4.dev.hce-project.com%3A3480'
  # headersDict['--proxy-http-domains'] = 'www.facebook.com'

  fetchType = BaseFetcher.TYP_DYNAMIC
  # change current dir for webdriver executable run with path ./
  os.chdir("../../bin/")

  try:
    print "headersDict:\n", varDump(headersDict)
    f = BaseFetcher.get_fetcher(fetchType)
    if userDataDir is not None:
      f.tmpDir = ''
    res = f.open(url, timeout=tm, headers=headersDict, allow_redirects=allowRedirects, proxies=proxies,
                 auth=authorization, data=postData, log=logger, allowed_content_types=process_content_types,
                 max_resource_size=maxResourceSize, max_redirects=maxHttpRedirects, filters=localFilters,
                 depth=urlObjDepth, macro=macroCode)

  except Exception, err:
    # logger.debug("Exception:\n%s", varDump(err))
    print "Exception:\n", varDump(err)

  # rd = varDump(res)
  rd = json.dumps(json.loads(jsonpickle.encode(res)), indent=2)
  # logger.debug("Result:\n%s", varDump(res))
  print "Result:\n", rd