prepairer_single_urls Namespace Reference

Functions
def	readTemplatesFromMySQL ()
def	cutURL (url)
def	generateTemplates ()
def	executeQuery (db_connector, query)
def	loadDBBackend ()
def	createSiteObj (input_url)
def	createURLObj (site, input_url)
def	addSite (site)
def	addURL (url, site)
def	updateURL (input_url, site)
def	updateSite (site)
def	processBatch ()

Variables
	filename
	filemode
	logger = logging.getLogger("Prepairer")
	db_connector = None
	dc_sites_db_connect = None
	dc_urls_db_connect = None
dictionary	site_templates_dic = {}
dictionary	templates_dic = {}

Detailed Description

HCE project, Python bindings, Distributed Tasks Manager application.
Event objects definitions.

@package: dc
@file prepairer.py
@author Oleksii <developers.hce@gmail.com>
@link: http://hierarchical-cluster-engine.com/
@copyright: Copyright &copy; 2013-2014 IOIX Ukraine
@license: http://hierarchical-cluster-engine.com/license/
@since: 0.1

Function Documentation

addSite()

def prepairer_single_urls.addSite

(

site

)

Definition at line 159 of file prepairer_single_urls.py.

def addSite(site):
  file_name = "site_" + str(site.id) + ".json"
  open(file_name, "w").write(site.toJSON())
  cmd = "./dc-client.py --config=../ini/dc-client.ini --command=SITE_NEW --file=./%s" % file_name
  process = Popen(cmd, stdout=PIPE, stdin=PIPE, shell=True, close_fds=True)
  (output, err) = process.communicate()
  exit_code = process.wait()
  open("dc-client_new_site_output.txt", "w").write(output)
  return exit_code

Here is the caller graph for this function:

addURL()

def prepairer_single_urls.addURL	(		url,
			site
	)

Definition at line 170 of file prepairer_single_urls.py.

def addURL(url, site):
  file_name = "url_" + str(site.id) + ".json"
  open(file_name, "w").write("[" + url.toJSON() + "]")
  cmd = "./dc-client.py --config=../ini/dc-client.ini --command=URL_NEW --file=./%s" % file_name
  process = Popen(cmd, stdout=PIPE, stdin=PIPE, shell=True, close_fds=True)
  (output, err) = process.communicate()
  exit_code = process.wait()
  open("dc-client_new_url_output.txt", "w").write(output)
  return exit_code

Here is the caller graph for this function:

createSiteObj()

def prepairer_single_urls.createSiteObj

(

input_url

)

Definition at line 118 of file prepairer_single_urls.py.

def createSiteObj(input_url):

  # strip input url
  input_url = input_url.strip()

  # root url
  root_url = input_url

  # get url for md5
  norm_url = cutURL(input_url)

  # create site
  site = Site(norm_url)

  # create site filters
  site_filter_pattern = ".*" + norm_url + ".*"
  site_filters = SiteFilter(site.id, site_filter_pattern)


  # create site properties templates
  if site.id in templates_dic:
    site.properties["template"] = templates_dic[site.id]

  # fill site
  # site.urls = [root_url]
  site.urls = []
  site.filters = [site_filters]
  # site.maxResources = 5
  # site.maxURLs = 5
  # site.state = Site.STATE_SUSPENDED
  # site.filters = [SiteFilter(site.id, "(.*)")]
  return site

Here is the call graph for this function:

Here is the caller graph for this function:

createURLObj()

def prepairer_single_urls.createURLObj	(		site,
			input_url
	)

Definition at line 152 of file prepairer_single_urls.py.

def createURLObj(site, input_url):
  url = URL(site.id, input_url)
  url.status = URL.STATUS_SELECTED_CRAWLING
  url.type = URL.TYPE_SINGLE
  return url

Here is the caller graph for this function:

cutURL()

def prepairer_single_urls.cutURL

(

url

)

Definition at line 62 of file prepairer_single_urls.py.

def cutURL(url):
  a = urlparse(url).netloc.split(":")[0].split(".")
  arr = None
  b = None
  if len(a) > 2 and a[-3] != "www":
    arr = a[-3:]
    b = str(arr[-3] + "." + arr[-2] + "." + arr[-1])
  else:
    arr = a[-2:]
    b = str(arr[-2] + "." + arr[-1])
  return b

Here is the caller graph for this function:

executeQuery()

def prepairer_single_urls.executeQuery	(		db_connector,
			query
	)

Definition at line 90 of file prepairer_single_urls.py.

def executeQuery(db_connector, query):
  try:
    with closing(db_connector.cursor(MySQLdb.cursors.DictCursor)) as cursor:
      cursor.execute(query)
      db_connector.commit()
      return cursor.fetchall()
  except mdb.Error as err:  # @todo logging in db_task
    db_connector.rollback()
    raise

Here is the caller graph for this function:

generateTemplates()

def prepairer_single_urls.generateTemplates

(

)

Definition at line 75 of file prepairer_single_urls.py.

def generateTemplates():
  templates = readTemplatesFromMySQL()
  print templates
  for template in templates:
    print template
    site_templates_dic[template["URL"]] = template["Value"]
  with open("sites_templates_dic", "w") as f:
    f.write(json.dumps(site_templates_dic))
  for (key, value) in site_templates_dic.items():
    url = cutURL(key)
    md5 = hashlib.md5(url).hexdigest()
    templates_dic[md5] = MySQLdb.escape_string(value)
  pass

Here is the call graph for this function:

loadDBBackend()

def prepairer_single_urls.loadDBBackend

(

)

Definition at line 101 of file prepairer_single_urls.py.

def loadDBBackend():
  global db_connector
  global dc_sites_db_connect
  global dc_urls_db_connect

  dbHost = "127.0.0.1"
  dbPort = 3306
  dbUser = "hce"
  dbPWD = "hce12345"

  db_dc_sites = "dc_sites"
  db_dc_urls = "dc_urls"

  dc_sites_db_connect = mdb.connect(dbHost, dbUser, dbPWD, db_dc_sites, dbPort)
  dc_urls_db_connect = mdb.connect(dbHost, dbUser, dbPWD, db_dc_urls, dbPort)

processBatch()

def prepairer_single_urls.processBatch

(

)

Definition at line 207 of file prepairer_single_urls.py.

def processBatch():
  # input_url = sys.stdin.read()[0:-1]
  for input_url in sys.stdin:
    input_url = input_url.strip()
    logger.debug(input_url)
    # site_url = "http://" + urlparse(urllib.unquote(input_url).decode('utf8')).hostname
    # site = createSiteObj(site_url)
    site = createSiteObj(input_url)
    open(site.id, "w").write(input_url)
    url = createURLObj(site, input_url)
    addSite(site)
    addURL(url, site)
    # updateURL(input_url, site)
    # updateSite(site)
    time.sleep(1)
    # bItem = BatchItem(site.id, url_updated.urlMd5)
    bItem = BatchItem(site.id, url.urlMd5, url)
    url_list = [bItem]
    input_object = Batch(11, url_list)
    input_pickled_object = pickle.dumps(input_object)
    print input_pickled_object

Here is the call graph for this function:

readTemplatesFromMySQL()

def prepairer_single_urls.readTemplatesFromMySQL

(

)

Definition at line 55 of file prepairer_single_urls.py.

def readTemplatesFromMySQL():
  query = "SELECT sites_urls.URL, sites_properties.`Value` FROM  `sites_properties` INNER JOIN sites_urls ON sites_urls.Site_Id = sites_properties.Site_Id AND sites_properties.Name = 'template'"
  print query
  rows = executeQuery(dc_sites_db_connect, query)
  return rows

Here is the call graph for this function:

Here is the caller graph for this function:

updateSite()

def prepairer_single_urls.updateSite

(

site

)

Definition at line 195 of file prepairer_single_urls.py.

def updateSite(site):
  site_updated = SiteUpdate(site.id)
  site_updated.state = Site.STATE_ACTIVE
  file_name = "updated_site_" + str(site_updated.id) + ".json"
  open(file_name, "w").write(site_updated.toJSON())
  cmd = "./dc-client.py --config=../ini/dc-client.ini --command=SITE_UPDATE --file=./%s" % file_name
  process = Popen(cmd, stdout=PIPE, stdin=PIPE, shell=True, close_fds=True)
  (output, err) = process.communicate()
  exit_code = process.wait()
  open("dc-client_update_site_output.txt", "w").write(output)

updateURL()

def prepairer_single_urls.updateURL	(		input_url,
			site
	)

Definition at line 181 of file prepairer_single_urls.py.

def updateURL(input_url, site):
  url_updated = URLUpdate(site.id, input_url)
  url_updated.status = URL.STATUS_SELECTED_CRAWLING
  # url_updated.type = URL.TYPE_SINGLE
  file_name = "url_" + str(url_updated.urlMd5) + ".json"
  open(file_name, "w").write("[" + url_updated.toJSON() + "]")
  cmd = "./dc-client.py --config=../ini/dc-client.ini --command=URL_UPDATE --file=./%s" % file_name
  process = Popen(cmd, stdout=PIPE, stdin=PIPE, shell=True, close_fds=True)
  (output, err) = process.communicate()
  exit_code = process.wait()
  open("dc-client_update_url_output.txt", "w").write(output)
  return url_updated

Variable Documentation

db_connector

prepairer_single_urls.db_connector = None

Definition at line 46 of file prepairer_single_urls.py.

dc_sites_db_connect

prepairer_single_urls.dc_sites_db_connect = None

Definition at line 47 of file prepairer_single_urls.py.

dc_urls_db_connect

prepairer_single_urls.dc_urls_db_connect = None

Definition at line 48 of file prepairer_single_urls.py.

filemode

prepairer_single_urls.filemode

Definition at line 42 of file prepairer_single_urls.py.

filename

prepairer_single_urls.filename

Definition at line 42 of file prepairer_single_urls.py.

logger

prepairer_single_urls.logger = logging.getLogger("Prepairer")

Definition at line 43 of file prepairer_single_urls.py.

site_templates_dic

dictionary prepairer_single_urls.site_templates_dic = {}

Definition at line 51 of file prepairer_single_urls.py.

templates_dic

dictionary prepairer_single_urls.templates_dic = {}

Definition at line 52 of file prepairer_single_urls.py.