generate_sites_jsons Namespace Reference

Functions
def	readTemplatesFromMySQL ()
def	cutURL (url)
def	generateTemplates ()
def	fillTemplates ()
def	readTemplatesFromFile ()
def	executeQuery (db_connector, query)
def	loadDBBackend ()
def	createSiteObj (input_url)
def	addSite (site)

Variables
	filename
	filemode
	logger = logging.getLogger("Prepairer")
	db_connector = None
	dc_sites_db_connect = None
	dc_urls_db_connect = None
dictionary	site_templates_dic = {}
dictionary	templates_dic = {}
def	site = createSiteObj(input_url)

Detailed Description

HCE project, Python bindings, Distributed Tasks Manager application.
Event objects definitions.

@package: dc
@file prepairer.py
@author Oleksii <developers.hce@gmail.com>
@link: http://hierarchical-cluster-engine.com/
@copyright: Copyright &copy; 2013-2014 IOIX Ukraine
@license: http://hierarchical-cluster-engine.com/license/
@since: 0.1

Function Documentation

addSite()

def generate_sites_jsons.addSite

(

site

)

Definition at line 169 of file generate_sites_jsons.py.

def addSite(site):
  file_name = "site_" + str(site.id) + ".json"
  open(file_name, "w").write(site.toJSON())

Here is the call graph for this function:

createSiteObj()

def generate_sites_jsons.createSiteObj

(

input_url

)

Definition at line 129 of file generate_sites_jsons.py.

def createSiteObj(input_url):

  # strip input url
  input_url = input_url.strip()

  # root url
  root_url = input_url

  # get url for md5
  norm_url = cutURL(input_url)
  # norm_url = input_url

  # create site
  site = Site(norm_url)
  site.urls = [input_url]
  # create site filters
  # site_filter_pattern = ".*" + norm_url + ".*"
  site_filter_pattern = ".*" + cutURL(input_url) + ".*"
  site_filters = SiteFilter(site.id, site_filter_pattern)


  # create site properties templates
  print templates_dic
  print site.id
  if site.id in templates_dic:
    site.properties["template"] = templates_dic[site.id]

  # fill site
  # site.urls = [root_url]
  # site.urls = []
  site.filters = [site_filters]
  site.maxResources = 100000
  site.maxURLs = 100000
  site.maxErrors = 100000
  site.maxResourceSize = 1000000
  # site.state = Site.STATE_SUSPENDED
  # site.filters = [SiteFilter(site.id, "(.*)")]
  return site

Here is the call graph for this function:

cutURL()

def generate_sites_jsons.cutURL

(

url

)

Definition at line 62 of file generate_sites_jsons.py.

def cutURL(url):
  b = url
  arr = None
  a = urlparse(url).netloc.split(":")[0].split(".")
  if len(a) > 2 and a[-3] != "www":
    arr = a[-3:]
    b = str(arr[-3] + "." + arr[-2] + "." + arr[-1])
  else:
    arr = a[-2:]
    b = str(arr[-2] + "." + arr[-1])
  return b

Here is the caller graph for this function:

executeQuery()

def generate_sites_jsons.executeQuery	(		db_connector,
			query
	)

Definition at line 101 of file generate_sites_jsons.py.

def executeQuery(db_connector, query):
  try:
    with closing(db_connector.cursor(MySQLdb.cursors.DictCursor)) as cursor:
      cursor.execute(query)
      db_connector.commit()
      return cursor.fetchall()
  except mdb.Error as err:  # @todo logging in db_task
    db_connector.rollback()
    raise

Here is the caller graph for this function:

fillTemplates()

def generate_sites_jsons.fillTemplates

(

)

Definition at line 85 of file generate_sites_jsons.py.

def fillTemplates():
  global site_templates_dic
  print site_templates_dic
  for (key, value) in site_templates_dic.items():
    url = cutURL(key)
    md5 = hashlib.md5(url).hexdigest()
    templates_dic[md5] = MySQLdb.escape_string(value)

Here is the call graph for this function:

Here is the caller graph for this function:

generateTemplates()

def generate_sites_jsons.generateTemplates

(

)

Definition at line 75 of file generate_sites_jsons.py.

def generateTemplates():
  templates = readTemplatesFromMySQL()
  print templates
  for template in templates:
    # print template
    site_templates_dic[template["URL"]] = template["Value"]
  with open("sites_templates_dic", "w") as f:
    f.write(json.dumps(site_templates_dic))

Here is the call graph for this function:

loadDBBackend()

def generate_sites_jsons.loadDBBackend

(

)

Definition at line 112 of file generate_sites_jsons.py.

def loadDBBackend():
  global db_connector
  global dc_sites_db_connect
  global dc_urls_db_connect

  dbHost = "127.0.0.1"
  dbPort = 3306
  dbUser = "hce"
  dbPWD = "hce12345"

  db_dc_sites = "dc_sites"
  db_dc_urls = "dc_urls"

  dc_sites_db_connect = mdb.connect(dbHost, dbUser, dbPWD, db_dc_sites, dbPort)
  dc_urls_db_connect = mdb.connect(dbHost, dbUser, dbPWD, db_dc_urls, dbPort)

Here is the caller graph for this function:

readTemplatesFromFile()

def generate_sites_jsons.readTemplatesFromFile

(

)

Definition at line 94 of file generate_sites_jsons.py.

def readTemplatesFromFile():
  global site_templates_dic
  with open("sites_templates_dic", "r") as f:
    site_templates_dic = json.loads(f.read())
  print site_templates_dic

Here is the caller graph for this function:

readTemplatesFromMySQL()

def generate_sites_jsons.readTemplatesFromMySQL

(

)

Definition at line 55 of file generate_sites_jsons.py.

def readTemplatesFromMySQL():
  query = "SELECT sites_urls.URL, sites_properties.`Value` FROM  `sites_properties` INNER JOIN sites_urls ON sites_urls.Site_Id = sites_properties.Site_Id AND sites_properties.Name = 'template'"
  print query
  rows = executeQuery(dc_sites_db_connect, query)
  return rows

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

db_connector

generate_sites_jsons.db_connector = None

Definition at line 46 of file generate_sites_jsons.py.

dc_sites_db_connect

generate_sites_jsons.dc_sites_db_connect = None

Definition at line 47 of file generate_sites_jsons.py.

dc_urls_db_connect

generate_sites_jsons.dc_urls_db_connect = None

Definition at line 48 of file generate_sites_jsons.py.

filemode

generate_sites_jsons.filemode

Definition at line 42 of file generate_sites_jsons.py.

filename

generate_sites_jsons.filename

Definition at line 42 of file generate_sites_jsons.py.

logger

generate_sites_jsons.logger = logging.getLogger("Prepairer")

Definition at line 43 of file generate_sites_jsons.py.

site

def generate_sites_jsons.site = createSiteObj(input_url)

Definition at line 181 of file generate_sites_jsons.py.

site_templates_dic

dictionary generate_sites_jsons.site_templates_dic = {}

Definition at line 51 of file generate_sites_jsons.py.

templates_dic

dictionary generate_sites_jsons.templates_dic = {}

Definition at line 52 of file generate_sites_jsons.py.