prepairer_regular_urls Namespace Reference

Functions
def	executeQuery (db_connector, query)
def	loadDBBackend ()
def	createSiteObj (input_url)
def	createURLObj (site, input_url)
def	addSite (site)
def	addURL (url, site)
def	updateURL (input_url, site)
def	updateSite (site)
def	processBatch ()

Variables
	filename
	filemode
	logger = logging.getLogger("Prepairer")
	db_connector = None
	dc_sites_db_connect = None
	dc_urls_db_connect = None

Detailed Description

HCE project, Python bindings, Distributed Tasks Manager application.
Event objects definitions.

@package: dc
@file prepairer.py
@author Oleksii <developers.hce@gmail.com>
@link: http://hierarchical-cluster-engine.com/
@copyright: Copyright &copy; 2013-2014 IOIX Ukraine
@license: http://hierarchical-cluster-engine.com/license/
@since: 0.1

Function Documentation

addSite()

def prepairer_regular_urls.addSite

(

site

)

Definition at line 94 of file prepairer_regular_urls.py.

def addSite(site):
  file_name = "site_" + str(site.id) + ".json"
  open(file_name, "w").write(site.toJSON())
  cmd = "./dc-client.py --config=../ini/dc-client.ini --command=SITE_NEW --file=./%s" % file_name
  process = Popen(cmd, stdout=PIPE, stdin=PIPE, shell=True, close_fds=True)
  (output, err) = process.communicate()
  exit_code = process.wait()
  open("dc-client_new_site_output.txt", "w").write(output)
  return exit_code

Here is the caller graph for this function:

addURL()

def prepairer_regular_urls.addURL	(		url,
			site
	)

Definition at line 105 of file prepairer_regular_urls.py.

def addURL(url, site):
  file_name = "url_" + str(site.id) + ".json"
  open(file_name, "w").write("[" + url.toJSON() + "]")
  cmd = "./dc-client.py --config=../ini/dc-client.ini --command=URL_NEW --file=./%s" % file_name
  process = Popen(cmd, stdout=PIPE, stdin=PIPE, shell=True, close_fds=True)
  (output, err) = process.communicate()
  exit_code = process.wait()
  open("dc-client_new_url_output.txt", "w").write(output)
  return exit_code

createSiteObj()

def prepairer_regular_urls.createSiteObj

(

input_url

)

Definition at line 77 of file prepairer_regular_urls.py.

def createSiteObj(input_url):
  site = Site(input_url)
  # site.urls = []
  # site.maxResources = 5
  # site.maxURLs = 5
  # site.state = Site.STATE_SUSPENDED
  # site.filters = [SiteFilter(site.id, "(.*)")]
  return site

Here is the caller graph for this function:

createURLObj()

def prepairer_regular_urls.createURLObj	(		site,
			input_url
	)

Definition at line 87 of file prepairer_regular_urls.py.

def createURLObj(site, input_url):
  url = URL(site.id, input_url)
  url.status = URL.STATUS_SELECTED_CRAWLING
  url.type = URL.TYPE_SINGLE
  return url

Here is the caller graph for this function:

executeQuery()

def prepairer_regular_urls.executeQuery	(		db_connector,
			query
	)

Definition at line 49 of file prepairer_regular_urls.py.

def executeQuery(db_connector, query):
  try:
    with closing(db_connector.cursor(MySQLdb.cursors.DictCursor)) as cursor:
      cursor.execute(query)
      db_connector.commit()
      return cursor.fetchall()
  except mdb.Error as err:  # @todo logging in db_task
    db_connector.rollback()
    raise

loadDBBackend()

def prepairer_regular_urls.loadDBBackend

(

)

Definition at line 60 of file prepairer_regular_urls.py.

def loadDBBackend():
  global db_connector
  global dc_sites_db_connect
  global dc_urls_db_connect

  dbHost = "127.0.0.1"
  dbPort = 3306
  dbUser = "hce"
  dbPWD = "hce12345"

  db_dc_sites = "dc_sites"
  db_dc_urls = "dc_urls"

  dc_sites_db_connect = mdb.connect(dbHost, dbUser, dbPWD, db_dc_sites, dbPort)
  dc_urls_db_connect = mdb.connect(dbHost, dbUser, dbPWD, db_dc_urls, dbPort)

Here is the caller graph for this function:

processBatch()

def prepairer_regular_urls.processBatch

(

)

Definition at line 142 of file prepairer_regular_urls.py.

def processBatch():
  # input_url = sys.stdin.read()[0:-1]
  for input_url in sys.stdin:
    # input_url = input_url.strip()
    logger.debug(input_url)
    # site_url = "http://" + urlparse(urllib.unquote(input_url).decode('utf8')).hostname
    # site = createSiteObj(site_url)
    site = createSiteObj(input_url)
    open(site.id, "w").write(input_url)
    url = createURLObj(site, input_url)
    addSite(site)
    # addURL(url, site)
    # updateURL(input_url, site)
    # updateSite(site)
    time.sleep(1)
    # bItem = BatchItem(site.id, url_updated.urlMd5)
    bItem = BatchItem(site.id, site.id, url)
    url_list = [bItem]
    input_object = Batch(11, url_list)
    input_pickled_object = pickle.dumps(input_object)
    print input_pickled_object

Here is the call graph for this function:

updateSite()

def prepairer_regular_urls.updateSite

(

site

)

Definition at line 130 of file prepairer_regular_urls.py.

def updateSite(site):
  site_updated = SiteUpdate(site.id)
  site_updated.state = Site.STATE_ACTIVE
  file_name = "updated_site_" + str(site_updated.id) + ".json"
  open(file_name, "w").write(site_updated.toJSON())
  cmd = "./dc-client.py --config=../ini/dc-client.ini --command=SITE_UPDATE --file=./%s" % file_name
  process = Popen(cmd, stdout=PIPE, stdin=PIPE, shell=True, close_fds=True)
  (output, err) = process.communicate()
  exit_code = process.wait()
  open("dc-client_update_site_output.txt", "w").write(output)

updateURL()

def prepairer_regular_urls.updateURL	(		input_url,
			site
	)

Definition at line 116 of file prepairer_regular_urls.py.

def updateURL(input_url, site):
  url_updated = URLUpdate(site.id, input_url)
  url_updated.status = URL.STATUS_SELECTED_CRAWLING
  url_updated.type = URL.TYPE_SINGLE
  file_name = "url_" + str(url_updated.urlMd5) + ".json"
  open(file_name, "w").write("[" + url_updated.toJSON() + "]")
  cmd = "./dc-client.py --config=../ini/dc-client.ini --command=URL_UPDATE --file=./%s" % file_name
  process = Popen(cmd, stdout=PIPE, stdin=PIPE, shell=True, close_fds=True)
  (output, err) = process.communicate()
  exit_code = process.wait()
  open("dc-client_update_url_output.txt", "w").write(output)
  return url_updated

Here is the caller graph for this function:

Variable Documentation

db_connector

prepairer_regular_urls.db_connector = None

Definition at line 44 of file prepairer_regular_urls.py.

dc_sites_db_connect

prepairer_regular_urls.dc_sites_db_connect = None

Definition at line 45 of file prepairer_regular_urls.py.

dc_urls_db_connect

prepairer_regular_urls.dc_urls_db_connect = None

Definition at line 46 of file prepairer_regular_urls.py.

filemode

prepairer_regular_urls.filemode

Definition at line 40 of file prepairer_regular_urls.py.

filename

prepairer_regular_urls.filename

Definition at line 40 of file prepairer_regular_urls.py.

logger

prepairer_regular_urls.logger = logging.getLogger("Prepairer")

Definition at line 41 of file prepairer_regular_urls.py.