Functions
def	processFullBatch (input_object)

Variables
	siteId1 = str(md5.new("http://www.yomiuri.co.jp").hexdigest())

	urlId1 = str(md5.new("http://www.yomiuri.co.jp/sports/mlb/20140407-OYT1T50015.html?from=ytop_top").hexdigest())

	bItem1 = BatchItem(siteId1, urlId1)

	siteId2 = str(md5.new("http://localhost/www.yomiuri.co.jp").hexdigest())

	urlId2 = str(md5.new("http://localhost/www.yomiuri.co.jp/template1.html").hexdigest())

	bItem2 = BatchItem(siteId2, urlId2)

string	siteId3 = ""

	urlId3 = str(md5.new("http://localhost/www.yomiuri.co.jp/template1.html").hexdigest())

	bItem3 = BatchItem(siteId3, urlId3)

list	url_list

string	PWD = "cd ../../bin"

string	PYTHON_BINARY = "/usr/bin/python"

string	CRAWLER_TASK_BINARY = "./crawler-task.py"

string	CRAWLER_TASK_CFG = "--config=../ini/crawler-task.ini"

string	PROCESSOR_TASK_BINARY = "./processor-task.py"

string	PROCESSOR_TASK_CFG = "--config=../ini/processor-task.ini"

	Results = namedtuple("Results", "exit_code, output, err")

	input_object = Batch(1,url_list)

def	result = processFullBatch(input_object)

	response = pickle.loads(result.output)

Detailed Description

HCE project, Python bindings, Distributed Tasks Manager application.
Event objects definitions.

@package: dc
@file ftest_dc_full_batch_processing.py
@author Oleksii <developers.hce@gmail.com>
@link: http://hierarchical-cluster-engine.com/
@copyright: Copyright &copy; 2013-2014 IOIX Ukraine
@license: http://hierarchical-cluster-engine.com/license/
@since: 0.1

Function Documentation

◆ processFullBatch()

def ftests.ftest_dc_full_batch_processing.processFullBatch ( input_object )

Definition at line 65 of file ftest_dc_full_batch_processing.py.

 def processFullBatch(input_object):
   input_pickled_object = pickle.dumps(input_object)
   #process = Popen([PYTHON_BINARY, CRAWLER_TASK_BINARY, CRAWLER_TASK_CFG, " | ", PYTHON_BINARY, PROCESSOR_TASK_BINARY, PROCESSOR_TASK_CFG], stdout=PIPE, stdin=PIPE, shell=True)
   process = Popen(PWD+" && "+PYTHON_BINARY+" "+CRAWLER_TASK_BINARY+" "+CRAWLER_TASK_CFG+" | "+PYTHON_BINARY+" "+PROCESSOR_TASK_BINARY+" "+PROCESSOR_TASK_CFG, stdout=PIPE, stdin=PIPE, shell=True)
   (output, err) = process.communicate(input=input_pickled_object)
   #print output
   exit_code = process.wait()
   return Results(exit_code, output, err)
 
 

Variable Documentation

◆ bItem1

ftests.ftest_dc_full_batch_processing.bItem1 = BatchItem(siteId1, urlId1)

Definition at line 34 of file ftest_dc_full_batch_processing.py.

◆ bItem2

ftests.ftest_dc_full_batch_processing.bItem2 = BatchItem(siteId2, urlId2)

Definition at line 38 of file ftest_dc_full_batch_processing.py.

◆ bItem3

ftests.ftest_dc_full_batch_processing.bItem3 = BatchItem(siteId3, urlId3)

Definition at line 42 of file ftest_dc_full_batch_processing.py.

◆ CRAWLER_TASK_BINARY

string ftests.ftest_dc_full_batch_processing.CRAWLER_TASK_BINARY = "./crawler-task.py"

Definition at line 56 of file ftest_dc_full_batch_processing.py.

◆ CRAWLER_TASK_CFG

string ftests.ftest_dc_full_batch_processing.CRAWLER_TASK_CFG = "--config=../ini/crawler-task.ini"

Definition at line 57 of file ftest_dc_full_batch_processing.py.

◆ input_object

ftests.ftest_dc_full_batch_processing.input_object = Batch(1,url_list)

Definition at line 78 of file ftest_dc_full_batch_processing.py.

◆ PROCESSOR_TASK_BINARY

string ftests.ftest_dc_full_batch_processing.PROCESSOR_TASK_BINARY = "./processor-task.py"

Definition at line 58 of file ftest_dc_full_batch_processing.py.

◆ PROCESSOR_TASK_CFG

string ftests.ftest_dc_full_batch_processing.PROCESSOR_TASK_CFG = "--config=../ini/processor-task.ini"

Definition at line 59 of file ftest_dc_full_batch_processing.py.

◆ PWD

string ftests.ftest_dc_full_batch_processing.PWD = "cd ../../bin"

Definition at line 54 of file ftest_dc_full_batch_processing.py.

◆ PYTHON_BINARY

string ftests.ftest_dc_full_batch_processing.PYTHON_BINARY = "/usr/bin/python"

Definition at line 55 of file ftest_dc_full_batch_processing.py.

◆ response

ftests.ftest_dc_full_batch_processing.response = pickle.loads(result.output)

Definition at line 82 of file ftest_dc_full_batch_processing.py.

◆ result

def ftests.ftest_dc_full_batch_processing.result = processFullBatch(input_object)

Definition at line 80 of file ftest_dc_full_batch_processing.py.

◆ Results

ftests.ftest_dc_full_batch_processing.Results = namedtuple("Results", "exit_code, output, err")

Definition at line 62 of file ftest_dc_full_batch_processing.py.

◆ siteId1

ftests.ftest_dc_full_batch_processing.siteId1 = str(md5.new("http://www.yomiuri.co.jp").hexdigest())

Definition at line 32 of file ftest_dc_full_batch_processing.py.

◆ siteId2

ftests.ftest_dc_full_batch_processing.siteId2 = str(md5.new("http://localhost/www.yomiuri.co.jp").hexdigest())

Definition at line 36 of file ftest_dc_full_batch_processing.py.

◆ siteId3

string ftests.ftest_dc_full_batch_processing.siteId3 = ""

Definition at line 40 of file ftest_dc_full_batch_processing.py.

◆ url_list

list ftests.ftest_dc_full_batch_processing.url_list

Initial value:

 =  [
   #bItem1,
   bItem2,
   bItem3
   #"http://192.168.1.61/article1.html"
   #"http://www.yomiuri.co.jp/sports/mlb/20140407-OYT1T50015.html?from=ytop_top"
   ]

Definition at line 45 of file ftest_dc_full_batch_processing.py.

◆ urlId1

ftests.ftest_dc_full_batch_processing.urlId1 = str(md5.new("http://www.yomiuri.co.jp/sports/mlb/20140407-OYT1T50015.html?from=ytop_top").hexdigest())

Definition at line 33 of file ftest_dc_full_batch_processing.py.

◆ urlId2

ftests.ftest_dc_full_batch_processing.urlId2 = str(md5.new("http://localhost/www.yomiuri.co.jp/template1.html").hexdigest())

Definition at line 37 of file ftest_dc_full_batch_processing.py.

◆ urlId3

ftests.ftest_dc_full_batch_processing.urlId3 = str(md5.new("http://localhost/www.yomiuri.co.jp/template1.html").hexdigest())

Definition at line 41 of file ftest_dc_full_batch_processing.py.

Functions

Variables