HCE Project Python language Distributed Tasks Manager Application, Distributed Crawler Application and client API bindings.  2.0.0-chaika
Hierarchical Cluster Engine Python language binding
ftests.ftest_PostprocessorTask Namespace Reference

Functions

def getLogger ()
 
def getFileLogger ()
 
def executeCommand (cmd, inputStream=None, log=None)
 
def generateBatch (id=0)
 
def generateInputStream (id=0)
 
def test (id=0, log=None)
 
def threadRun (id=0, log=None)
 

Variables

def logger = getLogger()
 
int testCount = 5
 
list threadsList = []
 

Function Documentation

◆ executeCommand()

def ftests.ftest_PostprocessorTask.executeCommand (   cmd,
  inputStream = None,
  log = None 
)

Definition at line 76 of file ftest_PostprocessorTask.py.

76 def executeCommand(cmd, inputStream=None, log=None):
77  if log is not None:
78  log.debug("Popen: %s", str(cmd))
79  process = Popen(cmd, stdout=PIPE, stdin=PIPE, stderr=PIPE, shell=True, close_fds=True)
80  if isinstance(inputStream, basestring) and log is not None:
81  log.debug("process.communicate(), len(inputStream)=" + str(len(inputStream)))
82  (output, err) = process.communicate(input=inputStream)
83  if log is not None:
84  log.debug("Process std_error: %s", str(err))
85  log.debug("Process output len =" + str(len(output)))
86  exitCode = process.wait()
87  if log is not None:
88  log.debug("Process response exitCode: %s", str(exitCode))
89 
90  return output, exitCode
91 
92 
def executeCommand(cmd, inputStream=None, log=None)
Here is the caller graph for this function:

◆ generateBatch()

def ftests.ftest_PostprocessorTask.generateBatch (   id = 0)

Definition at line 93 of file ftest_PostprocessorTask.py.

93 def generateBatch(id=0):
94  siteId = id
95  url = 'https://retrip.jp/external-link/?article_content_id=482406'
96  urlObj = URL(siteId, url)
97 
98  processedContent = {'link':url}
99  processedContents = [base64.b64encode(json.dumps(processedContent))]
100  urlContentResponse = URLContentResponse(url=url, processedContents=processedContents)
101 
102  batchItem = BatchItem(siteId=siteId, urlId=urlObj.urlMd5, urlObj=urlObj, urlContentResponse=urlContentResponse)
103  batchItem.properties = {"LINK_RESOLVE":{"method":{"retrip.jp/external-link":"GET"}}}
104  # batchItem.properties = {}
105  return Batch(id, batchItems=[batchItem])
106 
107 
Here is the caller graph for this function:

◆ generateInputStream()

def ftests.ftest_PostprocessorTask.generateInputStream (   id = 0)

Definition at line 108 of file ftest_PostprocessorTask.py.

108 def generateInputStream(id=0):
109  return pickle.dumps(generateBatch(id=0))
110 
111 
Here is the call graph for this function:
Here is the caller graph for this function:

◆ getFileLogger()

def ftests.ftest_PostprocessorTask.getFileLogger ( )

Definition at line 54 of file ftest_PostprocessorTask.py.

54 def getFileLogger():
55 
56  configName = '../ini/postprocessor_task_log-rt.ini'
57 
58  retval = os.getcwd()
59  os.chdir('..')
60  # read config
61  logging.config.fileConfig(configName)
62 
63  # create logger
64  log = Utils.MPLogger().getLogger()
65  # log = logging.getLogger(APP_CONSTS.LOGGER_NAME)
66  os.chdir(retval)
67 
68  return log
69 
70 
71 # # execute command line command
72 #
73 # @param cmd - command line string
74 # @param inputStream - input stream to popen
75 # @return result object of execution and exit code
Here is the call graph for this function:

◆ getLogger()

def ftests.ftest_PostprocessorTask.getLogger ( )

Definition at line 32 of file ftest_PostprocessorTask.py.

32 def getLogger():
33 
34  # create logger
35  logger = logging.getLogger('console')
36  logger.setLevel(logging.DEBUG)
37 
38  # create console handler and set level to debug
39  ch = logging.StreamHandler()
40  ch.setLevel(logging.DEBUG)
41 
42  # create formatter
43  formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
44 
45  # add formatter to ch
46  ch.setFormatter(formatter)
47 
48  # add ch to logger
49  logger.addHandler(ch)
50 
51  return logger
52 
53 
Here is the caller graph for this function:

◆ test()

def ftests.ftest_PostprocessorTask.test (   id = 0,
  log = None 
)

Definition at line 112 of file ftest_PostprocessorTask.py.

112 def test(id=0, log=None):
113 
114  inputFile = '/tmp/input.tmp'
115  outputFile = '/tmp/output.tmp'
116 
117  cmd = "cd ..; ../bin/postprocessor_task.py --config=../ini/postprocessor_task-rt.ini --inputFile=%s > %s" % (str(inputFile), str(outputFile))
118 
119  f = open(inputFile, 'w')
120  f.write(generateInputStream())
121  f.close()
122 
123  output, exitCode = executeCommand(cmd, log=log)
124 
125  if log is not None:
126  log.debug("len(output) = %s", str(len(output)))
127  log.debug("exitCode: %s", str(exitCode))
128  log.debug("===Finish===")
129 
130 
def executeCommand(cmd, inputStream=None, log=None)
Here is the call graph for this function:
Here is the caller graph for this function:

◆ threadRun()

def ftests.ftest_PostprocessorTask.threadRun (   id = 0,
  log = None 
)

Definition at line 131 of file ftest_PostprocessorTask.py.

131 def threadRun(id=0, log=None):
132 
133  sys.stdout.write("Thread ID = %s started.\n" % str(id))
134 
135  test(id=id, log=log)
136 
137  sys.stdout.write("Thread ID = %s stopped.\n" % str(id))
138 
139 
Here is the call graph for this function:

Variable Documentation

◆ logger

def ftests.ftest_PostprocessorTask.logger = getLogger()

Definition at line 142 of file ftest_PostprocessorTask.py.

◆ testCount

int ftests.ftest_PostprocessorTask.testCount = 5

Definition at line 145 of file ftest_PostprocessorTask.py.

◆ threadsList

list ftests.ftest_PostprocessorTask.threadsList = []

Definition at line 146 of file ftest_PostprocessorTask.py.