HCE Project Python language Distributed Tasks Manager Application, Distributed Crawler Application and client API bindings.
2.0.0-chaika
Hierarchical Cluster Engine Python language binding
Main Page
Related Pages
+
Namespaces
Namespace List
+
Namespace Members
+
All
_
a
b
c
d
e
f
g
h
i
j
k
l
m
n
o
p
q
r
s
t
u
v
w
x
+
Functions
_
a
b
c
d
e
f
g
i
j
l
m
o
p
r
s
t
u
v
w
+
Variables
_
a
b
c
d
e
f
g
h
i
j
k
l
m
n
o
p
q
r
s
t
u
v
w
x
+
Classes
Class List
Class Index
Class Hierarchy
+
Class Members
+
All
_
a
b
c
d
e
f
g
h
i
j
k
l
m
n
o
p
q
r
s
t
u
v
w
x
z
+
Functions
_
a
b
c
d
e
f
g
h
i
j
k
l
m
n
o
p
r
s
t
u
v
w
x
+
Variables
_
a
b
c
d
e
f
g
h
i
j
k
l
m
n
o
p
q
r
s
t
u
v
w
x
z
+
Files
File List
•
All
Classes
Namespaces
Files
Functions
Variables
Pages
urls-to-batch-task.py
Go to the documentation of this file.
1
#!/usr/bin/python
2
3
4
"""
5
HCE project, Python bindings, Distributed Tasks Manager application.
6
Converter of the list of the URLs object from the URLFetch request to the Batch object.
7
Used for the processing batching as part of the regular processing on DC service.
8
9
@package: dc
10
@file urls-to-batch-task.py
11
@author Oleksii, bgv <developers.hce@gmail.com>, Alexander Vybornyh <alexander.hce.cluster@gmail.com>
12
@link: http://hierarchical-cluster-engine.com/
13
@copyright: Copyright © 2013-2015 IOIX Ukraine
14
@license: http://hierarchical-cluster-engine.com/license/
15
@since: 0.1
16
"""
17
18
import
ppath
19
from
ppath
import
sys
20
21
# For profiling
22
import
app.Profiler
as
Profiler
23
24
25
# Start profiling
26
pr = Profiler.Profiler()
27
if
pr
and
pr.status > 0:
28
pr.start()
29
30
31
# import pickle
32
import
os
33
import
sys
34
import
app.Utils
as
Utils
35
import
app.Consts
as
APP_CONSTS
36
37
from
app.UrlsToBatchTask
import
UrlsToBatchTask
38
39
40
# That script create main urls-to-batch-task application
41
42
43
app =
None
44
exit_code = APP_CONSTS.EXIT_FAILURE
45
46
if
__name__ ==
"__main__"
:
47
try
:
48
# create the app
49
app =
UrlsToBatchTask
()
50
# setup the application
51
app.setup()
52
# add support command line arguments
53
app.args.add_argument(
'-c'
,
'--config'
, action=
'store'
, metavar=
'config_file'
, help=
'config ini-file'
,
54
required=
True
)
55
56
# run the application
57
app.run()
58
59
# get exit code
60
exit_code = app.exitCode
61
62
# log message about profiler
63
if
pr.errorMsg
and
app.logger:
64
app.logger.error(pr.errorMsg)
65
66
except
Exception, err:
67
sys.stderr.write(str(err) +
'\n'
)
68
exit_code = APP_CONSTS.EXIT_FAILURE
69
except
:
70
exit_code = APP_CONSTS.EXIT_FAILURE
71
finally
:
72
# stop profiling
73
if
pr:
74
pr.stop()
75
# close the app
76
if
app:
77
app.close()
78
79
sys.stdout.flush()
80
os._exit(exit_code)
81
app.UrlsToBatchTask
Definition:
UrlsToBatchTask.py:1
app.Utils
Definition:
Utils.py:1
app.UrlsToBatchTask.UrlsToBatchTask
Definition:
UrlsToBatchTask.py:42
app.Consts
Definition:
Consts.py:1
app.Profiler
Definition:
Profiler.py:1
sources
bin
urls-to-batch-task.py
Generated on Fri Nov 24 2017 18:53:53 for HCE Project Python language Distributed Tasks Manager Application, Distributed Crawler Application and client API bindings. by
1.8.13