HCE Project Python language Distributed Tasks Manager Application, Distributed Crawler Application and client API bindings.
2.0.0-chaika
Hierarchical Cluster Engine Python language binding
url_fetch_json_to_db-task_convertor.py
Go to the documentation of this file.
1
#!/usr/bin/python
2
3
"""
4
HCE project, Python bindings, Distributed Tasks Manager application.
5
Converter of the list of the URLs object from the URLFetch request to the DBTask.
6
7
@package: app
8
@file URLFetchToJsonDBTaskConvertor.py
9
@author Oleksii <developers.hce@gmail.com>, Alexander Vybornyh <alexander.hce.cluster@gmail.com>
10
@link: http://hierarchical-cluster-engine.com/
11
@copyright: Copyright © 2013-2015 IOIX Ukraine
12
@license: http://hierarchical-cluster-engine.com/license/
13
@since: 0.1
14
"""
15
16
import
ppath
17
from
ppath
import
sys
18
19
# For profiling
20
import
app.Profiler
as
Profiler
21
22
23
# Start profiling
24
pr = Profiler.Profiler()
25
if
pr
and
pr.status > 0:
26
pr.start()
27
28
29
# import pickle
30
import
os
31
import
sys
32
import
app.Utils
as
Utils
33
import
app.Consts
as
APP_CONSTS
34
35
from
app.UrlFetchJsonToDBTaskConvertor
import
UrlFetchToJsonDBTaskConvertor
36
37
38
# That script create main URLFetch request to the DBTask application
39
40
41
app =
None
42
exit_code = APP_CONSTS.EXIT_FAILURE
43
44
if
__name__ ==
"__main__"
:
45
try
:
46
# create the app
47
app =
UrlFetchToJsonDBTaskConvertor
()
48
# setup the application
49
app.setup()
50
# add support command line arguments
51
app.args.add_argument(
'-c'
,
'--config'
, action=
'store'
, metavar=
'config_file'
, help=
'config ini-file'
,
52
required=
True
)
53
54
# run the application
55
app.run()
56
57
# get exit code
58
exit_code = app.exitCode
59
60
# log message about profiler
61
if
pr.errorMsg
and
app.logger:
62
app.logger.error(pr.errorMsg)
63
64
except
Exception, err:
65
sys.stderr.write(str(err) +
'\n'
)
66
exit_code = APP_CONSTS.EXIT_FAILURE
67
except
:
68
exit_code = APP_CONSTS.EXIT_FAILURE
69
finally
:
70
# stop profiling
71
if
pr:
72
pr.stop()
73
# close the app
74
if
app:
75
app.close()
76
77
sys.stdout.flush()
78
os._exit(exit_code)
app.Utils
Definition:
Utils.py:1
app.UrlFetchJsonToDBTaskConvertor.UrlFetchToJsonDBTaskConvertor
Definition:
UrlFetchJsonToDBTaskConvertor.py:37
app.Consts
Definition:
Consts.py:1
app.UrlFetchJsonToDBTaskConvertor
Definition:
UrlFetchJsonToDBTaskConvertor.py:1
app.Profiler
Definition:
Profiler.py:1
sources
bin
url_fetch_json_to_db-task_convertor.py
Generated on Fri Nov 24 2017 18:53:53 for HCE Project Python language Distributed Tasks Manager Application, Distributed Crawler Application and client API bindings. by
1.8.13