HCE Project Python language Distributed Tasks Manager Application, Distributed Crawler Application and client API bindings.
2.0.0-chaika
Hierarchical Cluster Engine Python language binding
scraper.py
Go to the documentation of this file.
1
#!/usr/bin/python
2
3
4
"""
5
HCE project, Python bindings, Distributed Tasks Manager application.
6
Event objects definitions.
7
8
@package: dc
9
@file scraper.py
10
@author Oleksii <developers.hce@gmail.com>
11
@link: http://hierarchical-cluster-engine.com/
12
@copyright: Copyright © 2013-2014 IOIX Ukraine
13
@license: http://hierarchical-cluster-engine.com/license/
14
@since: 0.1
15
"""
16
17
18
import
ppath
19
from
ppath
import
sys
20
21
# For profiling
22
import
app.Profiler
as
Profiler
23
24
25
# Start profiling
26
pr = Profiler.Profiler()
27
if
pr
and
pr.status > 0:
28
pr.start()
29
30
31
import
pickle
32
import
os
33
import
sys
34
from
dc_processor.Scraper
import
Scraper
35
from
dc_processor.Scraper
import
EXIT_SUCCESS
36
from
dc_processor.Scraper
import
EXIT_FAILURE
37
import
app.Consts
as
APP_CONSTS
38
39
40
# That script create main application
41
42
43
app =
None
44
exit_code = EXIT_SUCCESS
45
46
try
:
47
# create the app
48
app =
Scraper
()
49
# setup the application
50
app.setup()
51
# add support command line arguments
52
app.args.add_argument(
'-c'
,
'--config'
, action=
'store'
, metavar=
'config_file'
, help=
'config ini-file'
, required=
True
)
53
54
# run the application
55
app.run()
56
57
exit_code = app.getExitCode()
58
59
# log message about profiler
60
if
pr.errorMsg
and
app.logger:
61
app.logger.error(pr.errorMsg)
62
63
except
Exception
as
err:
64
sys.stderr.write(str(err) +
'\n'
)
65
exit_code = EXIT_FAILURE
66
except
:
67
exit_code = EXIT_FAILURE
68
finally
:
69
# stop profiling
70
if
pr:
71
pr.stop()
72
# close the app
73
if
app:
74
app.close()
75
76
sys.stdout.flush()
77
os._exit(exit_code)
dc_processor.Scraper
Definition:
Scraper.py:1
dc_processor.Scraper.Scraper
Definition:
Scraper.py:106
app.Consts
Definition:
Consts.py:1
app.Profiler
Definition:
Profiler.py:1
sources
bin
scraper.py
Generated on Fri Nov 24 2017 18:53:53 for HCE Project Python language Distributed Tasks Manager Application, Distributed Crawler Application and client API bindings. by
1.8.13