HCE Project Python language Distributed Tasks Manager Application, Distributed Crawler Application and client API bindings.
2.0.0-chaika
Hierarchical Cluster Engine Python language binding
|
Classes | |
class | EVENT_TYPES |
Variables | |
DRCESyncTasksCover = namedtuple('DRCESyncTasksCover', 'eventType eventObject') | |
LOGGER_NAME = APP_CONSTS.LOGGER_NAME | |
string | BATCHES_CRAWL_COUNTER_TOTAL_NAME = "batches_crawl_total" |
string | BATCHES_CRAWL_COUNTER_QUEUE_NAME = "batches_crawl_queue" |
string | BATCHES_CRAWL_COUNTER_FAULT_NAME = "batches_crawl_fault" |
string | BATCHES_CRAWL_COUNTER_FILLED_NAME = "batches_crawl_filled" |
string | BATCHES_CRAWL_COUNTER_URLS_NAME = "batches_crawl_urls" |
string | BATCHES_CRAWL_COUNTER_URLS_FAULT_NAME = "batches_crawl_urls_fault" |
string | BATCHES_CRAWL_COUNTER_URL_FETCH_NAME = "batches_crawl_url_fetch" |
string | BATCHES_CRAWL_COUNTER_URL_FETCH_CANCELLED_NAME = "batches_crawl_url_fetch_cancelled" |
string | BATCHES_CRAWL_COUNTER_DELETE_FAULT_NAME = "batches_crawl_delete_fault" |
string | BATCHES_CRAWL_COUNTER_FAULT_TTL_NAME = "batches_crawl_fault_ttl" |
string | BATCHES_CRAWL_COUNTER_CHECK_FAULT_NAME = "batches_crawl_check_fault" |
string | BATCHES_CRAWL_COUNTER_URLS_RET_NAME = "batches_crawl_urls_ret" |
string | BATCHES_CRAWL_COUNTER_URL_FETCH_INCR_NAME = "batches_crawl_url_fetch_incr" |
string | SITES_RECRAWL_COUNTER_NAME = "sites_recrawl_cnt" |
string | SITES_RECRAWL_UPDATED_COUNTER_NAME = "sites_recrawl_updated_cnt" |
string | SITES_RECRAWL_DELETED_COUNTER_NAME = "sites_recrawl_deleted_cnt" |
string | SITES_DRCE_COUNTER_NAME = "sites_recrawl_drce_cnt" |
string | BATCHES_CRAWL_COUNTER_TIME_AVG_NAME = "batches_crawl_time_avg" |
string | BATCHES_REALTIME_THREADS_NAME = "batches_realtime_threads" |
string | BATCHES_REALTIME_THREADS_CREATED_COUNTER_NAME = "batches_realtime_threads_created" |
string | BATCHES_CRAWL_COUNTER_ITEMS_AVG_NAME = "batches_crawl_items_avg" |
string | BATCHES_CRAWL_COUNTER_FETCHER_DYNAMIC = "batches_crawl_fetcher_dynamic" |
string | BATCHES_CRAWL_COUNTER_FETCHER_STATIC = "batches_crawl_fetcher_static" |
string | BATCHES_CRAWL_COUNTER_FETCHER_MIXED = "batches_crawl_fetcher_mixed" |
string | BATCHES_CRAWL_COUNTER_URL_FETCH_REQUESTS_NAME = "batches_crawl_url_fetch_requests" |
string | RECRAWL_THREADS_COUNTER_QUEUE_NAME = "recrawl_threads" |
string | RECRAWL_SITES_QUEUE_NAME = "recrawl_sites_queue" |
string | RECRAWL_THREADS_CREATED_COUNTER_NAME = "recrawl_threads_created" |
string | COMMON_THREADS_COUNTER_QUEUE_NAME = "common_threads" |
string | COMMON_OPERATIONS_COUNTER_NAME = "common_operations_cnt" |
string | COMMON_THREADS_CREATED_COUNTER_NAME = "common_threads_created" |
string | BATCHES_PURGE_COUNTER_NAME = "purge_batches" |
string | BATCHES_PURGE_COUNTER_CANCELLED_NAME = "purge_batches_canceled" |
string | BATCHES_PURGE_COUNTER_TOTAL_NAME = "purge_batches_total" |
string | BATCHES_PURGE_COUNTER_ERROR_NAME = "purge_batches_error" |
string | BATCHES_PURGE_COUNTER_FAULT_NAME = "purge_batches_fault" |
string | BATCHES_PURGE_COUNTER_DELETE_FAULT_NAME = "purge_batches_delete_fault" |
string | BATCHES_PURGE_COUNTER_CHECK_FAULT_NAME = "purge_batches_check_fault" |
string | BATCHES_PROCESS_COUNTER_TOTAL_NAME = "batches_process_total" |
string | BATCHES_PROCESS_COUNTER_QUEUE_NAME = "batches_process_queue" |
string | BATCHES_PROCESS_COUNTER_FAULT_NAME = "batches_process_fault" |
string | BATCHES_PROCESS_COUNTER_FILLED_NAME = "batches_process_filled" |
string | BATCHES_PROCESS_COUNTER_URLS_NAME = "batches_process_urls" |
string | BATCHES_PROCESS_COUNTER_URLS_FAULT_NAME = "batches_process_urls_fault" |
string | BATCHES_PROCESS_COUNTER_DELETE_FAULT_NAME = "batches_process_delete_fault" |
string | BATCHES_PROCESS_COUNTER_CHECK_FAULT_NAME = "batches_process_check_fault" |
string | BATCHES_PROCESS_COUNTER_FAULT_TTL_NAME = "batches_process_fault_ttl" |
string | BATCHES_PROCESS_COUNTER_CANCELLED_NAME = "batches_process_cancelled" |
string | BATCHES_AGE_COUNTER_NAME = "age_batches" |
string | BATCHES_AGE_COUNTER_CANCELLED_NAME = "age_batches_canceled" |
string | BATCHES_AGE_COUNTER_TOTAL_NAME = "age_batches_total" |
string | BATCHES_AGE_COUNTER_ERROR_NAME = "age_batches_error" |
string | BATCHES_AGE_COUNTER_FAULT_NAME = "age_batches_fault" |
string | BATCHES_AGE_COUNTER_DELETE_FAULT_NAME = "age_batches_delete_fault" |
string | BATCHES_AGE_COUNTER_CHECK_FAULT_NAME = "age_batches_check_fault" |
string | INCR_MIN_FREQ_CONFIG_VAR_NAME = "INCR_MIN_FREQ" |
string | INCR_MAX_DEPTH_CONFIG_VAR_NAME = "INCR_MAX_DEPTH" |
string | INCR_MAX_URLS_CONFIG_VAR_NAME = "INCR_MAX_URLS" |
string | MERGE_PARAM_NAME = "MERGE_RESULTS" |
string | RAW_DATA_SUFF = ".bin" |
string | RAW_DATA_HEADERS_SUFF = ".headers.txt" |
string | RAW_DATA_REQESTS_SUFF = ".requests.txt" |
string | RAW_DATA_META_SUFF = ".meta.txt" |
string | RAW_DATA_COOKIES_SUFF = ".cookies.txt" |
string | RAW_DATA_TIDY_SUFF = ".tidy" |
string | RAW_DATA_DYNAMIC_SUFF = ".dyn" |
string | RAW_DATA_CHAIN_SUFF = ".chain" |
string | SITE_PROP_AUTO_REMOVE_RESOURCES = "AUTO_REMOVE_RESOURCES" |
string | SITE_PROP_AUTO_REMOVE_ORDER = "AUTO_REMOVE_ORDER" |
string | SITE_PROP_AUTO_REMOVE_WHERE = "AUTO_REMOVE_WHERE" |
string | SITE_PROP_AUTO_REMOVE_WHERE_ACTIVE = "AUTO_REMOVE_WHERE_ACTIVE" |
string | SITE_PROP_RECRAWL_DELETE_WHERE = "RECRAWL_DELETE_WHERE" |
string | SITE_PROP_SAVE_COOKIES = "STORE_COOKIES" |
string | DRCE_REQUEST_ROUTING_ROUND_ROBIN = '{"role":1}' |
string | DRCE_REQUEST_ROUTING_RESOURCE_USAGE = '{"role":5}' |
string | DRCE_REQUEST_ROUTING_MULTICAST = '{"role":0}' |
string | DRCE_REQUEST_ROUTING_RND = '{"role":4}' |
HCE project, Python bindings, Distributed Crawler application. Application level constants and enumerations. @package: dc @author bgv bgv.hce@gmail.com @link: http://hierarchical-cluster-engine.com/ @copyright: Copyright © 2013-2014 IOIX Ukraine @license: http://hierarchical-cluster-engine.com/license/ @since: 0.1
string dc.Constants.BATCHES_AGE_COUNTER_CANCELLED_NAME = "age_batches_canceled" |
Definition at line 224 of file Constants.py.
string dc.Constants.BATCHES_AGE_COUNTER_CHECK_FAULT_NAME = "age_batches_check_fault" |
Definition at line 234 of file Constants.py.
string dc.Constants.BATCHES_AGE_COUNTER_DELETE_FAULT_NAME = "age_batches_delete_fault" |
Definition at line 232 of file Constants.py.
string dc.Constants.BATCHES_AGE_COUNTER_ERROR_NAME = "age_batches_error" |
Definition at line 228 of file Constants.py.
string dc.Constants.BATCHES_AGE_COUNTER_FAULT_NAME = "age_batches_fault" |
Definition at line 230 of file Constants.py.
string dc.Constants.BATCHES_AGE_COUNTER_NAME = "age_batches" |
Definition at line 222 of file Constants.py.
string dc.Constants.BATCHES_AGE_COUNTER_TOTAL_NAME = "age_batches_total" |
Definition at line 226 of file Constants.py.
string dc.Constants.BATCHES_CRAWL_COUNTER_CHECK_FAULT_NAME = "batches_crawl_check_fault" |
Definition at line 139 of file Constants.py.
string dc.Constants.BATCHES_CRAWL_COUNTER_DELETE_FAULT_NAME = "batches_crawl_delete_fault" |
Definition at line 135 of file Constants.py.
string dc.Constants.BATCHES_CRAWL_COUNTER_FAULT_NAME = "batches_crawl_fault" |
Definition at line 123 of file Constants.py.
string dc.Constants.BATCHES_CRAWL_COUNTER_FAULT_TTL_NAME = "batches_crawl_fault_ttl" |
Definition at line 137 of file Constants.py.
string dc.Constants.BATCHES_CRAWL_COUNTER_FETCHER_DYNAMIC = "batches_crawl_fetcher_dynamic" |
Definition at line 161 of file Constants.py.
string dc.Constants.BATCHES_CRAWL_COUNTER_FETCHER_MIXED = "batches_crawl_fetcher_mixed" |
Definition at line 165 of file Constants.py.
string dc.Constants.BATCHES_CRAWL_COUNTER_FETCHER_STATIC = "batches_crawl_fetcher_static" |
Definition at line 163 of file Constants.py.
string dc.Constants.BATCHES_CRAWL_COUNTER_FILLED_NAME = "batches_crawl_filled" |
Definition at line 125 of file Constants.py.
string dc.Constants.BATCHES_CRAWL_COUNTER_ITEMS_AVG_NAME = "batches_crawl_items_avg" |
Definition at line 159 of file Constants.py.
string dc.Constants.BATCHES_CRAWL_COUNTER_QUEUE_NAME = "batches_crawl_queue" |
Definition at line 121 of file Constants.py.
string dc.Constants.BATCHES_CRAWL_COUNTER_TIME_AVG_NAME = "batches_crawl_time_avg" |
Definition at line 153 of file Constants.py.
string dc.Constants.BATCHES_CRAWL_COUNTER_TOTAL_NAME = "batches_crawl_total" |
Definition at line 119 of file Constants.py.
string dc.Constants.BATCHES_CRAWL_COUNTER_URL_FETCH_CANCELLED_NAME = "batches_crawl_url_fetch_cancelled" |
Definition at line 133 of file Constants.py.
string dc.Constants.BATCHES_CRAWL_COUNTER_URL_FETCH_INCR_NAME = "batches_crawl_url_fetch_incr" |
Definition at line 143 of file Constants.py.
string dc.Constants.BATCHES_CRAWL_COUNTER_URL_FETCH_NAME = "batches_crawl_url_fetch" |
Definition at line 131 of file Constants.py.
string dc.Constants.BATCHES_CRAWL_COUNTER_URL_FETCH_REQUESTS_NAME = "batches_crawl_url_fetch_requests" |
Definition at line 169 of file Constants.py.
string dc.Constants.BATCHES_CRAWL_COUNTER_URLS_FAULT_NAME = "batches_crawl_urls_fault" |
Definition at line 129 of file Constants.py.
string dc.Constants.BATCHES_CRAWL_COUNTER_URLS_NAME = "batches_crawl_urls" |
Definition at line 127 of file Constants.py.
string dc.Constants.BATCHES_CRAWL_COUNTER_URLS_RET_NAME = "batches_crawl_urls_ret" |
Definition at line 141 of file Constants.py.
string dc.Constants.BATCHES_PROCESS_COUNTER_CANCELLED_NAME = "batches_process_cancelled" |
Definition at line 219 of file Constants.py.
string dc.Constants.BATCHES_PROCESS_COUNTER_CHECK_FAULT_NAME = "batches_process_check_fault" |
Definition at line 215 of file Constants.py.
string dc.Constants.BATCHES_PROCESS_COUNTER_DELETE_FAULT_NAME = "batches_process_delete_fault" |
Definition at line 213 of file Constants.py.
string dc.Constants.BATCHES_PROCESS_COUNTER_FAULT_NAME = "batches_process_fault" |
Definition at line 205 of file Constants.py.
string dc.Constants.BATCHES_PROCESS_COUNTER_FAULT_TTL_NAME = "batches_process_fault_ttl" |
Definition at line 217 of file Constants.py.
string dc.Constants.BATCHES_PROCESS_COUNTER_FILLED_NAME = "batches_process_filled" |
Definition at line 207 of file Constants.py.
string dc.Constants.BATCHES_PROCESS_COUNTER_QUEUE_NAME = "batches_process_queue" |
Definition at line 203 of file Constants.py.
string dc.Constants.BATCHES_PROCESS_COUNTER_TOTAL_NAME = "batches_process_total" |
Definition at line 201 of file Constants.py.
string dc.Constants.BATCHES_PROCESS_COUNTER_URLS_FAULT_NAME = "batches_process_urls_fault" |
Definition at line 211 of file Constants.py.
string dc.Constants.BATCHES_PROCESS_COUNTER_URLS_NAME = "batches_process_urls" |
Definition at line 209 of file Constants.py.
string dc.Constants.BATCHES_PURGE_COUNTER_CANCELLED_NAME = "purge_batches_canceled" |
Definition at line 188 of file Constants.py.
string dc.Constants.BATCHES_PURGE_COUNTER_CHECK_FAULT_NAME = "purge_batches_check_fault" |
Definition at line 198 of file Constants.py.
string dc.Constants.BATCHES_PURGE_COUNTER_DELETE_FAULT_NAME = "purge_batches_delete_fault" |
Definition at line 196 of file Constants.py.
string dc.Constants.BATCHES_PURGE_COUNTER_ERROR_NAME = "purge_batches_error" |
Definition at line 192 of file Constants.py.
string dc.Constants.BATCHES_PURGE_COUNTER_FAULT_NAME = "purge_batches_fault" |
Definition at line 194 of file Constants.py.
string dc.Constants.BATCHES_PURGE_COUNTER_NAME = "purge_batches" |
Definition at line 186 of file Constants.py.
string dc.Constants.BATCHES_PURGE_COUNTER_TOTAL_NAME = "purge_batches_total" |
Definition at line 190 of file Constants.py.
string dc.Constants.BATCHES_REALTIME_THREADS_CREATED_COUNTER_NAME = "batches_realtime_threads_created" |
Definition at line 157 of file Constants.py.
string dc.Constants.BATCHES_REALTIME_THREADS_NAME = "batches_realtime_threads" |
Definition at line 155 of file Constants.py.
string dc.Constants.COMMON_OPERATIONS_COUNTER_NAME = "common_operations_cnt" |
Definition at line 181 of file Constants.py.
string dc.Constants.COMMON_THREADS_COUNTER_QUEUE_NAME = "common_threads" |
Definition at line 179 of file Constants.py.
string dc.Constants.COMMON_THREADS_CREATED_COUNTER_NAME = "common_threads_created" |
Definition at line 183 of file Constants.py.
string dc.Constants.DRCE_REQUEST_ROUTING_MULTICAST = '{"role":0}' |
Definition at line 265 of file Constants.py.
string dc.Constants.DRCE_REQUEST_ROUTING_RESOURCE_USAGE = '{"role":5}' |
Definition at line 264 of file Constants.py.
string dc.Constants.DRCE_REQUEST_ROUTING_RND = '{"role":4}' |
Definition at line 266 of file Constants.py.
string dc.Constants.DRCE_REQUEST_ROUTING_ROUND_ROBIN = '{"role":1}' |
Definition at line 263 of file Constants.py.
dc.Constants.DRCESyncTasksCover = namedtuple('DRCESyncTasksCover', 'eventType eventObject') |
Definition at line 112 of file Constants.py.
string dc.Constants.INCR_MAX_DEPTH_CONFIG_VAR_NAME = "INCR_MAX_DEPTH" |
Definition at line 238 of file Constants.py.
string dc.Constants.INCR_MAX_URLS_CONFIG_VAR_NAME = "INCR_MAX_URLS" |
Definition at line 239 of file Constants.py.
string dc.Constants.INCR_MIN_FREQ_CONFIG_VAR_NAME = "INCR_MIN_FREQ" |
Definition at line 237 of file Constants.py.
dc.Constants.LOGGER_NAME = APP_CONSTS.LOGGER_NAME |
Definition at line 117 of file Constants.py.
string dc.Constants.MERGE_PARAM_NAME = "MERGE_RESULTS" |
Definition at line 242 of file Constants.py.
string dc.Constants.RAW_DATA_CHAIN_SUFF = ".chain" |
Definition at line 252 of file Constants.py.
string dc.Constants.RAW_DATA_COOKIES_SUFF = ".cookies.txt" |
Definition at line 249 of file Constants.py.
string dc.Constants.RAW_DATA_DYNAMIC_SUFF = ".dyn" |
Definition at line 251 of file Constants.py.
string dc.Constants.RAW_DATA_HEADERS_SUFF = ".headers.txt" |
Definition at line 246 of file Constants.py.
string dc.Constants.RAW_DATA_META_SUFF = ".meta.txt" |
Definition at line 248 of file Constants.py.
string dc.Constants.RAW_DATA_REQESTS_SUFF = ".requests.txt" |
Definition at line 247 of file Constants.py.
string dc.Constants.RAW_DATA_SUFF = ".bin" |
Definition at line 245 of file Constants.py.
string dc.Constants.RAW_DATA_TIDY_SUFF = ".tidy" |
Definition at line 250 of file Constants.py.
string dc.Constants.RECRAWL_SITES_QUEUE_NAME = "recrawl_sites_queue" |
Definition at line 174 of file Constants.py.
string dc.Constants.RECRAWL_THREADS_COUNTER_QUEUE_NAME = "recrawl_threads" |
Definition at line 172 of file Constants.py.
string dc.Constants.RECRAWL_THREADS_CREATED_COUNTER_NAME = "recrawl_threads_created" |
Definition at line 176 of file Constants.py.
string dc.Constants.SITE_PROP_AUTO_REMOVE_ORDER = "AUTO_REMOVE_ORDER" |
Definition at line 256 of file Constants.py.
string dc.Constants.SITE_PROP_AUTO_REMOVE_RESOURCES = "AUTO_REMOVE_RESOURCES" |
Definition at line 255 of file Constants.py.
string dc.Constants.SITE_PROP_AUTO_REMOVE_WHERE = "AUTO_REMOVE_WHERE" |
Definition at line 257 of file Constants.py.
string dc.Constants.SITE_PROP_AUTO_REMOVE_WHERE_ACTIVE = "AUTO_REMOVE_WHERE_ACTIVE" |
Definition at line 258 of file Constants.py.
string dc.Constants.SITE_PROP_RECRAWL_DELETE_WHERE = "RECRAWL_DELETE_WHERE" |
Definition at line 259 of file Constants.py.
string dc.Constants.SITE_PROP_SAVE_COOKIES = "STORE_COOKIES" |
Definition at line 261 of file Constants.py.
string dc.Constants.SITES_DRCE_COUNTER_NAME = "sites_recrawl_drce_cnt" |
Definition at line 151 of file Constants.py.
string dc.Constants.SITES_RECRAWL_COUNTER_NAME = "sites_recrawl_cnt" |
Definition at line 145 of file Constants.py.
string dc.Constants.SITES_RECRAWL_DELETED_COUNTER_NAME = "sites_recrawl_deleted_cnt" |
Definition at line 149 of file Constants.py.
string dc.Constants.SITES_RECRAWL_UPDATED_COUNTER_NAME = "sites_recrawl_updated_cnt" |
Definition at line 147 of file Constants.py.