HCE Project Python language Distributed Tasks Manager Application, Distributed Crawler Application and client API bindings.  2.0.0-chaika
Hierarchical Cluster Engine Python language binding
All Classes Namespaces Files Functions Variables Pages
Class Index
_ | a | b | c | d | e | f | g | h | i | j | l | m | n | o | p | r | s | t | u | w
  _  
DBProxyWrapper (dc_crawler.DBProxyWrapper)   DTMD.Meta (dtm.DTMD)   Result (dc_processor.scraper_result)   TestClientInterfaceService (tests.test_dtm_ClientInterfaceService)   
DBTask (dc_db.DBTask)   DCD.Meta (dc.DCD)   RobotExclusionRulesParser (dc_crawler.OwnRobots)   TestClientInterfaceServiceDC (ftests.ftest_ClientInterfaceServiceDC)   
_Ruleset (dc_crawler.OwnRobots)   DBTasksWrapper (dc_crawler.DBTasksWrapper)   RTCPreprocessor.Meta (dc_crawler.RTCPreprocessor)   RobotFileParserLookalike (dc_crawler.OwnRobots)   TestCommand (tests.test_admin_Command)   
  a  
DCC (dcc.DCC)   RTCFinalizer.Meta (dc_crawler.RTCFinalizer)   RobotsParser (dc_crawler.RobotsParser)   TestCommandConvertor (tests.test_drce_CommandConvertor)   
DCCObjectsSerializator (dcc.DCCObjectsSerializator)   DTMA.Meta (dtma.DTMA)   RTCFinalizer (dc_crawler.RTCFinalizer)   TestCommandExecutor (tests.test_drce_CommandExecutor)   
ADMIN_HANDLER_TYPES (admin.Constants)   DCD (dc.DCD)   ResponseExtractor.Meta (app.ResponseExtractor)   RTCPreprocessor (dc_crawler.RTCPreprocessor)   TestConnection (tests.test_transport_Connection)   
AdminConfigVars (dtm.EventObjects)   DeleteEEResponseData (dtm.EventObjects)   DTMC.Meta (dtmc.DTMC)   
  s  
TestConnectionBuilder (tests.test_transport_ConnectionBuilder)   
AdminInterfaceServer (app.AdminInterfaceServer)   DeleteTask (dtm.EventObjects)   ContentUpdater.Meta (app.ContentUpdater)   TestConnectionBuilderLight (tests.test_transport_ConnectionBuilderLight)   
AdminStatData (dtm.EventObjects)   DeleteTaskData (dtm.EventObjects)   DBTask.Meta (dc_db.DBTask)   ScheduledTask (dtm.EventObjects)   TestConnectionLight (tests.test_transport_ConnectionLight)   
AdminState (dtm.EventObjects)   DeleteTaskResults (dtm.EventObjects)   MyController.Meta (ftests.ftest_cement_app)   Scheduler (dtm.Scheduler)   TestDCC (ftests.ftest_DCC)   
AdminSuspend (dtm.EventObjects)   DemoBackLogTask (demoTask)   MyApp.Meta (ftests.ftest_cement_app)   SchedulerTask (dtm.SchedulerTask)   TestDRCEManager (ftests.ftest_DRCEManager)   
AdminTimeoutException (admin.AdminExceptions)   DemoResourcesManager (ftests_dbi_insert_on_update)   UrlFetchToJsonDBTaskConvertor.Meta (app.UrlFetchJsonToDBTaskConvertor)   SchedulerTaskScheme (dtm.SchedulerTaskScheme)   TestDRCEManager (tests.test_drce_DRCEManager)   
AdminWrongConnectionKey (admin.AdminExceptions)   DemoResourcesManager (resources_manager_dbi_demo)   TestApplication.Meta (ftests.ftest_PostProcessingApplicationClass)   Scraper (dc_processor.Scraper)   TestDTMA (ftests.ftest_DTMA)   
AlchemyAPI (dc_processor.alchemyapi)   DemoTaskManager (dbi_demo)   JsonChecker.Meta (jchecker.JsonChecker)   ScraperCustomJson (dc_processor.ScraperCustomJson)   TestDTMC (ftests.ftest_DTMC)   
AlchemyExtractor (dc_processor.alchemy_extractor)   DeserilizeException (app.Exceptions)   UrlsToBatchTask.Meta (app.UrlsToBatchTask)   ScraperInData (dc_processor.ScraperInData)   TestIDGenerator (tests.test_transport_IDGenerator)   
AttrDeleteTask (dc_db.AttrDeleteTask)   DetectModified (dc_crawler.DetectModified)   PostProcessingApplicationClass.Meta (dc_postprocessor.PostProcessingApplicationClass)   ScraperLangDetector (dc_processor.ScraperLangDetector)   TestNode (tests.test_admin_Node)   
AttrFetchTask (dc_db.AttrFetchTask)   HTTPCookieResolver.DomainProperty (dc_crawler.HTTPCookieResolver)   PostprocessorTask.Meta (dc_postprocessor.PostprocessorTask)   ScraperMultiItemsTask (dc_processor.ScraperMultiItemsTask)   testNodeManager (tests.test_admin_NodeManager)   
Attribute (dc.EventObjects)   DRCE_FIELDS (dtm.Constants)   ProcessorFeedParser.Meta (dc_processor.ProcessorFeedParser)   ScraperResponse (dc_processor.ScraperResponse)   TestNodeManagerRequest (tests.test_admin_NodeManagerRequest)   
AttributeDelete (dc.EventObjects)   DRCECover (drce.Commands)   ProcessorStoreContentKVDB.Meta (dc_processor.ProcessorStoreContentKVDB)   ScraperResultDocuments (dc_processor.ScraperMultiItemsTask)   TestObj (dbi_sql_test)   
AttributeFetch (dc.EventObjects)   DRCEManager (drce.DRCEManager)   ProcessorTask.Meta (dc_processor.ProcessorTask)   ScrapyExtractor (dc_processor.scrapy_extractor)   TestPollerManager (tests.test_app_PollerManager)   
AttributeUpdate (dc.EventObjects)   DTMA (dtma.DTMA)   Scraper.Meta (dc_processor.Scraper)   SelectorWrapper (app.SelectorWrapper)   TestRequest (tests.test_transport_Request)   
AttrSetTask (dc_db.AttrSetTask)   DTMAEmptyClasses (dtma.DTMAExceptions)   Meta (dc_processor.ScraperCustomJson)   SeleniumFetcher (dc_crawler.Fetcher)   TestRequest (ftests.ftest_test_async_MsgSend)   
AttrUpdateTask (dc_db.AttrUpdateTask)   DTMAEmptyFields (dtma.DTMAExceptions)   CrawlingOptimiser.Meta (dc_co.CrawlingOptimiser)   SeleniumFetcherException (app.Exceptions)   TestResoursesManager (ftests.ftest_ResourcesManager)   
AuthorType (dc_processor.AuthorType)   DTMANameValueException (dtma.DTMAExceptions)   Meta (dc_processor.ScraperMultiItemsTask)   Server (ftests.ftest_server_connection)   TestResponse (tests.test_transport_Response)   
AvailableTaskIds (dtm.EventObjects)   DTMAObjectsFiller (dtma.DTMAObjectsFiller)   DCC.Meta (dcc.DCC)   Server (ftests.ftest_test_async_MsgSend)   TestScheduler (tests.test_dtm_Scheduler)   
  b  
DTMC (dtmc.DTMC)   MetricContentSize (algorithms.MetricContentSize)   ServerConnectionBuilder (transport.ServerConnectionBuilder)   TestSchedulerTaskScheme (tests.test_dtm_SchedulerTaskScheme)   
DTMCObjectsSerializator (dtmc.DTMCObjectsSerializator)   Metrics (app.Metrics)   Session (drce.Commands)   TestScraperLangDetector (ftests.ftest_ScraperLangDetector)   
BaseExtractor (dc_processor.base_extractor)   DTMD (dtm.DTMD)   MetricTagsCount (algorithms.MetricTagsCount)   SimpleCharsetDetector (dc_crawler.Fetcher)   TestServerConnectionBuilder (tests.test_transport_ServerConnectionBuilder)   
BaseFetcher (dc_crawler.Fetcher)   
  e  
MetricWCount (algorithms.MetricWCount)   SimpleCharsetDetector (ftests.ftest_SimpleCharsetDetector)   TestSession (tests.test_drce_Commands)   
BaseMetric (algorithms.BaseMetric)   MLExtractor (dc_processor.ml_extractor)   Singleton (transport.Singelon)   TestTaskCheckRequest (tests.test_drce_Commands)   
BaseRequest (drce.Commands)   EEResponseData (dtm.EventObjects)   MLStripper (app.Utils)   Site (dc.EventObjects)   TestTaskCheckRequest (tests.test_drce_CommandConvertor)   
BaseServerManager (app.BaseServerManager)   EEResponsesTable (dtm.EEResponsesTable)   MPLogger (app.Utils)   SiteCleanup (dc.EventObjects)   TestTaskDataManager (ftests.ftest_TaskDataManager)   
BaseTask (dc_db.BaseTask)   Event (transport.Event)   MyApp (ftests.ftest_cement_app)   SiteCleanUpTask (dc_db.SiteCleanUpTask)   TestTaskExecuteRequest (tests.test_drce_CommandConvertor)   
Batch (dc.EventObjects)   EVENT_TYPES (dtm.Constants)   MyController (ftests.ftest_cement_app)   SiteDelete (dc.EventObjects)   TestTaskExecuteRequest (tests.test_drce_Commands)   
BatchItem (dc.EventObjects)   EVENT_TYPES (dc.Constants)   
  n  
SiteDeleteTask (dc_db.SiteDeleteTask)   TestTaskExecuteStruct (tests.test_drce_CommandConvertor)   
BatchTasksManager (dc.BatchTasksManager)   EventBuilder (transport.Event)   SiteFilter (dc.EventObjects)   TestTaskExecuteStruct (tests.test_drce_Commands)   
BatchTasksManagerProcess (dc.BatchTasksManagerProcess)   ExceptionLog (app.Utils)   NewspaperExtractor (dc_processor.newspaper_extractor)   SiteFind (dc.EventObjects)   TestTaskGetDataRequest (tests.test_drce_Commands)   
BatchTasksManagerRealTime (dc.BatchTasksManagerRealTime)   ExecuteTask (dtm.EventObjects)   NewspaperWrapper (dc_processor.NewspaperWrapper)   SiteFindTask (dc_db.SiteFindTask)   TestTaskLogScheme (tests.test_dtm_TaskLogScheme)   
bcolors (ftests.ftest_UrlNormalization)   ExecutionEnvironmentManager (dtm.ExecutionEnvironmentManager)   NewTask (dtm.EventObjects)   SitesManager (dc.SitesManager)   TestTasksDataManager (ftests.ftest_TasksDataManager)   
BoilerpipeExtractor (dc_processor.boilerpipe_extractor)   ExtendInnerText (app.ExtendInnerText)   Node (admin.Node)   SiteStatus (dc.EventObjects)   TestTasksManager (tests.test_dtm_TasksManager)   
  c  
  f  
NodeManager (admin.NodeManager)   SiteStatusTask (dc_db.SiteStatusTask)   TestTasksStateUpdateService (tests.test_dtm_TasksStateUpdateService)   
NodeManagerRequest (admin.NodeManagerRequest)   SiteTask (dc_db.SiteTask)   TestTaskTerminateRequest (tests.test_drce_Commands)   
CheckTaskState (dtm.EventObjects)   FakeDBWrapper (ftests.ftest_ProxyResolver)   NodeManagerResponse (admin.NodeManagerResponse)   SiteUpdate (dc.EventObjects)   TestTaskTerminateRequest (tests.test_drce_CommandConvertor)   
Client (ftests.ftest_test_async_MsgSend)   FakeDBWrapper (ftests.ftest_SiteProxy)   NotModifiedException (dc_crawler.DetectModified)   SiteUpdateTask (dc_db.SiteUpdateTask)   TransportInternalErr (transport.Connection)   
ClientInterfaceService (dtm.ClientInterfaceService)   FetchAvailabelTaskIds (dtm.EventObjects)   
  o  
SiteURL (dc.EventObjects)   
  u  
ClientInterfaceService (dc.ClientInterfaceService)   FetchEEResponseData (dtm.EventObjects)   SocialModule (dc_postprocessor.SocialModule)   
ClientResponse (dc.EventObjects)   FetcherType (dc_crawler.FetcherType)   OffsetTzInfo (app.DateTimeType)   SomeClass (ftests.ftest_profiler)   UIDGenerator (transport.UIDGenerator)   
ClientResponseItem (dc.EventObjects)   FetchTaskData (dtm.EventObjects)   OutputFile (ftests.ftest_UrlNormalization)   SomeClass (ftests.ftest_admin_command_shutdown)   UpdateScheduledTasks (dtm.EventObjects)   
CollectProperties (dc_crawler.CollectProperties)   FetchTasksResults (dtm.EventObjects)   
  p  
SourceTemplateExtractor (dc_processor.SourceTemplateExtractor)   UpdateTask (dtm.EventObjects)   
CollectURLs (dc_crawler.CollectURLs)   FetchTasksResultsFromCache (dtm.EventObjects)   SQLCustomTask (dc_db.SQLCustomTask)   UpdateTaskFields (dtm.EventObjects)   
Command (admin.Command)   FieldRecalculator (dc_db.FieldRecalculator)   PathMaker (app.Utils)   SQLExpression (app.Utils)   URL (dc.EventObjects)   
COMMAND_NAMES (admin.Constants)   FieldRecalculatorObj (dc.EventObjects)   PDateTimezonesHandler (dc_processor.PDateTimezonesHandler)   StatFreqConstants (dc_db.Constants)   Url (app.Url)   
CommandConvertor (drce.CommandConvertor)   FieldRecalculatorTask (dc_db.FieldRecalculatorTask)   PDateTimezonesProperties (dc_processor.PDateTimezonesHandler)   StatisticLogManager (dc_db.StatisticLogManager)   URLAge (dc.EventObjects)   
CommandConvertor_old (drce.CommandConvertor)   FieldsSQLExpressionEvaluator (app.FieldsSQLExpressionEvaluator)   PollerManager (app.PollerManager)   SyncronizeException (dc_crawler.Exceptions)   URLAgeTask (dc_db.URLAgeTask)   
CommandConvertorError (drce.CommandConvertor)   Filters (app.Filters)   PostProcessingApplicationClass (dc_postprocessor.PostProcessingApplicationClass)   System (dtm.EventObjects)   UrlAvailableException (app.Exceptions)   
CommandExecutor (drce.CommandExecutor)   
  g  
PostProcessingModuleClass (dc_postprocessor.PostProcessingModuleClass)   SystemCommandHandler (app.SystemCommandHandler)   URLCleanup (dc.EventObjects)   
CommandExecutorErr (drce.CommandExecutor)   PostprocessorTask (dc_postprocessor.PostprocessorTask)   
  t  
URLCleanUpTask (dc_db.URLCleanupTask)   
ContentUpdater.ConfigOptions (app.ContentUpdater)   GeneralResponse (dtm.EventObjects)   ProcessedContentInternalStruct (dc_db.ProcessedContentInternalStruct)   URLContentRequest (dc.EventObjects)   
ConfigParamsList (app.Utils)   GetHandler (simple_server_test)   ProcessorException (dc_processor.ProcessorException)   Task (dtm.EventObjects)   URLContentResponse (dc.EventObjects)   
Connection (transport.Connection)   GetScheduledTask (dtm.EventObjects)   ProcessorFeedParser (dc_processor.ProcessorFeedParser)   Task (task)   URLContentTask (dc_db.URLContentTask)   
ConnectionBuilder (transport.ConnectionBuilder)   GetScheduledTasks (dtm.EventObjects)   ProcessorStoreContentKVDB (dc_processor.ProcessorStoreContentKVDB)   TaskBackLogScheme (dtm.TaskBackLogScheme)   URLDelete (dc.EventObjects)   
ConnectionBuilderLight (transport.ConnectionBuilderLight)   GetScheduledTasksResponse (dtm.EventObjects)   ProcessorTask (dc_processor.ProcessorTask)   TaskCheckRequest (drce.Commands)   URLDeleteTask (dc_db.URLDeleteTask)   
ConnectionLight (transport.ConnectionLight)   GetTaskManagerFields (dtm.EventObjects)   Profiler (app.Profiler)   TaskDeleteRequest (drce.Commands)   URLFetch (dc.EventObjects)   
ConnectionManager (drce.ConnectionManager)   GetTasksStatus (dtm.EventObjects)   PropertiesValidator (app.Utils)   TaskExecuteRequest (drce.Commands)   URLFetchTask (dc_db.URLFetchTask)   
ConnectionStub (ftests.ftest_DCC)   GooseExtractor (dc_processor.goose_extractor)   Proxy (dc.EventObjects)   TaskExecuteRequestEncoder (drce.CommandConvertor)   UrlFetchToJsonDBTaskConvertor (app.UrlFetchJsonToDBTaskConvertor)   
ConnectionStub (ftests.ftest_DTMA)   
  h  
ProxyDelete (dc.EventObjects)   TaskExecuteStruct (drce.Commands)   URLHistoryRequest (dc.EventObjects)   
ConnectionStub (ftests.ftest_DTMC)   ProxyDeleteTask (dc_db.ProxyDeleteTask)   TaskExecuteStructEncoder (drce.CommandConvertor)   URLHistoryResponse (dc.EventObjects)   
ConnectionTimeout (transport.Connection)   HostRequestStorage (app.HostRequestStorage)   ProxyException (app.Exceptions)   TaskGetDataRequest (drce.Commands)   URLHistoryTask (dc_db.URLHistoryTask)   
Content (dc.EventObjects)   HTML5_SEMANTIC_TAGS (dc_processor.Constants)   ProxyFind (dc.EventObjects)   TaskLog (dtm.TaskLog)   URLLibFetcher (dc_crawler.Fetcher)   
ContentCheck (app.ContentCheck)   HTTPCookieResolver (dc_crawler.HTTPCookieResolver)   ProxyFindTask (dc_db.ProxyFindTask)   TaskLogScheme (dtm.TaskLogScheme)   URLNewTask (dc_db.URLNewTask)   
ContentEvaluator (app.ContentEvaluator)   HTTPProxyResolver (dc_crawler.HTTPProxyResolver)   ProxyJsonWrapper (dc_crawler.ProxyJsonWrapper)   TaskManagerFields (dtm.EventObjects)   UrlNormalizator (app.Utils)   
ContentFetcher (dc_crawler.Fetcher)   HTTPRedirectResolver (dc_crawler.HTTPRedirectResolver)   ProxyNewTask (dc_db.ProxyNewTask)   TaskNoPresentErr (dtm.TasksManager)   UrlNormalize (app.UrlNormalize)   
ContentHashCalculator (app.ContentHashCalculator)   
  i  
ProxyResolver (dc_crawler.ProxyResolver)   TaskResponse (drce.Commands)   UrlParseException (app.Exceptions)   
ContentUpdater (app.ContentUpdater)   ProxyStatus (dc.EventObjects)   TaskResponseDecoder (drce.CommandConvertor)   UrlParser (app.Utils)   
HTTPCookieResolver.Cookie (dc_crawler.HTTPCookieResolver)   IDGenerator (transport.IDGenerator)   ProxyStatusTask (dc_db.ProxyStatusTask)   TasksDataManager (dtm.TasksDataManager)   URLProcess (dc_crawler.URLProcess)   
CrawledResource (dc_crawler.CrawledResource)   MediaLimitsHandler.ImageLimits (dc_processor.MediaLimitsHandler)   ProxyUpdate (dc.EventObjects)   TasksDataTable (dtm.TasksDataTable)   URLPurge (dc.EventObjects)   
CrawlerException (dc_crawler.Exceptions)   MediaLimitsHandler.ImageProperty (dc_processor.MediaLimitsHandler)   ProxyUpdateTask (dc_db.ProxyUpdateTask)   TasksDataTable (ftests_db_in_memory)   URLPurgeTask (dc_db.URLPurgeTask)   
CrawlerFilterException (dc_crawler.Exceptions)   InternalCrawlerException (dc_crawler.Exceptions)   
  r  
TasksExecutor (dtm.TasksExecutor)   URLPut (dc.EventObjects)   
CrawlerTask (dc_crawler.CrawlerTask)   InterruptableThread (app.Utils)   TasksManager (dtm.TasksManager)   URLPutResponse (dc.EventObjects)   
CrawlingOptimiser (dc_co.CrawlingOptimiser)   
  j  
HTTPRedirectResolver.RedirectProperty (dc_crawler.HTTPRedirectResolver)   TasksManager (dc_db.TasksManager)   URLPutTask (dc_db.URLPutTask)   
CustomExtractor (dc_processor.custom_extractor)   RefererHeaderResolver (dc_crawler.RefererHeaderResolver)   TasksStateUpdateService (dtm.TasksStateUpdateService)   UrlSchema (dc_crawler.UrlSchema)   
CustomRequest (dbi.EventObjects)   JsonChecker (jchecker.JsonChecker)   Request (transport.Request)   TasksStatus (dtm.EventObjects)   URLStatsRequest (dc.EventObjects)   
CustomResponse (dbi.EventObjects)   JsonSerializable (app.Utils)   RequestsFetcher (dc_crawler.Fetcher)   TaskTerminateRequest (drce.Commands)   URLStatsResponse (dc.EventObjects)   
  d  
  l  
RequestsRedirectWrapper (dc_crawler.RequestsRedirectWrapper)   TCPServerRaw (ftests.ftest_tcp_server_raw)   URLStatsTask (dc_db.URLStatsTask)   
Resource (dtm.EventObjects)   TemplateExtractorXPathPreparing (dc_processor.TemplateExtractorXPathPreparing)   URLStatus (dc.EventObjects)   
DatabaseException (app.Exceptions)   LangDetectorTest (tests.test_LangDetector)   Resource (dc_processor.scraper_resource)   Test (ftests.ftest_Metrics)   URLStatusTask (dc_db.URLStatusTask)   
DataCreateRequest (dc.EventObjects)   LFSDataStorage (app.LFSDataStorage)   ResourceProcess (dc_crawler.ResourceProcess)   Test (ftests.ftest_Filters)   UrlsToBatchTask (app.UrlsToBatchTask)   
DataCreateResponse (dc.EventObjects)   Limits (drce.Commands)   Resources (Resources)   Test (ftests.ftest_DBTaskStatistic)   URLUpdate (dc.EventObjects)   
DataDeleteRequest (dc.EventObjects)   LinkResolver (dc_postprocessor.LinkResolver)   ResourcesAVG (dtm.EventObjects)   Test (ftests.ftest_DBTask)   URLUpdateTask (dc_db.URLUpdateTask)   
DataDeleteResponse (dc.EventObjects)   LogFormatterEvent (app.LogFormatter)   ResourcesManager (dtm.ResourcesManager)   Test (ftests.ftest_DBTasksWrapper)   URLVerify (dc.EventObjects)   
DataFetchRequest (dc.EventObjects)   LoggerFileName (app.Utils)   ResourcesRecalculating (dtm.ResourcesRecalculating)   Test (ftests.ftest_UrlSchema)   URLVerifyTask (dc_db.URLVerifyTask)   
DataFetchResponse (dc.EventObjects)   LogicErr (dtm.Scheduler)   ResourcesStateMonitor (dtm.ResourcesStateMonitor)   Test (ftests.ftest_FieldsSQLExpressionEvaluator)   UserClass (ftests.ftest_STAT_admin_command)   
DataReplacementConstants (app.Utils)   
  m  
ResourcesTable (dtm.ResourcesTable)   Test (ftests.ftest_SiteProxy)   UserProxyJsonWrapper (dc_crawler.UserProxyJsonWrapper)   
DateTimeType (app.DateTimeType)   Response (transport.Response)   TestAdminProcessorDataHandler (ftests.ftest_AdminProcessorDataHandler)   
  w  
DBDataTask (dc_db.DBDataTask)   Matcher (tests.test_app_BaseServerManager)   Response (dc_crawler.Fetcher)   TestApp (ftests.ftest_MPLogger)   
DBI (dbi.dbi)   Matcher (tests.test_dtm_TasksManager)   ResponseExtractor (app.ResponseExtractor)   TestApplication (ftests.ftest_PostProcessingApplicationClass)   WrongEventObjectTypeException (app.Exceptions)   
DBI (ftests_db_in_memory)   MediaLimitsHandler (dc_processor.MediaLimitsHandler)   ResponseFormatErr (transport.Response)   TestBaseServerManager (ftests.ftest_BaseServerManager)   
DBIErr (dbi.dbi)   CrawlerTask.Meta (dc_crawler.CrawlerTask)   ResponseItem (drce.Commands)   TestBaseServerManager (tests.test_app_BaseServerManager)   
_ | a | b | c | d | e | f | g | h | i | j | l | m | n | o | p | r | s | t | u | w