batch_generator.py File Reference

Go to the source code of this file.

Namespaces
	batch_generator

Variables
int	batch_generator.EXIT_SUCCESS = 0
int	batch_generator.EXIT_FAILURE = 1
string	batch_generator.LOGGER_NAME = "batch_generator"
string	batch_generator.MSG_ERROR_READ_BATCH = "ERROR READ BATCH FROM STDIN"
	batch_generator.filename
	batch_generator.filemode
	batch_generator.logger = logging.getLogger(LOGGER_NAME)
int	batch_generator.error = EXIT_SUCCESS
	batch_generator.input_json = sys.stdin.read()
	batch_generator.batch_data = json.loads(input_json)
	batch_generator.site_id = batch_data["site_id"]
	batch_generator.urls = batch_data["urls"]
	batch_generator.tags = batch_data["tags"]
	batch_generator.id = batch_data["id"]
int	batch_generator.item_no = 1
list	batch_generator.batch_items = []
	batch_generator.item_url = item["url"]
	batch_generator.item_site_id = None
	batch_generator.sid = item_site_id or site_id
	batch_generator.uid = hashlib.md5(item_url).hexdigest()
	batch_generator.url_obj = dc_event.URL(sid, item_url)
	batch_generator.batch_item = dc_event.BatchItem(sid, uid, url_obj)
	batch_generator.batch_obj = dc_event.Batch(id, batch_items, dc_event.Batch.TYPE_REAL_TIME_CRAWLER)