ftest_SocialModule.py

Go to the documentation of this file.

#!/usr/bin/python
# coding: utf-8

import os
import json
import base64
import logging
import ConfigParser
from dc.EventObjects import URL
from dc.EventObjects import Batch
from dc.EventObjects import BatchItem
from dc.EventObjects import URLContentResponse
from dc_postprocessor.SocialModule import SocialModule
from dc_postprocessor.PostProcessingApplicationClass import PostProcessingApplicationClass
import app.Consts as APP_CONSTS
from app.Utils import varDump
import app.Utils as Utils

def getLogger():
  # create logger
  log = logging.getLogger(APP_CONSTS.LOGGER_NAME)
  log.setLevel(logging.DEBUG)

  # create console handler and set level to debug
  ch = logging.StreamHandler()
  ch.setLevel(logging.DEBUG)

  # create formatter
  formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')

  # add formatter to ch
  ch.setFormatter(formatter)

  # add ch to logger
  log.addHandler(ch)

  return log


def getFileLogger():

  configFileName = '../ini/postprocessor_task_log-rt.ini'

  retval = os.getcwd()
  os.chdir('..')
  # read config
  logging.config.fileConfig(configFileName)

  # create logger
  log = Utils.MPLogger().getLogger()
  # log = logging.getLogger(APP_CONSTS.LOGGER_NAME)
  os.chdir(retval)

  return log


if __name__ == '__main__':

  logger = getLogger()

  configName = '../ini/postprocessor_task.ini'
  headerFileName = '../ini/crawler-task_headers.txt'

  postProcessingApplicationClass = PostProcessingApplicationClass()
  postProcessingApplicationClass.configParser = ConfigParser.ConfigParser()
  postProcessingApplicationClass.configParser.optionxform = str
  readOk = postProcessingApplicationClass.configParser.read(configName)
  logger.debug("Read config: %s", str(readOk))

  # postProcessingApplicationClass.configParser.set('SocialModule', 'cmd', '')

  siteId = 12345
  url = 'https://www.theguardian.com/us-news/2016/jan/05/obama-gun-control-executive-action-background-checks-licenses-gun-shows-mental-health-funding'
  urlObj = URL(siteId, url)

  processedContent = [{"title":"Tearful Obama tightens gun control and tells inactive Congress: 'We can't wait'",
                      "source_url": url}]

  processedContents = [base64.b64encode(json.dumps(processedContent))]
  urlContentResponse = URLContentResponse(url=url, processedContents=processedContents)

  batchItem = BatchItem(siteId=siteId, urlId=urlObj.urlMd5, urlObj=urlObj, urlContentResponse=urlContentResponse)
#   batchItem.properties = {"SOCIAL_RATE":json.dumps({"retries":2, "retries_delay":5, "retries_type":1, "interval":10, "lang":"en", "sentiment":1, "debug":1, "timeout":400, "social_list":{"tw":["https://www.twitter.com", "window.IFRAME_QUERY_URL=\"https://twitter.com/search?f=tweets&vertical=default&q=%25QUERY_STRING%25&src=typd\",window.IFRAME_CSCROLL_COUNT=10;window.IFRAME_MAX_TIME=350;window.IFRAME_SFIELD='source_url';", {"name":"tests", "sets":[{"name":"set1", "items":["1", "%MACRO_DATA%", "http://127.0.0.1/social.js", "!5:76:return window.IFRAME_DATA_READY;", "return window.MACRO_COLLECT;"], "repeat":1, "delay":0}], "result_type":0, "result_content_type":"text/json"}]} })}
#   batchItem.properties = {"SOCIAL_RATE": "{\"retries\":2, \"retries_delay\":10, \"retries_type\":1, \"interval\":10,\"lang\":\"en\", \"sentiment\":1, \"debug\":1, \"timeout\":400, \"social_list\":{\"tw\":[\"https:\/\/www.twitter.com\/\",\"window.IFRAME_QUERY_URL=\\\"https:\/\/twitter.com\/search?f=tweets&vertical=default&q=%25QUERY_STRING%25&src=typd\\\",window.IFRAME_CSCROLL_COUNT=10;window.IFRAME_MAX_TIME=360;window.IFRAME_SFIELD='source_url';\",{\"name\":\"tests\", \"sets\":[{\"name\":\"set1\", \"items\":[\"3\", \"%MACRO_DATA%\", \"http:\/\/127.0.0.1\/social.js\", \"!5:76:return window.IFRAME_DATA_READY;\", \"return window.MACRO_COLLECT;\"], \"repeat\":1, \"delay\":0}], \"result_type\":0, \"result_content_type\":\"text\/json\"}]} }"}
  batchItem.properties = {"SOCIAL_RATE": "{\"retries\":3, \"retries_delay\":5, \"retries_type\":1, \"interval\":10,\"lang\":\"en\", \"sentiment\":1, \"debug\":1, \"timeout\":400, \"social_list\":{\"fb\":[\"https:\/\/www.facebook.com\",\"window.IFRAME_QUERY_URL=\\\"https:\/\/www.facebook.com\/search\/top\/?q=%25QUERY_STRING%25\\\";window.IFRAME_CSCROLL_COUNT=100;window.IFRAME_MAX_TIME=350;window.IFRAME_SFIELD='title';\",{\"name\":\"tests\", \"sets\":[{\"name\":\"set1\", \"items\":[\"1\", \"%MACRO_DATA%\", \"http:\/\/127.0.0.1\/social.js\", \"!5:76:return window.IFRAME_DATA_READY;\", \"return window.MACRO_COLLECT;\"], \"repeat\":1, \"delay\":0}], \"result_type\":0, \"result_content_type\":\"text\/json\"}],\"tw\":[\"https:\/\/www.twitter.com\",\"window.IFRAME_QUERY_URL=\\\"https:\/\/twitter.com\/search?f=tweets&vertical=default&q=%25QUERY_STRING%25&src=typd\\\",window.IFRAME_CSCROLL_COUNT=100;window.IFRAME_MAX_TIME=350;window.IFRAME_SFIELD='source_url';\",{\"name\":\"tests\", \"sets\":[{\"name\":\"set1\", \"items\":[\"1\", \"%MACRO_DATA%\", \"http:\/\/127.0.0.1\/social.js\", \"!5:76:return window.IFRAME_DATA_READY;\", \"return window.MACRO_COLLECT;\"], \"repeat\":1, \"delay\":0}], \"result_type\":0, \"result_content_type\":\"text\/json\"}]} }"}
  batch = Batch(77777, [batchItem])

  logger.debug("Input batch: %s", varDump(batch))
  socialModule = SocialModule(postProcessingApplicationClass.getConfigOption, postProcessingApplicationClass.logger)
  socialModule.init()
  batch = socialModule.processBatch(batch)
  logger.debug("Output batch: %s", varDump(batch))

  # logger.debug("Resolved url: %s", str(json.loads(base64.b64decode(batch.items[0].urlContentResponse.processedContents[0]))['link']))