app.HostRequestStorage.HostRequestStorage Class Reference

Inheritance diagram for app.HostRequestStorage.HostRequestStorage:

Collaboration diagram for app.HostRequestStorage.HostRequestStorage:

Public Member Functions
def	__init__ (self, httpFreqLimits)
def	checkHost (self, path, url, siteId)

Public Attributes
	httpFreqLimits

Static Public Attributes
int	ITEM_PROCESS = 0
int	ITEM_BREAK = 1
string	JSON_SUFF = ".json"
string	PATH_CONFIG_KEY = "PATH_CONFIG"
string	TAIL_SLASH_LAMBDA = lambda self, path: '' if (path is None or len(path) == 0 or path[-1] == '/') else '/'
int	FREQ_AVERAGE_LIST_SIZE = 5
string	HTTP_FREQ_LIMITS_FILE_NAME_PREFIX = "http_freq_limits_"

Detailed Description

Definition at line 25 of file HostRequestStorage.py.

Constructor & Destructor Documentation

__init__()

def app.HostRequestStorage.HostRequestStorage.__init__	(		self,
			httpFreqLimits
	)

Definition at line 38 of file HostRequestStorage.py.

  def __init__(self, httpFreqLimits):
    self.httpFreqLimits = None
    try:
      self.httpFreqLimits = json.loads(httpFreqLimits)
    except Exception as excp:
      logger.debug(">>> wrong with httpFreqLimits json.loads; " + str(excp))

Member Function Documentation

checkHost()

def app.HostRequestStorage.HostRequestStorage.checkHost	(		self,
			path,
			url,
			siteId
	)

Definition at line 52 of file HostRequestStorage.py.

  def checkHost(self, path, url, siteId):
    ret = self.ITEM_PROCESS
    domain = Utils.UrlParser.getDomain(url)
    if path is None and self.httpFreqLimits is not None and self.PATH_CONFIG_KEY in self.httpFreqLimits:
      path = self.httpFreqLimits[self.PATH_CONFIG_KEY]

    if self.httpFreqLimits is not None and path is not None and domain is not None and siteId is not None:
      httpFreqLimit = None
      for elem in self.httpFreqLimits:
        if elem != self.PATH_CONFIG_KEY:
          if re.compile(elem).match(domain) is not None:
            httpFreqLimit = self.httpFreqLimits[elem]
            break
      if httpFreqLimit is not None:
        fpath = path + self.TAIL_SLASH_LAMBDA(path) + domain + '/' + \
        self.HTTP_FREQ_LIMITS_FILE_NAME_PREFIX + str(siteId) + self.JSON_SUFF
        freqJson = None
        try:
          with open(fpath, 'r') as fd:
            freqBuf = fd.read()
            freqJson = json.loads(freqBuf)
        except Exception as excp:
          logger.debug(">>> wrong in HostRequestStorage.checkHost method; " + str(excp))
#        if freqJson is not None and "requestData" in freqJson:
#          nowTime = int(time.time())
#          if (freqJson["requestData"] + httpFreqLimit["max_freq"]) > nowTime:
#            delayTime = (freqJson["requestData"] + httpFreqLimit["max_freq"]) - nowTime
#            if delayTime > httpFreqLimit["max_delay"]:
#              ret = self.ITEM_BREAK
#            else:
#              if "randomized" in httpFreqLimit and int(httpFreqLimit["randomized"]) > 0:
#                delayTime += random.randint(1, int(httpFreqLimit["max_delay"]))
#                logger.debug(">>> Use 'randomized' for delay")
#
#              logger.debug(">>> start delay = " + str(delayTime))
#              time.sleep(delayTime)
#              logger.debug(">>> finish delay = " + str(delayTime))
#          freqJson["requestData"] = nowTime
#        else:
#          freqJson = {"requestData": int(time.time())}
        if freqJson is not None and "start" in freqJson and "count" in freqJson:
          nowTime = int(time.time())
          start = int(freqJson["start"])
          count = int(freqJson["count"])

          freqAverage = 0.0
          delayTime = 0
          delta = int(nowTime - start)
          if delta > 0:
            freqAverage = float(count) / delta
          if count > 0:
            delayTime = delta / count

          logger.debug('freqAverage = ' + str(freqAverage) + ' compare ' + str(float(httpFreqLimit["max_freq"])))
          logger.debug('delayTime = ' + str(delayTime) + ' compare ' + str(int(httpFreqLimit["max_delay"])))
          logger.debug('count = ' + str(count))

          if float(freqAverage) > float(httpFreqLimit["max_freq"]):
            logger.debug("Checking 'max_freq' passed: " + str(freqAverage) + " > " + \
                         str(float(httpFreqLimit["max_freq"])))
            ret = self.ITEM_BREAK
          # elif delayTime > 2 * int(httpFreqLimit["max_delay"]):
          #  logger.debug("Checking 'max_delay' passed: " + str(delayTime) + " > " + \
          #                 str(float(httpFreqLimit["max_delay"])))
          #  ret = self.ITEM_BREAK
          else:
            if delayTime <= int(httpFreqLimit["max_delay"]):
              delayTime = int(httpFreqLimit["max_delay"]) - delayTime

            if "randomized" in httpFreqLimit and int(httpFreqLimit["randomized"]) > 0:
              delayTime += random.randint(1, int(httpFreqLimit["max_delay"]))
              logger.debug(">>> Use 'randomized' for delay")

            if delayTime > 0 and delayTime <= int(httpFreqLimit["max_delay"]):
              logger.debug(">>> start max_delay = " + str(delayTime))
              time.sleep(delayTime)
              logger.debug(">>> finish max_delay = " + str(delayTime))

          freqJson["count"] = count + 1
        else:
          freqJson = {"start": int(time.time()), "count":0}


        if os.path.exists(path):
          if not os.path.exists(path + self.TAIL_SLASH_LAMBDA(path) + domain):
            try:
              os.mkdir(path + self.TAIL_SLASH_LAMBDA(path) + domain)
            except Exception as excp:
              logger.debug(">>> makedir exception " + str(excp))
          with open(fpath, 'w') as fd:
            freqBuf = json.dumps(freqJson)
            fd.write(freqBuf)
        else:
          logger.debug(">>> path not exists !!! path: " + path)
    return ret

Member Data Documentation

FREQ_AVERAGE_LIST_SIZE

int app.HostRequestStorage.HostRequestStorage.FREQ_AVERAGE_LIST_SIZE = 5

static

Definition at line 32 of file HostRequestStorage.py.

HTTP_FREQ_LIMITS_FILE_NAME_PREFIX

string app.HostRequestStorage.HostRequestStorage.HTTP_FREQ_LIMITS_FILE_NAME_PREFIX = "http_freq_limits_"

static

Definition at line 33 of file HostRequestStorage.py.

httpFreqLimits

app.HostRequestStorage.HostRequestStorage.httpFreqLimits

Definition at line 39 of file HostRequestStorage.py.

ITEM_BREAK

int app.HostRequestStorage.HostRequestStorage.ITEM_BREAK = 1

static

Definition at line 28 of file HostRequestStorage.py.

ITEM_PROCESS

int app.HostRequestStorage.HostRequestStorage.ITEM_PROCESS = 0

static

Definition at line 27 of file HostRequestStorage.py.

JSON_SUFF

string app.HostRequestStorage.HostRequestStorage.JSON_SUFF = ".json"

static

Definition at line 29 of file HostRequestStorage.py.

PATH_CONFIG_KEY

string app.HostRequestStorage.HostRequestStorage.PATH_CONFIG_KEY = "PATH_CONFIG"

static

Definition at line 30 of file HostRequestStorage.py.

TAIL_SLASH_LAMBDA

string app.HostRequestStorage.HostRequestStorage.TAIL_SLASH_LAMBDA = lambda self, path: '' if (path is None or len(path) == 0 or path[-1] == '/') else '/'

static

Definition at line 31 of file HostRequestStorage.py.

---

The documentation for this class was generated from the following file:

• sources/hce/app/HostRequestStorage.py