HCE Project Python language Distributed Tasks Manager Application, Distributed Crawler Application and client API bindings.  2.0.0-chaika
Hierarchical Cluster Engine Python language binding
ContentCheck.py
Go to the documentation of this file.
1 """
2 HCE project, Python bindings, Distributed Tasks Manager application.
3 Event objects definitions.
4 
5 @package: dc
6 @file ContentCheck.py
7 @author scorp <developers.hce@gmail.com>
8 @copyright: Copyright &copy; 2013-2014 IOIX Ukraine
9 @license: http://hierarchical-cluster-engine.com/license/
10 @since: 0.1
11 """
12 
13 import base64
14 import json
15 import collections
16 from app.Metrics import Metrics
17 import app.Utils as Utils # pylint: disable=F0401
18 
19 TmpObj = collections.namedtuple('TmpObj', 'metrics')
20 
22 
23 
24 # #The ContentCheck class, used for content related metrics, used in some modules like RTCFinalizer
25 #
26 class ContentCheck(object):
27 
28  CHECK_TYPE_SIMPLE = 0
29  CHECK_TYPE_OR = 1
30  CHECK_TYPE_AND = 2
31 
32 
33  # # class constructor
34  #
35  def __init__(self):
36  pass
37 
38 
39  # # lookMetricsinContent static method, which finds metric structure in incoming processed content
40  # (field of urlPutObj)
41  #
42  # @param urlPutObj - incoming urlPutObj
43  # @return - bool value, is metric struct available or isn't
44  @staticmethod
45  def lookMetricsinContent(urlPutObj):
46  ret = False
47  try:
48  dataBuf = base64.b64decode(urlPutObj.putDict["data"])
49  if dataBuf is not None:
50  dataElem = json.loads(dataBuf)
51  if dataElem is not None and len(dataElem) > 0 and "metrics" in dataElem[0]:
52  metricsBuf = dataElem[0]["metrics"]
53  metrics = json.loads(metricsBuf)
54  if metrics is not None and isinstance(metrics, dict):
55  ret = True
56  except Exception as excp:
57  logger.debug(">>> Wrong content checking=" + str(excp))
58  return ret
59 
60 
61  # # checkUrlObj method check tagsCount, as urlObj field, with static value
62  #
63  # @param urlObj - incoming urlObj onject
64  # @param checkType - check's type
65  # @return - bool value, result of check
66  def checkUrlObj(self, urlObj, checkType=CHECK_TYPE_SIMPLE):
67  ret = True
68  if checkType == self.CHECK_TYPE_SIMPLE:
69  if urlObj.tagsCount <= 3:
70  ret = False
71  return ret
72 
73 
74  # # checkUrlPutObj method check content by incoming metrics
75  #
76  # @param urlPutObj - incoming urlPutObj onject
77  # @param checkType - check's type
78  # @param metricProperty - incoming json with metric's limits for checking
79  # @return - bool value, result of check
80  def checkUrlPutObj(self, urlPutObj, checkType=CHECK_TYPE_SIMPLE, metricProperty=None):
81  ret = True
82  useMetricProperty = False
83  try:
84  dataBuf = base64.b64decode(urlPutObj.putDict["data"])
85  if dataBuf is not None:
86  dataElem = json.loads(dataBuf)
87  if dataElem is not None:
88  Metrics.fillMetricModulesList()
89  metricsBuf = dataElem[0]["metrics"]
90  metrics = json.loads(metricsBuf)
91  if metrics is not None and isinstance(metrics, dict):
92  localList = []
93  localList.append(TmpObj(metrics=metrics))
94  resObjs = []
95  if metricProperty is not None:
96  try:
97  localMetricProperty = json.loads(metricProperty)
98  for elem in localMetricProperty["contentMetrics"]:
99  localResObjs = Metrics.selectElementsByMetric(localList, elem["NAME"], elem["LIMIT_MAX"],
100  elem["LIMIT_MIN"])
101  if localMetricProperty["type"] == self.CHECK_TYPE_SIMPLE:
102  resObjs = localResObjs
103  break
104  elif localMetricProperty["type"] == self.CHECK_TYPE_OR:
105  resObjs += localResObjs
106  elif localMetricProperty["type"] == self.CHECK_TYPE_AND:
107  if len(localResObjs) == 0:
108  resObjs = localResObjs
109  break
110  useMetricProperty = True
111  except Exception as excp:
112  logger.debug(">>> Wrong metric property = " + str(excp))
113  if not useMetricProperty:
114  if checkType == self.CHECK_TYPE_SIMPLE:
115  resObjs = Metrics.selectElementsByMetric(localList, "TAGS_NUMBER", None, 3)
116  if len(resObjs) == 0:
117  ret = False
118  except Exception as excp:
119  logger.debug(">>> ContentCheck.checkUrlPutObj something wrong, err=" + str(excp))
120  return ret
def checkUrlPutObj(self, urlPutObj, checkType=CHECK_TYPE_SIMPLE, metricProperty=None)
Definition: ContentCheck.py:80
def lookMetricsinContent(urlPutObj)
Definition: ContentCheck.py:45
def checkUrlObj(self, urlObj, checkType=CHECK_TYPE_SIMPLE)
Definition: ContentCheck.py:66