HCE Project Python language Distributed Tasks Manager Application, Distributed Crawler Application and client API bindings.
2.0.0-chaika
Hierarchical Cluster Engine Python language binding
MetricContentSize.py
Go to the documentation of this file.
1
# coding: utf-8 # pylint: disable-all
2
3
"""@package algorithms
4
@file MetricContentSize.py
5
@author scorp <developers.hce@gmail.com>
6
@link http://hierarchical-cluster-engine.com/
7
@copyright Copyright © 2013 IOIX Ukraine
8
@license http://hierarchical-cluster-engine.com/license/
9
@package HCE project node API
10
@since 0.1
11
"""
12
13
import
logging
14
15
import
app.Consts
as
APP_CONSTS
16
import
app.Utils
as
Utils
# pylint: disable=F0401
17
import
unicodedata
18
import
types
19
from
BaseMetric
import
BaseMetric
20
21
# Logger initialization
22
logger = Utils.MPLogger().
getLogger
()
23
24
25
# #The MetricContentSize class, class that implements metric counters for contetn size Metric
26
#
27
class
MetricContentSize
(
BaseMetric
):
28
29
CHAR_CATEGORIES_LIST = [
'Lu'
,
'Ll'
,
'Lt'
,
'Lm'
,
'Lo'
,
'Nd'
,
'Nl'
,
'No'
]
30
31
# # class constructor
32
#
33
# @param name - metric's name
34
def
__init__
(self, name):
35
super(MetricContentSize, self).
__init__
(name)
36
37
38
# # internalCalculating methods makes internal content calculating
39
#
40
# @param dataDict
41
# @param buf
42
def
internalCalculating
(self, dataDict, buf):
43
if
type
(buf)
is
types.StringType:
44
buf = unicode(buf)
45
if
type
(buf)
is
types.UnicodeType:
46
for
ch
in
buf:
47
if
unicodedata.category(ch)
in
self.
CHAR_CATEGORIES_LIST
:
48
dataDict[
"validCharsCount"
] += 1
49
dataDict[
"count"
] += 1
50
51
52
# # precalculate makes MetricTagsCount metrics precalculating
53
#
54
# @param result - param, that content calculating data in common format
55
# @return precalculated data in common format
56
def
precalculate
(self, result, metricName):
57
ret = {
"count"
: 0,
"percent"
: 0,
"validCharsCount"
: 0}
58
for
key
in
result.tags:
59
if
type
(result.tags[key])
is
types.DictType
and
"data"
in
result.tags[key]:
60
if
type
(result.tags[key][
"data"
])
in
types.StringTypes:
61
self.
internalCalculating
(ret, result.tags[key][
"data"
])
62
elif
type
(result.tags[key][
"data"
])
is
types.ListType:
63
for
buf
in
result.tags[key][
"data"
]:
64
self.
internalCalculating
(ret, buf)
65
if
ret[
"count"
] > 0:
66
ret[
"percent"
] = ret[
"validCharsCount"
] * 100 / ret[
"count"
]
67
ret = self.
retForMultiNames
(ret, metricName)
68
return
ret
ftests.ftest_ContentEvaluator.getLogger
def getLogger()
Definition:
ftest_ContentEvaluator.py:16
algorithms.BaseMetric.BaseMetric.retForMultiNames
def retForMultiNames(self, retDict, metricName)
Definition:
BaseMetric.py:39
algorithms.MetricContentSize.MetricContentSize.precalculate
def precalculate(self, result, metricName)
Definition:
MetricContentSize.py:56
algorithms.MetricContentSize.MetricContentSize.CHAR_CATEGORIES_LIST
list CHAR_CATEGORIES_LIST
Definition:
MetricContentSize.py:29
app.Utils
Definition:
Utils.py:1
algorithms.MetricContentSize.MetricContentSize.__init__
def __init__(self, name)
Definition:
MetricContentSize.py:34
algorithms.MetricContentSize.MetricContentSize
Definition:
MetricContentSize.py:27
algorithms.BaseMetric.BaseMetric
Definition:
BaseMetric.py:24
algorithms.MetricContentSize.MetricContentSize.internalCalculating
def internalCalculating(self, dataDict, buf)
Definition:
MetricContentSize.py:42
app.Consts
Definition:
Consts.py:1
integrity-urlContent.type
type
Definition:
integrity-urlContent.py:27
sources
hce
algorithms
MetricContentSize.py
Generated on Fri Nov 24 2017 18:53:54 for HCE Project Python language Distributed Tasks Manager Application, Distributed Crawler Application and client API bindings. by
1.8.13