HCE Project Python language Distributed Tasks Manager Application, Distributed Crawler Application and client API bindings.  2.0.0-chaika
Hierarchical Cluster Engine Python language binding
Public Member Functions | Static Public Member Functions | Static Public Attributes | List of all members
app.FieldsSQLExpressionEvaluator.FieldsSQLExpressionEvaluator Class Reference
Inheritance diagram for app.FieldsSQLExpressionEvaluator.FieldsSQLExpressionEvaluator:
Inheritance graph
[legend]
Collaboration diagram for app.FieldsSQLExpressionEvaluator.FieldsSQLExpressionEvaluator:
Collaboration graph
[legend]

Public Member Functions

def __init__ (self)
 

Static Public Member Functions

def execute (siteProperties, dbWrapper, siteObj, urlObj, logger, propertyName=APP_CONSTS.SQL_EXPRESSION_FIELDS_UPDATE_CRAWLER)
 
def evaluateElement (dbWrapper, logger, obj, fieldsDict, objPropertyStruct)
 
def macroReplace (pattern, values, marker, case=MACRO_CASE_ORIGINAL)
 
def evaluatePDateTime (siteProperties, dbWrapper, urlObj, logger, defaultPubdateValue=None)
 

Static Public Attributes

string OBJECT_NAME_SITE = 'Site'
 
string OBJECT_NAME_URL = 'URL'
 
int MACRO_CASE_ORIGINAL = 0
 
int MACRO_CASE_UPPER = 1
 
int MACRO_CASE_LOWER = 2
 
int CAST_TYPE_TO_INTEGER = 0
 
int CAST_TYPE_TO_STRING = 1
 
int CAST_TYPE_TO_DATETIME = 2
 
string REPLACE_MARKER = '%'
 
string DB_NAME = "dc_sites"
 

Detailed Description

Definition at line 25 of file FieldsSQLExpressionEvaluator.py.

Constructor & Destructor Documentation

◆ __init__()

def app.FieldsSQLExpressionEvaluator.FieldsSQLExpressionEvaluator.__init__ (   self)

Definition at line 43 of file FieldsSQLExpressionEvaluator.py.

43  def __init__(self):
44  pass
45 
transport.UIDGenerator.UIDGenerator.__init__
def __init__(self)
constructor
Definition: UIDGenerator.py:19

Member Function Documentation

◆ evaluateElement()

def app.FieldsSQLExpressionEvaluator.FieldsSQLExpressionEvaluator.evaluateElement (   dbWrapper,
  logger,
  obj,
  fieldsDict,
  objPropertyStruct 
)
static

Definition at line 98 of file FieldsSQLExpressionEvaluator.py.

98  def evaluateElement(dbWrapper, logger, obj, fieldsDict, objPropertyStruct):
99  logger.debug('evaluateElement enter.... \nobj: ' + varDump(obj) + '\nfieldsDict: ' + str(fieldsDict) + \
100  '\nobjPropertyStruct: ' + str(objPropertyStruct))
101  # variable for result
102  ret = {}
103  if obj is not None:
104  objFields = {}
105  for key, value in obj.__dict__.items():
106  if isinstance(value, basestring):
107  # objFields[key.upper()] = MySQLdb.escape_string(str(value)) # pylint: disable=E1101
108  objFields[key.upper()] = dbWrapper.dbTask.dbConnections[DB_CONSTS.PRIMARY_DB_ID].escape_string(str(value))
109  else:
110  objFields[key.upper()] = value
111 
112  logger.debug('objFields: %s', str(objFields))
113 
114  for fieldName, fieldValue in objPropertyStruct.items():
115  logger.debug('fieldName: %s, fieldValue: %s', str(fieldName), str(fieldValue))
116  for sqlExpression, valueType in fieldValue.items():
117  logger.debug('sqlExpression: %s, valueType: %s', str(sqlExpression), str(valueType))
118  sqlQuery = 'SELECT ' + \
119  FieldsSQLExpressionEvaluator.macroReplace(sqlExpression, objFields,
120  FieldsSQLExpressionEvaluator.REPLACE_MARKER,
121  case=1)
122  logger.debug("sqlQuery: " + str(sqlQuery))
123  affectDB = dbWrapper.affect_db
124  dbWrapper.affect_db = True
125  customResponse = None
126  try:
127  customResponse = dbWrapper.customRequest(sqlQuery, FieldsSQLExpressionEvaluator.DB_NAME)
128  except DatabaseException, err:
129  logger.error("Bad query: " + str(sqlQuery))
130 
131  dbWrapper.affect_db = affectDB
132  logger.debug("customResponse: " + str(customResponse))
133  if customResponse is not None and len(customResponse) > 0 and len(customResponse[0]) > 0:
134  result = None
135  try:
136  if valueType == FieldsSQLExpressionEvaluator.CAST_TYPE_TO_INTEGER:
137  result = int(customResponse[0][0])
138  elif valueType == FieldsSQLExpressionEvaluator.CAST_TYPE_TO_STRING:
139  result = str(customResponse[0][0]) # pylint: disable=R0204
140  elif valueType == FieldsSQLExpressionEvaluator.CAST_TYPE_TO_DATETIME:
141  dt = DateTimeType.parse(customResponse[0][0])
142  if dt is not None:
143  result = dt.strftime("%Y-%m-%d %H:%M:%S")
144  else:
145  logger.debug('Unknown type for cast: ' + str(valueType))
146 
147  except Exception, err:
148  logger.error("Customization result by type failed, error: %s", str(err))
149 
150  logger.debug('result after cast: ' + str(result) + ' type: ' + str(type(result)))
151  # Update field value
152  if result is not None:
153  for fieldObjName, fieldDBName in fieldsDict.items():
154  if fieldDBName == fieldName and hasattr(obj, fieldObjName):
155  logger.debug("Found attribute '" + str(fieldObjName) + "' in object...")
156  ret[fieldObjName] = result
157 
158  return ret
159 
160 
app.Utils.varDump
def varDump(obj, stringify=True, strTypeMaxLen=256, strTypeCutSuffix='...', stringifyType=1, ignoreErrors=False, objectsHash=None, depth=0, indent=2, ensure_ascii=False, maxDepth=10)
Definition: Utils.py:410
integrity-urlContent.type
type
Definition: integrity-urlContent.py:27
Here is the call graph for this function:

◆ evaluatePDateTime()

def app.FieldsSQLExpressionEvaluator.FieldsSQLExpressionEvaluator.evaluatePDateTime (   siteProperties,
  dbWrapper,
  urlObj,
  logger,
  defaultPubdateValue = None 
)
static

Definition at line 194 of file FieldsSQLExpressionEvaluator.py.

194  def evaluatePDateTime(siteProperties, dbWrapper, urlObj, logger, defaultPubdateValue=None):
195  logger.debug("evaluatePDateTime enter.... property: '" + str(APP_CONSTS.SQL_EXPRESSION_FIELDS_PDATE_TIME) + \
196  "' is exist = " + \
197  str(True if APP_CONSTS.SQL_EXPRESSION_FIELDS_PDATE_TIME in siteProperties else False))
198 
199  localUrlObj = copy.deepcopy(urlObj)
200  # variable for result
201  ret = defaultPubdateValue
202  propertyList = None
203  if siteProperties is not None and APP_CONSTS.SQL_EXPRESSION_FIELDS_PDATE_TIME in siteProperties:
204  try:
205  propertyList = json.loads(siteProperties[APP_CONSTS.SQL_EXPRESSION_FIELDS_PDATE_TIME])
206  logger.debug("propertyList: " + varDump(propertyList) + " type: " + str(type(propertyList)))
207  except Exception, err:
208  logger.error("Load from site property '%s' has error: %s",
209  str(APP_CONSTS.SQL_EXPRESSION_FIELDS_PDATE_TIME), str(err))
210 
211  if propertyList is not None:
212  for propertyStruct in propertyList:
213  if "pattern" in propertyStruct and "value" in propertyStruct:
214  pattern = propertyStruct["pattern"]
215  value = propertyStruct["value"]
216 
217  logger.debug("pattern: " + str(pattern))
218  logger.debug("value: " + str(value))
219  logger.debug("localUrlObj.url: " + str(localUrlObj.url))
220  if localUrlObj.pDate is None:
221  localUrlObj.pDate = defaultPubdateValue
222 
223  # Check pattern apply to url
224  if re.search(pattern, localUrlObj.url, re.UNICODE) is not None:
225  objPropertyStruct = {DB_CONSTS.URLTableDict['pDate']:\
226  {value:FieldsSQLExpressionEvaluator.CAST_TYPE_TO_STRING}}
227  # Evaluate for 'URL' object
228  resDict = FieldsSQLExpressionEvaluator.evaluateElement(dbWrapper, logger, localUrlObj, DB_CONSTS.URLTableDict,
229  objPropertyStruct)
230  logger.debug("!!! evaluatePDateTime resDict: %s", str(resDict))
231  if "pDate" in resDict:
232  rawDate = resDict["pDate"]
233  if rawDate.isdigit():
234  logger.debug("!!! Return numeric value: " + str(rawDate))
235  d = DateTimeType(int(rawDate))
236  ret = d.getString()
237  else:
238  logger.debug("!!! Return string value: " + str(rawDate))
239  # ret = rawDate
240  dt = DateTimeType.parse(rawDate, True, logger, False)
241  if dt is not None:
242  ret = dt.strftime("%Y-%m-%d %H:%M:%S")
243  else:
244  ret = rawDate
245  else:
246  logger.error("Not found mandatory fields for property '%s'", \
247  str(APP_CONSTS.SQL_EXPRESSION_FIELDS_PDATE_TIME))
248 
249  logger.debug("!!! evaluatePDateTime ret: %s", str(ret))
250 
251  return ret
252 
app.Utils.varDump
def varDump(obj, stringify=True, strTypeMaxLen=256, strTypeCutSuffix='...', stringifyType=1, ignoreErrors=False, objectsHash=None, depth=0, indent=2, ensure_ascii=False, maxDepth=10)
Definition: Utils.py:410
integrity-urlContent.type
type
Definition: integrity-urlContent.py:27
Here is the call graph for this function:

◆ execute()

def app.FieldsSQLExpressionEvaluator.FieldsSQLExpressionEvaluator.execute (   siteProperties,
  dbWrapper,
  siteObj,
  urlObj,
  logger,
  propertyName = APP_CONSTS.SQL_EXPRESSION_FIELDS_UPDATE_CRAWLER 
)
static

Definition at line 57 of file FieldsSQLExpressionEvaluator.py.

57  propertyName=APP_CONSTS.SQL_EXPRESSION_FIELDS_UPDATE_CRAWLER):
58 
59  logger.debug("Execute enter.... property: '" + str(propertyName) + "' is exist = " + \
60  str(True if propertyName in siteProperties else False))
61  # variable for result
62  ret = {}
63  if dbWrapper is not None:
64  propertyList = None
65  if siteProperties is not None and propertyName in siteProperties:
66  try:
67  propertyList = json.loads(siteProperties[propertyName])
68  except Exception, err:
69  logger.error("Load from site property error: %s, source: %s", str(err), str(propertyName))
70 
71  if propertyList is not None:
72  for propertyStruct in propertyList:
73  if FieldsSQLExpressionEvaluator.OBJECT_NAME_SITE in propertyStruct:
74  # Evaluate for 'Site' object
75  ret = FieldsSQLExpressionEvaluator.evaluateElement(dbWrapper, logger, siteObj, DB_CONSTS.siteDict,
76  propertyStruct[FieldsSQLExpressionEvaluator.\
77  OBJECT_NAME_SITE])
78  elif FieldsSQLExpressionEvaluator.OBJECT_NAME_URL in propertyStruct:
79  # Evaluate for 'URL' object
80  ret = FieldsSQLExpressionEvaluator.evaluateElement(dbWrapper, logger, urlObj, DB_CONSTS.URLTableDict,
81  propertyStruct[FieldsSQLExpressionEvaluator.\
82  OBJECT_NAME_URL])
83  else:
84  logger.error("Not support name of object in inputted json")
85 
86  return ret
87 
88 

◆ macroReplace()

def app.FieldsSQLExpressionEvaluator.FieldsSQLExpressionEvaluator.macroReplace (   pattern,
  values,
  marker,
  case = MACRO_CASE_ORIGINAL 
)
static

Definition at line 169 of file FieldsSQLExpressionEvaluator.py.

169  def macroReplace(pattern, values, marker, case=MACRO_CASE_ORIGINAL):
170  ret = copy.copy(pattern)
171  for key in values:
172  if values[key] is not None:
173  if case == FieldsSQLExpressionEvaluator.MACRO_CASE_UPPER:
174  rkey = key.upper()
175  elif case == FieldsSQLExpressionEvaluator.MACRO_CASE_LOWER:
176  rkey = key.lower()
177  else:
178  rkey = key
179  ret = ret.replace(marker + rkey + marker, "'" + str(values[key]) + "'" if isinstance(values[key], basestring) \
180  else str(values[key]))
181 
182  return ret
183 
184 
Here is the caller graph for this function:

Member Data Documentation

◆ CAST_TYPE_TO_DATETIME

int app.FieldsSQLExpressionEvaluator.FieldsSQLExpressionEvaluator.CAST_TYPE_TO_DATETIME = 2
static

Definition at line 37 of file FieldsSQLExpressionEvaluator.py.

◆ CAST_TYPE_TO_INTEGER

int app.FieldsSQLExpressionEvaluator.FieldsSQLExpressionEvaluator.CAST_TYPE_TO_INTEGER = 0
static

Definition at line 35 of file FieldsSQLExpressionEvaluator.py.

◆ CAST_TYPE_TO_STRING

int app.FieldsSQLExpressionEvaluator.FieldsSQLExpressionEvaluator.CAST_TYPE_TO_STRING = 1
static

Definition at line 36 of file FieldsSQLExpressionEvaluator.py.

◆ DB_NAME

string app.FieldsSQLExpressionEvaluator.FieldsSQLExpressionEvaluator.DB_NAME = "dc_sites"
static

Definition at line 40 of file FieldsSQLExpressionEvaluator.py.

◆ MACRO_CASE_LOWER

int app.FieldsSQLExpressionEvaluator.FieldsSQLExpressionEvaluator.MACRO_CASE_LOWER = 2
static

Definition at line 33 of file FieldsSQLExpressionEvaluator.py.

◆ MACRO_CASE_ORIGINAL

int app.FieldsSQLExpressionEvaluator.FieldsSQLExpressionEvaluator.MACRO_CASE_ORIGINAL = 0
static

Definition at line 31 of file FieldsSQLExpressionEvaluator.py.

◆ MACRO_CASE_UPPER

int app.FieldsSQLExpressionEvaluator.FieldsSQLExpressionEvaluator.MACRO_CASE_UPPER = 1
static

Definition at line 32 of file FieldsSQLExpressionEvaluator.py.

◆ OBJECT_NAME_SITE

string app.FieldsSQLExpressionEvaluator.FieldsSQLExpressionEvaluator.OBJECT_NAME_SITE = 'Site'
static

Definition at line 28 of file FieldsSQLExpressionEvaluator.py.

◆ OBJECT_NAME_URL

string app.FieldsSQLExpressionEvaluator.FieldsSQLExpressionEvaluator.OBJECT_NAME_URL = 'URL'
static

Definition at line 29 of file FieldsSQLExpressionEvaluator.py.

◆ REPLACE_MARKER

string app.FieldsSQLExpressionEvaluator.FieldsSQLExpressionEvaluator.REPLACE_MARKER = '%'
static

Definition at line 39 of file FieldsSQLExpressionEvaluator.py.

The documentation for this class was generated from the following file: