HCE Project Python language Distributed Tasks Manager Application, Distributed Crawler Application and client API bindings.  2.0.0-chaika
Hierarchical Cluster Engine Python language binding
DCCObjectsSerializator.py
Go to the documentation of this file.
1 '''
2 Created on Apr 11, 2014
3 
4 @package: dc
5 @author: scorp
6 @link: http://hierarchical-cluster-engine.com/
7 @copyright: Copyright © 2013-2014 IOIX Ukraine
8 @license: http://hierarchical-cluster-engine.com/license/
9 @since: 0.1
10 '''
11 
12 import hashlib
13 import types
14 import dc.EventObjects
15 import dbi.EventObjects
16 from app.Utils import SQLExpression
17 import app.Exceptions as Exceptions
18 from app.Utils import UrlNormalizator
19 # from app.Utils import varDump
20 import app.Utils as Utils # pylint: disable=F0401
21 
22 # Logger initialization
23 logger = Utils.MPLogger().getLogger()
24 
25 
26 # #DTMCObjectsSerializator Class contents serialize/deserialize methods for incoming "DTMC" commands
27 #
28 class DCCObjectsSerializator(object):
29 
30  def __init__(self):
31  pass
32 
33 
34  # #timeExtractor method, extracts time in various format
35  def timeExtractor(self, timeField):
36  ret = None
37  if timeField is not None:
38  if type(timeField) in types.StringTypes:
39  # if type(timeField) == type("") or type(timeField) == type(unicode("")):
40  ret = SQLExpression(str(timeField))
41  elif type(timeField) is types.DictType:
42  # elif type(timeField) == type({}):
43  ret = SQLExpression(str(timeField["str"]))
44  return ret
45 
46 
47  # #checkProperties method, validate incoming properties dict, rise excaption if properties contains bad values
48  # properties - incoming properties dict
49  def checkProperties(self, properties):
50  for value in properties.values():
51  if not isinstance(value, str) and not isinstance(value, unicode) and not isinstance(value, list):
52  raise Exceptions.DeserilizeException("Properties field values can be string or list[] type")
53  elif isinstance(value, list) and len(value) != 3:
54  raise Exceptions.DeserilizeException("Properties field values with list[] type must be len() == 3")
55 
56 
57  # #checkFilters method, gets filters list and return list of SiteFilter objects
58  # filters - incoming filters list
59  def checkFilters(self, filters, sid):
60  ret = None
61  siteId = None
62  siteFilter = None
63  if filters is not None:
64  ret = []
65  for localFilter in filters:
66  if "siteId" in localFilter and localFilter["siteId"] is not None:
67  siteId = str(localFilter["siteId"])
68  else:
69  siteId = sid
70  if "pattern" not in localFilter or localFilter["pattern"] is None:
71  raise Exceptions.DeserilizeException(".filters pattern not found or None")
72  if "type" not in localFilter:
73  siteFilter = dc.EventObjects.SiteFilter(siteId, str(localFilter["pattern"]))
74  else:
75  siteFilter = dc.EventObjects.SiteFilter(siteId, str(localFilter["pattern"]), int(localFilter["type"]))
76  if "subject" in localFilter and localFilter["subject"] is not None:
77  siteFilter.subject = str(localFilter["subject"])
78  if "opCode" in localFilter and localFilter["opCode"] is not None:
79  siteFilter.opCode = int(localFilter["opCode"])
80  if "stage" in localFilter and localFilter["stage"] is not None:
81  siteFilter.stage = int(localFilter["stage"])
82  if "action" in localFilter and localFilter["action"] is not None:
83  siteFilter.action = int(localFilter["action"])
84  if "mode" in localFilter and localFilter["mode"] is not None:
85  siteFilter.mode = int(localFilter["mode"])
86  if "state" in localFilter and localFilter["state"] is not None:
87  siteFilter.state = int(localFilter["state"])
88  if "uDate" in localFilter:
89  siteFilter.uDate = self.timeExtractor(localFilter["uDate"])
90  if "groupId" in localFilter:
91  siteFilter.groupId = int(localFilter["groupId"])
92  ret.append(siteFilter)
93  return ret
94 
95 
96  # #simpleValueExtractor method, extracts values for simple fields
97  # fieldName - field name
98  # jsonData - json container
99  # _type - field type
100  # excepStr - exception string
101  # i - index value
102  def simpleValueExtractor(self, fieldName, jsonData, _type, excepStr, i=None):
103  ret = None
104  if fieldName in jsonData:
105  ret = jsonData[fieldName]
106  if ret is not None:
107  ret = type(_type)(ret)
108  else:
109  if i is None:
110  raise Exceptions.DeserilizeException("%s json-field not found" % excepStr)
111  else:
112  raise Exceptions.DeserilizeException("%s json-field not found in [%s] elem" % (excepStr, str(i)))
113  return ret
114 
115 
116  # #fillSiteURLs method, fills dc.EventObjects.SiteURL object
117  # jsonData - json container
118  # siteId - site's id value
119  # returns dc.EventObjects.SiteURL object
120  def fillSiteURLs(self, jsonData, siteId, userId=None, urlNormalizeMask=UrlNormalizator.NORM_NONE):
121  ret = None
122  if "url" in jsonData and jsonData["url"] is not None:
123  ret = dc.EventObjects.SiteURL(siteId, jsonData["url"], normalizeMask=urlNormalizeMask)
124  ret.userId = userId
125  for key in jsonData:
126  if hasattr(ret, key) and jsonData[key] is not None and key != "url":
127  setattr(ret, key, jsonData[key])
128  else:
129  logger.error(">>> Some URL-dict elem doesn't content url field")
130  return ret
131 
132 
133  # #readBatchItems method, reads and returns bathitem list
134  # jsonData - json container
135  # returns batcjItem list
136  def readBatchItems(self, jsonData):
137  ret = []
138  i = 0
139  if hasattr(jsonData, '__iter__'):
140  for elem in jsonData:
141  localSiteId = self.simpleValueExtractor("siteId", elem, str(), "BatchItem.siteId", i)
142  localUrlId = self.simpleValueExtractor("urlId", elem, str(), "BatchItem.urlId", i)
143  if "urlObj" in elem and elem["urlObj"] is not None:
144  localUrlObj = dc.EventObjects.URL(0, "")
145  self.URLNewElementDeserialize(localUrlObj, elem["urlObj"], i)
146  else:
147  raise Exceptions.DeserilizeException("BatchItem.urlObj json-field not found in {0} elem".format(i))
148 
149  localBatchItem = dc.EventObjects.BatchItem(localSiteId, localUrlId, localUrlObj)
150  if "properties" in elem and elem["properties"] is not None and type(elem["properties"]) is types.DictType:
151  localBatchItem.properties = elem["properties"]
152  if "siteObj" in elem and elem["siteObj"] is not None:
153  localBatchItem.siteObj = self.siteNewDeserialize(elem["siteObj"])
154  if "depth" in elem:
155  localBatchItem.depth = self.simpleValueExtractor("depth", elem, int(), "BatchItem.depth", i)
156  ret.append(localBatchItem)
157  i = i + 1
158 
159  return ret
160 
161 
162  # #siteNewDeserialize method, deserializes incoming jsonData to the Site object
163  # jsonData - incoming json string
164  def siteNewDeserialize(self, jsonData):
165  siteObject = None
166  try:
167  if "id" in jsonData and jsonData["id"] is not None:
168  siteObject = dc.EventObjects.Site("")
169  siteObject.id = str(jsonData["id"])
170  else:
171  if "urls" in jsonData and hasattr(jsonData["urls"], '____') and \
172  len(jsonData["urls"]) > 0 and jsonData["urls"][0] is not None:
173  siteObject = dc.EventObjects.Site(jsonData["urls"][0])
174  else:
175  raise Exceptions.DeserilizeException("Site.id json-field not found and Site.urls[0] no found")
176  if "uDate" in jsonData:
177  siteObject.uDate = self.timeExtractor(jsonData["uDate"])
178  else:
179  raise Exceptions.DeserilizeException("Site.uDate json-field not found")
180  if "tcDate" in jsonData:
181  siteObject.tcDate = self.timeExtractor(jsonData["tcDate"])
182  else:
183  raise Exceptions.DeserilizeException("Site.tcDate json-field not found")
184  if "cDate" in jsonData:
185  siteObject.cDate = self.timeExtractor(jsonData["cDate"])
186  else:
187  raise Exceptions.DeserilizeException("Site.cDate json-field not found")
188  siteObject.resources = self.simpleValueExtractor("resources", jsonData, int(), "Site.resources")
189  siteObject.iterations = self.simpleValueExtractor("iterations", jsonData, int(), "Site.iterations")
190  siteObject.description = self.simpleValueExtractor("description", jsonData, str(), "Site.description")
191  siteObject.userId = self.simpleValueExtractor("userId", jsonData, int(), "Site.userId")
192 
193  if "urls" in jsonData:
194  if jsonData["urls"] is None:
195  siteObject.urls = None
196  elif not isinstance(jsonData["urls"], list):
197  raise Exceptions.DeserilizeException("Site.urls not list type")
198  else:
199  for jsonUrl in jsonData["urls"]:
200  localRet = None
201  if isinstance(jsonUrl, dict):
202  localRet = self.fillSiteURLs(jsonUrl, siteObject.id, siteObject.userId,
203  int(jsonUrl["_normalizeMask"]) if "_normalizeMask" in jsonUrl else \
204  UrlNormalizator.NORM_NONE)
205  elif type(jsonUrl) in types.StringTypes:
206  localRet = dc.EventObjects.SiteURL(siteId=siteObject.id, url=jsonUrl,
207  normalizeMask=int(jsonData["_urlNormalizeMask"])\
208  if "_urlNormalizeMask" in jsonData else UrlNormalizator.NORM_NONE)
209  localRet.userId = siteObject.userId
210  if localRet is not None:
211  if siteObject.urls is None:
212  siteObject.urls = []
213  siteObject.urls.append(localRet)
214  else:
215  raise Exceptions.DeserilizeException("Site.urls json-field not found")
216 
217  if "filters" in jsonData:
218  if jsonData["filters"] is not None and type(jsonData["filters"]) is not types.ListType:
219  raise Exceptions.DeserilizeException("Site.filters not list type")
220  siteObject.filters = self.checkFilters(jsonData["filters"], siteObject.id)
221  else:
222  raise Exceptions.DeserilizeException("Site.filters json-field not found")
223  if "properties" in jsonData:
224  if jsonData["properties"] is not None and type(jsonData["properties"]) is not types.DictType and \
225  type(jsonData["properties"]) is not types.ListType:
226  raise Exceptions.DeserilizeException("Site.properties not dict/list type")
227  if type(jsonData["properties"]) is not types.ListType:
228  self.checkProperties(jsonData["properties"])
229  siteObject.properties = jsonData["properties"]
230  else:
231  raise Exceptions.DeserilizeException("Site.properties json-field not found")
232  siteObject.state = self.simpleValueExtractor("state", jsonData, int(), "Site.state")
233  siteObject.priority = self.simpleValueExtractor("priority", jsonData, int(), "Site.priority")
234  siteObject.maxURLs = self.simpleValueExtractor("maxURLs", jsonData, int(), "Site.maxURLs")
235  siteObject.maxResources = self.simpleValueExtractor("maxResources", jsonData, int(), "Site.maxResources")
236  siteObject.maxErrors = self.simpleValueExtractor("maxErrors", jsonData, int(), "Site.maxErrors")
237  siteObject.maxResourceSize = self.simpleValueExtractor("maxResourceSize", jsonData, int(), "Site.maxResourceSize")
238  siteObject.requestDelay = self.simpleValueExtractor("requestDelay", jsonData, int(), "Site.requestDelay")
239  siteObject.httpTimeout = self.simpleValueExtractor("httpTimeout", jsonData, float(), "Site.httpTimeout")
240  siteObject.errorMask = self.simpleValueExtractor("errorMask", jsonData, int(), "Site.errorMask")
241  siteObject.errors = self.simpleValueExtractor("errors", jsonData, int(), "Site.errors")
242  siteObject.urlType = self.simpleValueExtractor("urlType", jsonData, int(), "Site.urlType")
243  siteObject.contents = self.simpleValueExtractor("contents", jsonData, int(), "Site.contents")
244  siteObject.processingDelay = self.simpleValueExtractor("processingDelay", jsonData, int(), "Site.processingDelay")
245  siteObject.size = self.simpleValueExtractor("size", jsonData, int(), "Site.size")
246  siteObject.avgSpeed = self.simpleValueExtractor("avgSpeed", jsonData, int(), "Site.avgSpeed")
247  siteObject.avgSpeedCounter = self.simpleValueExtractor("avgSpeedCounter", jsonData, int(), "Site.avgSpeedCounter")
248  # update by Oleksii
249  # add support maxURLsFromPage (dc-170)
250  siteObject.maxURLsFromPage = self.simpleValueExtractor("maxURLsFromPage", jsonData, int(), "Site.maxURLsFromPage")
251  siteObject.recrawlPeriod = self.simpleValueExtractor("recrawlPeriod", jsonData, int(), "Site.recrawlPeriod")
252  siteObject.maxURLsFromPage = self.simpleValueExtractor("maxURLsFromPage", jsonData, int(), "Site.maxURLsFromPage")
253  if "recrawlDate" in jsonData:
254  siteObject.recrawlDate = self.timeExtractor(jsonData["recrawlDate"])
255  else:
256  raise Exceptions.DeserilizeException("Site.recrawlDate json-field not found")
257  siteObject.collectedURLs = self.simpleValueExtractor("collectedURLs", jsonData, int(), "Site.collectedURLs")
258  siteObject.fetchType = self.simpleValueExtractor("fetchType", jsonData, int(), "Site.fetchType")
259  if "newURLs" in jsonData:
260  siteObject.newURLs = self.simpleValueExtractor("newURLs", jsonData, int(), "Site.newURLs")
261  if "deletedURLs" in jsonData:
262  siteObject.deletedURLs = self.simpleValueExtractor("deletedURLs", jsonData, int(), "Site.deletedURLs")
263  if "moveURLs" in jsonData and jsonData["moveURLs"] is not None:
264  siteObject.moveURLs = self.simpleValueExtractor("moveURLs", jsonData, int(), "Site.moveURLs")
265  siteObject.moveURLs = bool(siteObject.moveURLs)
266  if "tcDateProcess" in jsonData:
267  siteObject.tcDateProcess = self.timeExtractor(jsonData["tcDateProcess"])
268  if "categoryId" in jsonData:
269  siteObject.categoryId = self.simpleValueExtractor("categoryId", jsonData, int(), "Site.categoryId")
270  except Exceptions.DeserilizeException, e:
271  raise e
272  except (ValueError, TypeError):
273  raise Exceptions.DeserilizeException("siteNewDeserialize, some field has invalid type")
274  return siteObject
275 
276 
277  # #siteUpdateDeserialize method, deserializes incoming jsonData to the SiteUpdate object
278  # jsonData - incoming json string
279  def siteUpdateDeserialize(self, jsonData):
280  siteUpdateObject = dc.EventObjects.SiteUpdate("")
281  try:
282  siteUpdateObject.updateType = self.simpleValueExtractor("updateType", jsonData, int(), "SiteUpdate.updateType")
283  siteUpdateObject.id = self.simpleValueExtractor("id", jsonData, str(), "SiteUpdate.id")
284  if "uDate" in jsonData:
285  siteUpdateObject.uDate = self.timeExtractor(jsonData["uDate"])
286  else:
287  raise Exceptions.DeserilizeException("SiteUpdate.uDate json-field not found")
288  if "tcDate" in jsonData:
289  siteUpdateObject.tcDate = self.timeExtractor(jsonData["tcDate"])
290  else:
291  raise Exceptions.DeserilizeException("SiteUpdate.tcDate json-field not found")
292  if "cDate" in jsonData:
293  siteUpdateObject.cDate = self.timeExtractor(jsonData["cDate"])
294  else:
295  raise Exceptions.DeserilizeException("SiteUpdate.cDate json-field not found")
296  siteUpdateObject.resources = self.simpleValueExtractor("resources", jsonData, int(), "SiteUpdate.resources")
297  siteUpdateObject.iterations = self.simpleValueExtractor("iterations", jsonData, int(), "SiteUpdate.iterations")
298  siteUpdateObject.description = self.simpleValueExtractor("description", jsonData, str(), "SiteUpdate.description")
299  siteUpdateObject.userId = self.simpleValueExtractor("userId", jsonData, int(), "SiteUpdate.userId")
300 
301  if "urls" in jsonData:
302  if jsonData["urls"] is None:
303  siteUpdateObject.urls = None
304  elif not isinstance(jsonData["urls"], list):
305  raise Exceptions.DeserilizeException("SiteUpdate.urls not list type")
306  else:
307  siteUpdateObject.urls = []
308  for jsonUrl in jsonData["urls"]:
309  localRet = None
310  if isinstance(jsonUrl, dict):
311  localRet = self.fillSiteURLs(jsonUrl, siteUpdateObject.id, siteUpdateObject.userId,
312  int(jsonUrl["_normalizeMask"]) if "_normalizeMask" in jsonUrl else \
313  UrlNormalizator.NORM_NONE)
314  elif isinstance(jsonUrl, basestring):
315  localRet = dc.EventObjects.SiteURL(siteId=siteUpdateObject.id, url=jsonUrl,
316  normalizeMask=int(jsonData["_urlNormalizeMask"])\
317  if "_urlNormalizeMask" in jsonData else UrlNormalizator.NORM_NONE)
318  localRet.userId = siteUpdateObject.userId
319  if localRet is not None:
320  siteUpdateObject.urls.append(localRet)
321  # TODO remove it in feauture
322  if siteUpdateObject.urls is not None:
323  for url in siteUpdateObject.urls:
324  url.status = dc.EventObjects.URL.STATUS_PROCESSED
325  else:
326  raise Exceptions.DeserilizeException("SiteUpdate.urls json-field not found")
327 
328  if "filters" in jsonData:
329  if jsonData["filters"] is not None and type(jsonData["filters"]) is not types.ListType:
330  raise Exceptions.DeserilizeException("SiteUpdate.filters not list type")
331  siteUpdateObject.filters = self.checkFilters(jsonData["filters"], siteUpdateObject.id)
332  else:
333  raise Exceptions.DeserilizeException("SiteUpdate.filters json-field not found")
334  if "properties" in jsonData:
335  if jsonData["properties"] is not None and type(jsonData["properties"]) is not types.DictType and \
336  type(jsonData["properties"]) is not types.ListType:
337  raise Exceptions.DeserilizeException("SiteUpdate.properties not dict type")
338  if type(jsonData["properties"]) is types.DictType:
339  self.checkProperties(jsonData["properties"])
340  siteUpdateObject.properties = jsonData["properties"]
341  else:
342  raise Exceptions.DeserilizeException("SiteUpdate.properties json-field not found")
343  siteUpdateObject.state = self.simpleValueExtractor("state", jsonData, int(), "SiteUpdate.state")
344  siteUpdateObject.priority = self.simpleValueExtractor("priority", jsonData, int(), "SiteUpdate.priority")
345  siteUpdateObject.maxURLs = self.simpleValueExtractor("maxURLs", jsonData, int(), "SiteUpdate.maxURLs")
346  siteUpdateObject.maxResources = self.simpleValueExtractor("maxResources", jsonData, int(), \
347  "SiteUpdate.maxResources")
348  siteUpdateObject.maxErrors = self.simpleValueExtractor("maxErrors", jsonData, int(), "SiteUpdate.maxErrors")
349  siteUpdateObject.maxResourceSize = self.simpleValueExtractor("maxResourceSize", jsonData, int(), \
350  "SiteUpdate.maxResourceSize")
351  siteUpdateObject.requestDelay = self.simpleValueExtractor("requestDelay", jsonData, int(), \
352  "SiteUpdate.requestDelay")
353  siteUpdateObject.httpTimeout = self.simpleValueExtractor("httpTimeout", jsonData, float(), "SiteUpdate.httpTimeout")
354  siteUpdateObject.errorMask = self.simpleValueExtractor("errorMask", jsonData, int(), "SiteUpdate.errorMask")
355  siteUpdateObject.errors = self.simpleValueExtractor("errors", jsonData, int(), "SiteUpdate.errors")
356  siteUpdateObject.urlType = self.simpleValueExtractor("urlType", jsonData, int(), "SiteUpdate.urlType")
357  siteUpdateObject.contents = self.simpleValueExtractor("contents", jsonData, int(), "SiteUpdate.contents")
358  siteUpdateObject.processingDelay = self.simpleValueExtractor("processingDelay", jsonData, int(), \
359  "SiteUpdate.processingDelay")
360  siteUpdateObject.size = self.simpleValueExtractor("size", jsonData, int(), "SiteUpdate.size")
361  siteUpdateObject.avgSpeed = self.simpleValueExtractor("avgSpeed", jsonData, int(), "SiteUpdate.avgSpeed")
362  siteUpdateObject.avgSpeedCounter = self.simpleValueExtractor("avgSpeedCounter", jsonData, int(), \
363  "SiteUpdate.avgSpeedCounter")
364  siteUpdateObject.recrawlPeriod = self.simpleValueExtractor("recrawlPeriod", jsonData, int(), \
365  "siteUpdateObject.recrawlPeriod")
366  siteUpdateObject.maxURLsFromPage = self.simpleValueExtractor("maxURLsFromPage", jsonData, int(), \
367  "siteUpdateObject.maxURLsFromPage")
368  if "recrawlDate" in jsonData:
369  siteUpdateObject.recrawlDate = self.timeExtractor(jsonData["recrawlDate"])
370  else:
371  raise Exceptions.DeserilizeException("siteUpdateObject.recrawlDate json-field not found")
372 
373  if "criterions" in jsonData and jsonData["criterions"] is not None:
374  if type(jsonData["criterions"]) is not types.DictType:
375  raise Exceptions.DeserilizeException("URLCleanup.criterions json-field type")
376  if hasattr(siteUpdateObject.criterions, '__iter__'):
377  siteUpdateObject.criterions.update(jsonData["criterions"])
378  else:
379  siteUpdateObject.criterions = jsonData["criterions"]
380  siteUpdateObject.collectedURLs = self.simpleValueExtractor("collectedURLs",
381  jsonData, int(), "siteUpdateObject.collectedURLs")
382  siteUpdateObject.fetchType = self.simpleValueExtractor("fetchType", jsonData, int(), "siteUpdateObject.fetchType")
383  if "newURLs" in jsonData:
384  siteUpdateObject.newURLs = self.simpleValueExtractor("newURLs", jsonData, int(), "siteUpdateObject.newURLs")
385  if "deletedURLs" in jsonData:
386  siteUpdateObject.deletedURLs = self.simpleValueExtractor("deletedURLs", jsonData, int(),
387  "siteUpdateObject.deletedURLs")
388  if "tcDateProcess" in jsonData:
389  siteUpdateObject.tcDateProcess = self.timeExtractor(jsonData["tcDateProcess"])
390  if "categoryId" in jsonData:
391  siteUpdateObject.categoryId = self.simpleValueExtractor("categoryId", jsonData, int(),
392  "siteUpdateObject.categoryId")
393  except Exceptions.DeserilizeException:
394  raise
395  except (ValueError, TypeError):
396  raise Exceptions.DeserilizeException("siteUpdateDeserialize, some field has invalid type")
397  return siteUpdateObject
398 
399 
400  # #siteUpdateDeserialize method, deserializes incoming jsonData to the SiteUpdate object
401  # jsonData - incoming json string
402  def siteFindDeserialize(self, jsonData):
403  siteFindObject = dc.EventObjects.SiteFind("")
404  try:
405  if "url" in jsonData:
406  # if jsonData["url"] is None or jsonData["url"] == "":
407  # raise Exceptions.DeserilizeException("SiteFind.url not exist or empty")
408  siteFindObject.url = jsonData["url"]
409  if "criterions" in jsonData and jsonData["criterions"] is not None:
410  if type(jsonData["criterions"]) is not types.DictType:
411  raise Exceptions.DeserilizeException("URLFetch.sitesCriterions json-field type not dict")
412  siteFindObject.criterions.update(jsonData["criterions"])
413  if "excludeList" in jsonData and type(jsonData["excludeList"]) is types.ListType:
414  siteFindObject.excludeList = jsonData["excludeList"]
415  except (ValueError, TypeError):
416  raise Exceptions.DeserilizeException("siteFindDeserialize, some field has invalid type")
417  return siteFindObject
418 
419 
420  # #siteStatusDeserialize method, deserializes incoming jsonData to the SiteStatus object
421  # jsonData - incoming json string
422  def siteStatusDeserialize(self, jsonData):
423  siteStatusObject = dc.EventObjects.SiteStatus("")
424  try:
425  if "id" in jsonData:
426  siteStatusObject.id = jsonData["id"]
427  else:
428  raise Exceptions.DeserilizeException("SiteStatus.id json-field not found")
429  if "deleteTaskId" in jsonData:
430  siteStatusObject.deleteTaskId = jsonData["deleteTaskId"]
431  else:
432  raise Exceptions.DeserilizeException("SiteStatus.deleteTaskId json-field not found")
433  if "excludeList" in jsonData and type(jsonData["excludeList"]) is types.ListType:
434  siteStatusObject.excludeList = jsonData["excludeList"]
435  except (ValueError, TypeError):
436  raise Exceptions.DeserilizeException("siteStatusDeserialize, some field has invalid type")
437  return siteStatusObject
438 
439 
440  # #siteDeleteDeserialize method, deserializes incoming jsonData to the SiteDelete object
441  # jsonData - incoming json string
442  def siteDeleteDeserialize(self, jsonData):
443  if type(jsonData) is types.ListType:
444  siteDeleteObject = []
445  jsonDataElems = jsonData
446  else:
447  siteDeleteObject = dc.EventObjects.SiteDelete()
448  jsonDataElems = [jsonData]
449 
450  for elem in jsonDataElems:
451  try:
452  localSiteDeleteObject = dc.EventObjects.SiteDelete()
453  if "id" in elem:
454  localSiteDeleteObject.id = elem["id"]
455  if "taskType" in elem:
456  localSiteDeleteObject.taskType = int(elem["taskType"])
457  else:
458  raise Exceptions.DeserilizeException("SiteDelete.taskType json-field not found")
459  if "delayedType" in elem and elem["delayedType"] is not None:
460  localSiteDeleteObject.delayedType = int(elem["delayedType"])
461  if "criterions" in elem:
462  if type(elem["criterions"]) is not types.DictType:
463  raise Exceptions.DeserilizeException("SiteDelete.criterions not dict type")
464  if hasattr(localSiteDeleteObject.criterions, '__iter__'):
465  localSiteDeleteObject.criterions.update(elem["criterions"])
466  else:
467  localSiteDeleteObject.criterions = elem["criterions"]
468  except ValueError:
469  raise Exceptions.DeserilizeException("siteDeleteDeserialize, some field has invalid type")
470  if type(jsonData) is types.ListType:
471  siteDeleteObject.append(localSiteDeleteObject)
472  else:
473  siteDeleteObject = localSiteDeleteObject
474  return siteDeleteObject
475 
476 
477  # #siteCleanupDeserialize method, deserializes incoming jsonData to the SiteCleanup object
478  # jsonData - incoming json string
479  def siteCleanupDeserialize(self, jsonData):
480  if type(jsonData) is types.ListType:
481  siteCleanupObject = []
482  jsonDataElems = jsonData
483  else:
484  siteCleanupObject = dc.EventObjects.SiteCleanup("")
485  jsonDataElems = [jsonData]
486 
487  for elem in jsonDataElems:
488  localSiteCleanupObject = dc.EventObjects.SiteCleanup("")
489  try:
490  if "id" in elem:
491  localSiteCleanupObject.id = elem["id"]
492  else:
493  raise Exceptions.DeserilizeException("SiteCleanup.id json-field not found")
494  if "taskType" in elem:
495  localSiteCleanupObject.taskType = int(elem["taskType"])
496  else:
497  raise Exceptions.DeserilizeException("SiteCleanup.taskType json-field not found")
498  if "delayedType" in elem and elem["delayedType"] is not None:
499  localSiteCleanupObject.delayedType = int(elem["delayedType"])
500  if "moveURLs" in elem and elem["moveURLs"] is not None:
501  localSiteCleanupObject.moveURLs = self.simpleValueExtractor("moveURLs", elem, int(), "Site.moveURLs")
502  localSiteCleanupObject.moveURLs = bool(localSiteCleanupObject.moveURLs)
503  if "saveRootURLs" in elem and elem["saveRootURLs"] is not None:
504  localSiteCleanupObject.saveRootUrls = self.simpleValueExtractor("saveRootURLs", elem, int(),
505  "Site.saveRootURLs")
506  localSiteCleanupObject.saveRootUrls = bool(localSiteCleanupObject.saveRootUrls)
507  if "state" in elem and elem["state"] is not None:
508  localSiteCleanupObject.state = int(elem["state"])
509  if "historyCleanUp" in elem and elem["historyCleanUp"] is not None:
510  localSiteCleanupObject.historyCleanUp = int(elem["historyCleanUp"])
511  except (ValueError, TypeError):
512  raise Exceptions.DeserilizeException("siteCleanupDeserialize, some field has invalid type")
513  if type(jsonData) is types.ListType:
514  siteCleanupObject.append(localSiteCleanupObject)
515  else:
516  siteCleanupObject = localSiteCleanupObject
517  return siteCleanupObject
518 
519 
520  # #URLNewElementDeserialize method, fills data in one URLNew elemet
521  #
522  def URLNewElementDeserialize(self, urlNewObject, jsonData, i):
523  urlNewObject.siteId = self.simpleValueExtractor("siteId", jsonData, str(), "URLNew.siteId", i)
524  if "url" in jsonData:
525  urlNewObject.url = str(jsonData["url"])
526  else:
527  raise Exceptions.DeserilizeException("URLNew.url json-field not found in {0} jsonData".format(i))
528  urlNewObject.type = self.simpleValueExtractor("type", jsonData, int(), "URLNew.type", i)
529  urlNewObject.state = self.simpleValueExtractor("state", jsonData, int(), "URLNew.state", i)
530  urlNewObject.status = self.simpleValueExtractor("status", jsonData, int(), "URLNew.status", i)
531  urlNewObject.siteSelect = self.simpleValueExtractor("siteSelect", jsonData, int(), "URLNew.siteSelect", i)
532  urlNewObject.crawled = self.simpleValueExtractor("crawled", jsonData, int(), "URLNew.crawled", i)
533  urlNewObject.processed = self.simpleValueExtractor("processed", jsonData, int(), "URLNew.processed", i)
534  urlNewObject.urlMd5 = self.simpleValueExtractor("urlMd5", jsonData, str(), "URLNew.urlMd5", i)
535  if urlNewObject.urlMd5 is None:
536  urlNewObject.fillMD5 = hashlib.md5(urlNewObject.url).hexdigest()
537  urlNewObject.contentType = self.simpleValueExtractor("contentType", jsonData, str(), "URLNew.contentType", i)
538  urlNewObject.requestDelay = self.simpleValueExtractor("requestDelay", jsonData, int(), \
539  "URLNew.requestDelay", i)
540  urlNewObject.processingDelay = self.simpleValueExtractor("processingDelay", jsonData, int(), \
541  "URLNew.processingDelay", i)
542  urlNewObject.httpTimeout = self.simpleValueExtractor("httpTimeout", jsonData, float(), "URLNew.httpTimeout", i)
543  urlNewObject.charset = self.simpleValueExtractor("charset", jsonData, str(), "URLNew.charset", i)
544  urlNewObject.batchId = self.simpleValueExtractor("batchId", jsonData, int(), "URLNew.batchId", i)
545  urlNewObject.errorMask = self.simpleValueExtractor("errorMask", jsonData, int(), "URLNew.errorMask", i)
546  urlNewObject.crawlingTime = self.simpleValueExtractor("crawlingTime", jsonData, int(), \
547  "URLNew.crawlingTime", i)
548  urlNewObject.processingTime = self.simpleValueExtractor("processingTime", jsonData, int(), \
549  "URLNew.processingTime", i)
550  urlNewObject.totalTime = self.simpleValueExtractor("totalTime", jsonData, int(), "URLNew.totalTime", i)
551  urlNewObject.httpCode = self.simpleValueExtractor("httpCode", jsonData, int(), "URLNew.httpCode", i)
552  if "UDate" in jsonData:
553  urlNewObject.UDate = self.timeExtractor(jsonData["UDate"])
554  else:
555  raise Exceptions.DeserilizeException("URLNew.UDate json-field not found in {0} jsonData".format(i))
556  if "CDate" in jsonData:
557  urlNewObject.CDate = self.timeExtractor(jsonData["CDate"])
558  else:
559  raise Exceptions.DeserilizeException("URLNew.CDate json-field not found in {0} jsonData".format(i))
560  urlNewObject.httpMethod = self.simpleValueExtractor("httpMethod", jsonData, str(), "URLNew.httpMethod", i)
561  urlNewObject.size = self.simpleValueExtractor("size", jsonData, int(), "URLNew.size", i)
562  urlNewObject.linksI = self.simpleValueExtractor("linksI", jsonData, int(), "URLNew.linksI", i)
563  urlNewObject.linksE = self.simpleValueExtractor("linksE", jsonData, int(), "URLNew.linksE", i)
564  urlNewObject.freq = self.simpleValueExtractor("freq", jsonData, int(), "URLNew.freq", i)
565  urlNewObject.depth = self.simpleValueExtractor("depth", jsonData, int(), "URLNew.depth", i)
566  urlNewObject.rawContentMd5 = self.simpleValueExtractor("rawContentMd5", jsonData, str(), \
567  "URLNew.rawContentMd5", i)
568  urlNewObject.parentMd5 = self.simpleValueExtractor("parentMd5", jsonData, str(), "URLNew.parentMd5", i)
569  if "lastModified" in jsonData:
570  urlNewObject.lastModified = self.timeExtractor(jsonData["lastModified"])
571  else:
572  raise Exceptions.DeserilizeException("URLNew.lastModified json-field not found in {0} jsonData".format(i))
573  urlNewObject.eTag = self.simpleValueExtractor("eTag", jsonData, str(), "URLNew.eTag", i)
574  urlNewObject.mRate = self.simpleValueExtractor("mRate", jsonData, int(), "URLNew.mRate", i)
575  urlNewObject.mRateCounter = self.simpleValueExtractor("mRateCounter", jsonData, int(), "URLNew.mRateCounter", i)
576  urlNewObject.maxURLsFromPage = self.simpleValueExtractor("maxURLsFromPage", jsonData, int(), \
577  "URLNew.maxURLsFromPage", i)
578  if "tcDate" in jsonData:
579  urlNewObject.tcDate = self.timeExtractor(jsonData["tcDate"])
580  else:
581  raise Exceptions.DeserilizeException("URLNew.tcDate json-field not found in {0} jsonData".format(i))
582  urlNewObject.tagsMask = self.simpleValueExtractor("tagsMask", jsonData, int(), "URLNew.tagsMask", i)
583  urlNewObject.tagsCount = self.simpleValueExtractor("tagsCount", jsonData, int(), "URLNew.tagsCount", i)
584  if "pDate" in jsonData:
585  urlNewObject.pDate = self.timeExtractor(jsonData["pDate"])
586  else:
587  raise Exceptions.DeserilizeException("URLNew.pDate json-field not found in {0} jsonData".format(i))
588  urlNewObject.contentURLMd5 = self.simpleValueExtractor("contentURLMd5", jsonData, str(), "URLNew.contentURLMd5", i)
589  urlNewObject.priority = self.simpleValueExtractor("priority", jsonData, int(), "URLNew.priority", i)
590  if "urlUpdate" in jsonData and jsonData["urlUpdate"] is not None:
591  urlNewObject.urlUpdate = self.URLUpdateDeserializeOneElem(jsonData["urlUpdate"], 0)
592  if "urlPut" in jsonData and jsonData["urlPut"] is not None:
593  urlNewObject.urlPut = self.URLPutDeserializeOneElement(jsonData["urlPut"], 0)
594  if "chainId" in jsonData:
595  urlNewObject.chainId = self.simpleValueExtractor("chainId", jsonData, int(), "URLNew.chainId", i)
596  if "classifierMask" in jsonData:
597  urlNewObject.classifierMask = self.simpleValueExtractor("classifierMask", jsonData, int(),
598  "URLNew.classifierMask", i)
599  if "attributes" in jsonData:
600  urlNewObject.attributes = []
601  for attribute in jsonData["attributes"]:
602  if type(attribute) is types.DictType and len(attribute) == 1:
603  attrObj = dc.EventObjects.Attribute(urlNewObject.siteId, attribute.keys()[0])
604  attrObj.urlMd5 = urlNewObject.urlMd5
605  attrObj.value = attribute[attribute.keys()[0]]
606  urlNewObject.attributes.append(attrObj)
607 
608 
609  # #URLNewDeserialize method, deserializes incoming jsonData to the URL object
610  # jsonData - incoming json string
611  def URLNewDeserialize(self, jsonData):
612  URLNewObject = []
613  i = 0
614  try:
615  if type(jsonData) is types.ListType:
616  for elem in jsonData:
617  URLNewObject.append(dc.EventObjects.URL(0, ""))
618  self.URLNewElementDeserialize(URLNewObject[i], elem, i)
619  i += 1
620  else:
621  raise Exceptions.DeserilizeException("URLNew not list type")
622  except (ValueError, TypeError) as excp:
623  raise Exceptions.DeserilizeException("URLNewDeserialize, some field has invalid type; =" + str(excp))
624  return URLNewObject
625 
626 
627  # #URLStatusDeserialize method, deserializes incoming jsonData to the [URLStatus] object
628  # jsonData - incoming json string
629  def URLStatusDeserialize(self, jsonData):
630  URLStatusObject = []
631  i = 0
632  try:
633  if type(jsonData) is types.ListType:
634  for elem in jsonData:
635  URLStatusObject.append(dc.EventObjects.URLStatus(0, ""))
636  if "siteId" in elem:
637  URLStatusObject[i].siteId = str(elem["siteId"])
638  else:
639  raise Exceptions.DeserilizeException("URLStatus.siteId json-field not found in {0} elem".format(i))
640  if "url" in elem:
641  URLStatusObject[i].url = str(elem["url"])
642  else:
643  raise Exceptions.DeserilizeException("URLStatus.url json-field not found in {0} elem".format(i))
644  if "urlType" in elem:
645  URLStatusObject[i].urlType = int(elem["urlType"])
646  else:
647  raise Exceptions.DeserilizeException("URLStatus.urlType json-field not found in {0} elem".format(i))
648  i += 1
649  else:
650  raise Exceptions.DeserilizeException("URLStatus not list type")
651  except (ValueError, TypeError):
652  raise Exceptions.DeserilizeException("URLStatusDeserialize, some field has invalid type")
653  return URLStatusObject
654 
655 
656  # #URLUpdateDeserializeOneElem method, deserializes incoming jsonData to the URLUpdate object
657  # jsonData - incoming json string
658  def URLUpdateDeserializeOneElem(self, elem, i):
659  localObject = dc.EventObjects.URLUpdate(0, "")
660  if "siteId" in elem:
661  localObject.siteId = str(elem["siteId"])
662  else:
663  raise Exceptions.DeserilizeException("URLUpdate.siteId json-field not found in {0} elem".format(i))
664  localObject.url = self.simpleValueExtractor("url", elem, str(), "URLUpdate.url", i)
665  localObject.type = self.simpleValueExtractor("type", elem, int(), "URLUpdate.type", i)
666  if "urlMd5" in elem:
667  localObject.urlMd5 = self.simpleValueExtractor("urlMd5", elem, str(), "URLUpdate.urlMd5", i)
668  if localObject.urlMd5 is None:
669  localObject.fillMD5(localObject.url, localObject.type)
670  localObject.state = self.simpleValueExtractor("state", elem, int(), "URLUpdate.state", i)
671  localObject.status = self.simpleValueExtractor("status", elem, int(), "URLUpdate.status", i)
672  localObject.siteSelect = self.simpleValueExtractor("siteSelect", elem, int(), \
673  "URLUpdate.siteSelect", i)
674  localObject.crawled = self.simpleValueExtractor("crawled", elem, int(), "URLUpdate.crawled", i)
675  localObject.processed = self.simpleValueExtractor("processed", elem, int(), "URLUpdate.processed", i)
676  localObject.contentType = self.simpleValueExtractor("contentType", elem, str(), \
677  "URLUpdate.contentType", i)
678  localObject.requestDelay = self.simpleValueExtractor("requestDelay", elem, int(), \
679  "URLUpdate.requestDelay", i)
680  localObject.processingDelay = self.simpleValueExtractor("processingDelay", elem, int(), \
681  "URLUpdateObject.processingDelay", i)
682  localObject.httpTimeout = self.simpleValueExtractor("httpTimeout", elem, float(), \
683  "URLUpdate.httpTimeout", i)
684  localObject.charset = self.simpleValueExtractor("charset", elem, str(), "URLUpdate.charset", i)
685  localObject.batchId = self.simpleValueExtractor("batchId", elem, int(), "URLUpdate.batchId", i)
686  localObject.errorMask = self.simpleValueExtractor("errorMask", elem, int(), "URLUpdate.errorMask", i)
687  localObject.crawlingTime = self.simpleValueExtractor("crawlingTime", elem, int(), \
688  "URLUpdate.crawlingTime", i)
689  localObject.processingTime = self.simpleValueExtractor("processingTime", elem, int(), \
690  "URLUpdate.processingTime", i)
691  localObject.totalTime = self.simpleValueExtractor("totalTime", elem, int(), "URLUpdate.totalTime", i)
692  localObject.httpCode = self.simpleValueExtractor("httpCode", elem, int(), "URLUpdate.httpCode", i)
693 
694  if "UDate" in elem:
695  localObject.UDate = self.timeExtractor(elem["UDate"])
696  else:
697  raise Exceptions.DeserilizeException("URLNew.UDate json-field not found in {0} elem".format(i))
698  if "CDate" in elem:
699  localObject.CDate = self.timeExtractor(elem["CDate"])
700  else:
701  raise Exceptions.DeserilizeException("URLNew.CDate json-field not found in {0} elem".format(i))
702  localObject.httpMethod = self.simpleValueExtractor("httpMethod", elem, str(), \
703  "URLUpdateObject.httpMethod", i)
704  localObject.size = self.simpleValueExtractor("size", elem, int(), "URLUpdateObject.size", i)
705  localObject.linksI = self.simpleValueExtractor("linksI", elem, int(), "URLUpdateObject.linksI", i)
706  localObject.linksE = self.simpleValueExtractor("linksE", elem, int(), "URLUpdateObject.linksE", i)
707  localObject.freq = self.simpleValueExtractor("freq", elem, int(), "URLUpdateObject.freq", i)
708  localObject.depth = self.simpleValueExtractor("depth", elem, int(), "URLUpdateObject.depth", i)
709  localObject.rawContentMd5 = self.simpleValueExtractor("rawContentMd5", elem, str(), "URLNew.rawContentMd5", i)
710  localObject.parentMd5 = self.simpleValueExtractor("parentMd5", elem, str(), "URLUpdateObject.parentMd5", i)
711  if "lastModified" in elem:
712  localObject.lastModified = self.timeExtractor(elem["lastModified"])
713  else:
714  raise Exceptions.DeserilizeException("URLUpdateObject.lastModified " +
715  "json-field not found in {0} elem".format(i))
716  localObject.eTag = self.simpleValueExtractor("eTag", elem, str(), "URLUpdateObject.eTag", i)
717  localObject.mRate = self.simpleValueExtractor("mRate", elem, int(), "URLUpdateObject.mRate", i)
718  localObject.mRateCounter = self.simpleValueExtractor("mRateCounter", elem, int(), \
719  "URLUpdateObject.mRateCounter", i)
720  localObject.maxURLsFromPage = self.simpleValueExtractor("maxURLsFromPage", elem, int(), \
721  "URLUpdateObject.maxURLsFromPage", i)
722  if "tcDate" in elem:
723  localObject.tcDate = self.timeExtractor(elem["tcDate"])
724  else:
725  raise Exceptions.DeserilizeException("URLUpdateObject.tcDate json-field not found in {0} elem".format(i))
726  localObject.tagsMask = self.simpleValueExtractor("tagsMask", elem, int(), "URLUpdateObject.tagsMask", i)
727  localObject.tagsCount = self.simpleValueExtractor("tagsCount", elem, int(), "URLUpdateObject.tagsCount", i)
728  if "pDate" in elem:
729  localObject.pDate = self.timeExtractor(elem["pDate"])
730  else:
731  raise Exceptions.DeserilizeException("URLUpdateObject.pDate json-field not found in {0} jsonData".format(i))
732  localObject.contentURLMd5 = self.simpleValueExtractor("contentURLMd5", elem, str(),
733  "URLUpdateObject.contentURLMd5", i)
734  localObject.priority = self.simpleValueExtractor("priority", elem, int(), "URLUpdateObject.priority", i)
735  if "urlPut" in elem and elem["urlPut"] is not None:
736  localObject.urlPut = self.URLPutDeserializeOneElement(elem["urlPut"], 0)
737  if "chainId" in elem:
738  localObject.chainId = self.simpleValueExtractor("chainId", elem, int(), "URLUpdateObject.chainId", i)
739  if "classifierMask" in elem:
740  localObject.classifierMask = self.simpleValueExtractor("classifierMask", elem, int(),
741  "URLUpdateObject.classifierMask", i)
742  if "attributes" in elem:
743  localObject.attributes = []
744  for attribute in elem["attributes"]:
745  if type(attribute) is types.DictType and len(attribute) == 1:
746  attrObj = dc.EventObjects.Attribute(localObject.siteId, attribute.keys()[0])
747  attrObj.urlMd5 = localObject.urlMd5
748  attrObj.value = attribute[attribute.keys()[0]]
749  localObject.attributes.append(attrObj)
750  return localObject
751 
752 
753  # #URLUpdateDeserialize method, deserializes incoming jsonData to the [URLUpdate] object
754  # jsonData - incoming json string
755  def URLUpdateDeserialize(self, jsonData):
756  URLUpdateObject = []
757  i = 0
758  try:
759  if type(jsonData) is types.ListType:
760  for elem in jsonData:
761  URLUpdateObject.append(self.URLUpdateDeserializeOneElem(elem, i))
762  i += 1
763  else:
764  raise Exceptions.DeserilizeException("URLUpdate not list type")
765  except (ValueError, TypeError):
766  raise Exceptions.DeserilizeException("URLUpdateDeserialize, some field has invalid type")
767  return URLUpdateObject
768 
769 
770  # #URLFetchDeserialize method, deserializes incoming jsonData to the URLFetch object
771  # jsonData - incoming json string
772  def URLFetchDeserializeOneElem(self, elem, i):
773  localObject = dc.EventObjects.URLFetch()
774  if "sitesList" in elem:
775  if type(elem["sitesList"]) is not types.ListType:
776  raise Exceptions.DeserilizeException("URLFetch.sitesList json-field type not list in {0} elem".format(i))
777  localObject.sitesList = elem["sitesList"]
778  else:
779  raise Exceptions.DeserilizeException("URLFetch.sitesList json-field not found in {0} elem".format(i))
780 
781  if "sitesCriterions" in elem and elem["sitesCriterions"] is not None:
782  if type(elem["sitesCriterions"]) is not types.DictType:
783  raise Exceptions.DeserilizeException(("URLFetch.sitesCriterions json-field type" +
784  " not dict in {0} elem").format(i))
785  if hasattr(localObject.sitesCriterions, '__iter__'):
786  localObject.sitesCriterions.update(elem["sitesCriterions"])
787  else:
788  localObject.sitesCriterions = elem["sitesCriterions"]
789 
790  if "urlsCriterions" in elem and elem["urlsCriterions"] is not None:
791  if type(elem["urlsCriterions"]) is not types.DictType:
792  raise Exceptions.DeserilizeException(("URLFetch.urlsCriterions json-field type" +
793  " not dict in {0} elem").format(i))
794  if hasattr(localObject.urlsCriterions, '__iter__'):
795  localObject.urlsCriterions.update(elem["urlsCriterions"])
796  else:
797  localObject.urlsCriterions = elem["urlsCriterions"]
798 
799  localObject.maxURLs = self.simpleValueExtractor("maxURLs", elem, int(), "URLFetch.maxURLs", i)
800  if localObject.maxURLs is None:
801  localObject.maxURLs = dc.EventObjects.URLFetch.DEFAULT_LIMIT
802 
803  localObject.algorithm = self.simpleValueExtractor("algorithm", elem, int(), "URLFetch.algorithm", i)
804 
805  if localObject.algorithm is None:
806  localObject.algorithm = dc.EventObjects.URLFetch.DEFAULT_ALGORITHM
807 
808  if "isLocking" in elem and elem["isLocking"] is not None:
809  localObject.isLocking = bool(self.simpleValueExtractor("isLocking", elem, int(), "URLFetch.isLocking", i))
810 
811  if "lockIterationTimeout" in elem and elem["lockIterationTimeout"] is not None:
812  localObject.lockIterationTimeout = self.simpleValueExtractor("lockIterationTimeout", elem, int(),
813  "URLFetch.lockIterationTimeout", i)
814  if "siteUpdate" in elem and elem["siteUpdate"] is not None:
815  localObject.siteUpdate = self.siteNewDeserialize(elem["siteUpdate"])
816 
817  if "attributeNames" in elem:
818  if not isinstance(elem["attributeNames"], list):
819  raise Exceptions.DeserilizeException("URLFetch.attributeNames json-field type not list in {0} elem".format(i))
820  localObject.attributeNames = elem["attributeNames"]
821 
822  return localObject
823 
824 
825  # #URLFetchDeserialize method, deserializes incoming jsonData to the list of URLFetch objects
826  # jsonData - incoming json string
827  def URLFetchDeserialize(self, jsonData):
828  URLFetchObject = []
829  i = 0
830  try:
831  if type(jsonData) is types.ListType:
832  for elem in jsonData:
833  URLFetchObject.append(self.URLFetchDeserializeOneElem(elem, i))
834  i += 1
835  else:
836  raise Exceptions.DeserilizeException("URLFetchDeserialize's root element must be an list")
837  except (ValueError, TypeError):
838  raise Exceptions.DeserilizeException("URLFetchDeserialize's some field has invalid type")
839  return URLFetchObject
840 
841 
842  # #URLDeleteDeserialize method, deserializes incoming jsonData to the [URLDelete] object
843  # jsonData - incoming json string
844  def URLDeleteDeserialize(self, jsonData):
845  URLDeleteObject = []
846  i = 0
847  try:
848  if type(jsonData) is types.ListType:
849  for elem in jsonData:
850  URLDeleteObject.append(dc.EventObjects.URLDelete(0, ""))
851  if "siteId" in elem:
852  URLDeleteObject[i].siteId = str(elem["siteId"])
853  else:
854  raise Exceptions.DeserilizeException("URLDelete.siteId json-field not found in {0} elem".format(i))
855  URLDeleteObject[i].url = self.simpleValueExtractor("url", elem, str(), "URLDeleteObject.url", i)
856  if "urlType" in elem:
857  URLDeleteObject[i].urlType = int(elem["urlType"])
858  else:
859  raise Exceptions.DeserilizeException("URLDelete.urlType json-field not found in {0} elem".format(i))
860  if "criterions" in elem and elem["criterions"] is not None:
861  if type(elem["criterions"]) is not types.DictType:
862  raise Exceptions.DeserilizeException(("URLDelete.criterions json-field type" +
863  " not dict in {0} elem").format(i))
864  if hasattr(URLDeleteObject[i].criterions, '__iter__'):
865  URLDeleteObject[i].criterions.update(elem["criterions"])
866  else:
867  URLDeleteObject[i].criterions = elem["criterions"]
868  if "delayedType" in elem and elem["delayedType"] is not None:
869  URLDeleteObject[i].delayedType = int(elem["delayedType"])
870  i += 1
871  else:
872  raise Exceptions.DeserilizeException("URLDelete not list type")
873  except (ValueError, TypeError):
874  raise Exceptions.DeserilizeException("URLDeleteDeserialize, some field has invalid type")
875  return URLDeleteObject
876 
877 
878  # #URLCleanupDeserialize method, deserializes incoming jsonData to the [URLCleanup] object
879  # jsonData - incoming json string
880  def URLCleanupDeserialize(self, jsonData):
881  URLCleanupObject = []
882  i = 0
883  try:
884  if type(jsonData) is types.ListType:
885  for elem in jsonData:
886  URLCleanupObject.append(dc.EventObjects.URLCleanup(0, ""))
887  if "siteId" in elem:
888  URLCleanupObject[i].siteId = str(elem["siteId"])
889  else:
890  raise Exceptions.DeserilizeException("URLCleanupObject.siteId json-field not found in {0} elem".format(i))
891  URLCleanupObject[i].url = self.simpleValueExtractor("url", elem, str(), "URLCleanupObject.url", i)
892  if "urlType" in elem:
893  URLCleanupObject[i].urlType = int(elem["urlType"])
894  else:
895  raise Exceptions.DeserilizeException("URLCleanup.urlType json-field not found in {0} elem".format(i))
896  if "state" in elem:
897  URLCleanupObject[i].state = int(elem["state"])
898  else:
899  raise Exceptions.DeserilizeException("URLCleanup.state json-field not found in {0} elem".format(i))
900  if "status" in elem:
901  URLCleanupObject[i].status = int(elem["status"])
902  else:
903  raise Exceptions.DeserilizeException("URLCleanup.status json-field not found in {0} elem".format(i))
904  if "criterions" in elem and elem["criterions"] is not None:
905  if type(elem["criterions"]) is not types.DictType:
906  raise Exceptions.DeserilizeException(("URLCleanup.criterions json-field type" +
907  " not dict in {0} elem").format(i))
908  if hasattr(URLCleanupObject[i].criterions, '__iter__'):
909  URLCleanupObject[i].criterions.update(elem["criterions"])
910  else:
911  URLCleanupObject[i].criterions = elem["criterions"]
912  if "delayedType" in elem and elem["delayedType"] is not None:
913  URLCleanupObject[i].delayedType = int(elem["delayedType"])
914  i += 1
915  else:
916  raise Exceptions.DeserilizeException("URLDelete not list type")
917  except (ValueError, TypeError):
918  raise Exceptions.DeserilizeException("URLCleanupDeserialize, some field has invalid type")
919  return URLCleanupObject
920 
921 
922  # #URLContentDeserialize method, deserializes incoming jsonData to the [URLContentRequest] object
923  # jsonData - incoming json string
924  def URLContentDeserialize(self, jsonData):
925  URLContentObject = []
926  i = 0
927  try:
928  if type(jsonData) is types.ListType:
929  for elem in jsonData:
930  URLContentObject.append(dc.EventObjects.URLContentRequest(0, ""))
931  if "siteId" in elem:
932  URLContentObject[i].siteId = str(elem["siteId"])
933  else:
934  raise Exceptions.DeserilizeException("URLContentObject.siteId json-field not found in {0} elem".format(i))
935 
936  if "url" in elem:
937  URLContentObject[i].url = str(elem["url"])
938  else:
939  raise Exceptions.DeserilizeException("URLContentObject.url json-field not found in {0} elem".format(i))
940 
941  if "contentTypeMask" in elem:
942  URLContentObject[i].contentTypeMask = int(elem["contentTypeMask"])
943  else:
944  raise Exceptions.DeserilizeException("URLContentObject.contentTypeMask json-field not found in {0} elem".\
945  format(i))
946  if "urlMd5" in elem:
947  URLContentObject[i].urlMd5 = str(elem["urlMd5"])
948  else:
949  URLContentObject[i].fillMD5(URLContentObject[i].url)
950 
951  if "urlFetch" in elem and elem["urlFetch"] is not None:
952  URLContentObject[i].urlFetch = self.URLFetchDeserializeOneElem(elem["urlFetch"], 0)
953 
954  if (dc.EventObjects.URLFetch.CRITERION_WHERE not in URLContentObject[i].urlFetch.sitesCriterions or \
955  URLContentObject[i].urlFetch.sitesCriterions[dc.EventObjects.URLFetch.CRITERION_WHERE] is None) and \
956  URLContentObject[i].siteId not in URLContentObject[i].urlFetch.sitesList:
957  URLContentObject[i].urlFetch.sitesList.append(URLContentObject[i].siteId)
958 
959  if "attributeNames" in elem:
960  if not isinstance(elem["attributeNames"], list):
961  raise Exceptions.DeserilizeException("URLContentRequest.attributeNames json-field type not list in \
962  {0} elem".format(i))
963  URLContentObject[i].attributeNames = elem["attributeNames"]
964 
965  if "dbFieldsList" in elem:
966  # and elem["dbFieldsList"] is not None:
967  # TODO: fixed by bgv cause overrides default constructor's initialization if omitted
968  if type(elem["dbFieldsList"]) is types.ListType:
969  URLContentObject[i].dbFieldsList = elem["dbFieldsList"]
970  # else:
971  # URLContentObject[i].dbFieldsList = None
972  # TODO: fixed by bgv cause overrides default constructor's initialization if omitted
973  # else:
974  # URLContentObject[i].dbFieldsList = None
975  # TODO: fixed by bgv cause overrides default constructor's initialization if omitted
976  i += 1
977  else:
978  raise Exceptions.DeserilizeException("URLContentObject not list type")
979  except (ValueError, TypeError):
980  raise Exceptions.DeserilizeException("URLContentDeserialize, some field has invalid type")
981  return URLContentObject
982 
983 
984  # #SQLCustomDeserialize method, deserializes incoming jsonData to the [SQLCustomRequest] object
985  # jsonData - incoming json string
986  def SQLCustomDeserialize(self, jsonData):
987  rid = None
988  query = None
989  dbName = None
990  if "rid" in jsonData and jsonData["rid"] is not None:
991  rid = self.simpleValueExtractor("rid", jsonData, int(), "SQLCustom.rid")
992  else:
993  raise Exceptions.DeserilizeException("mandatory SQLCustom.rid absent or None")
994  if "query" in jsonData and jsonData["query"] is not None:
995  query = self.simpleValueExtractor("query", jsonData, str(), "SQLCustom.query")
996  else:
997  raise Exceptions.DeserilizeException("mandatory SQLCustom.query absent or None")
998  if "dbName" in jsonData and jsonData["dbName"] is not None:
999  dbName = self.simpleValueExtractor("dbName", jsonData, str(), "SQLCustom.dbName")
1000  else:
1001  raise Exceptions.DeserilizeException("mandatory SQLCustom.query dbName or None")
1002  sqlCustomObject = dbi.EventObjects.CustomRequest(rid, query, dbName)
1003  if "includeFieldsNames" in jsonData and jsonData["includeFieldsNames"] is not None:
1004  sqlCustomObject.includeFieldsNames = self.simpleValueExtractor("includeFieldsNames", jsonData, int(),
1005  "SQLCustom.includeFieldsNames")
1006  return sqlCustomObject
1007 
1008 
1009  # #BatchDeserialize method, deserializes incoming jsonData to the [Batch] object
1010  # jsonData - incoming json string
1011  def BatchDeserialize(self, jsonData):
1012 
1013  batchObject = dc.EventObjects.Batch(0)
1014  batchObject.id = self.simpleValueExtractor("id", jsonData, int(), "Batch.id")
1015  batchObject.crawlerType = self.simpleValueExtractor("crawlerType", jsonData, int(), "Batch.crawlerType")
1016  if "items" in jsonData and jsonData["items"] is not None:
1017  batchObject.items = self.readBatchItems(jsonData["items"])
1018  if "errorMask" in jsonData:
1019  batchObject.errorMask = self.simpleValueExtractor("errorMask", jsonData, int(), "Batch.errorMask")
1020  if "dbMode" in jsonData:
1021  batchObject.dbMode = self.simpleValueExtractor("dbMode", jsonData, int(), "Batch.dbMode")
1022  if "maxIterations" in jsonData:
1023  batchObject.maxIterations = self.simpleValueExtractor("maxIterations", jsonData, int(), "Batch.maxIterations")
1024  if "maxItems" in jsonData:
1025  batchObject.maxItems = self.simpleValueExtractor("maxItems", jsonData, int(), "Batch.maxItems")
1026  if "maxExecutionTime" in jsonData:
1027  batchObject.maxExecutionTime = self.simpleValueExtractor("maxExecutionTime", jsonData, int(),
1028  "Batch.maxExecutionTime")
1029  if "removeUnprocessedItems" in jsonData:
1030  batchObject.removeUnprocessedItems = bool(self.simpleValueExtractor("removeUnprocessedItems", jsonData, int(),
1031  "Batch.removeUnprocessedItems"))
1032 
1033  return batchObject
1034 
1035 
1036  # #URLPurgeDeserialize method, deserializes incoming jsonData to the [URLPurge] object
1037  # jsonData - incoming json string
1038  def URLPurgeDeserialize(self, jsonData):
1039  URLPurgeObject = []
1040  i = 0
1041  try:
1042  if type(jsonData) is types.ListType:
1043  for elem in jsonData:
1044  URLPurgeObject.append(dc.EventObjects.URLPurge(0, ""))
1045  if "siteId" in elem:
1046  if elem["siteId"] is None:
1047  URLPurgeObject[i].siteId = elem["siteId"]
1048  else:
1049  URLPurgeObject[i].siteId = str(elem["siteId"])
1050  else:
1051  raise Exceptions.DeserilizeException("URLPurge.siteId json-field not found in {0} elem".format(i))
1052  URLPurgeObject[i].url = self.simpleValueExtractor("url", elem, str(), "URLPurgeObject.url", i)
1053  if "urlType" in elem:
1054  URLPurgeObject[i].urlType = int(elem["urlType"])
1055  else:
1056  raise Exceptions.DeserilizeException("URLPurge.urlType json-field not found in {0} elem".format(i))
1057  if "criterions" in elem and elem["criterions"] is not None:
1058  if type(elem["criterions"]) is not types.DictType:
1059  raise Exceptions.DeserilizeException(("URLPurge.criterions json-field type" +
1060  " not dict in {0} elem").format(i))
1061  if hasattr(URLPurgeObject[i].criterions, '__iter__'):
1062  URLPurgeObject[i].criterions.update(elem["criterions"])
1063  else:
1064  URLPurgeObject[i].criterions = elem["criterions"]
1065  if "siteLimits" in elem and elem["siteLimits"] is not None:
1066  URLPurgeObject[i].siteLimits = elem["siteLimits"]
1067  i += 1
1068  else:
1069  raise Exceptions.DeserilizeException("URLPurge not list type")
1070  except (ValueError, TypeError):
1071  raise Exceptions.DeserilizeException("URLPurgeDeserialize, some field has invalid type")
1072  return URLPurgeObject
1073 
1074 
1075  # #FieldRecalculatorDeserialize method, deserializes incoming jsonData to the [FieldRecalculatorObj] object
1076  # jsonData - incoming json string
1077  def FieldRecalculatorDeserialize(self, jsonData):
1078  fieldRecalculatorObj = []
1079  i = 0
1080  try:
1081  if type(jsonData) is types.ListType:
1082  for elem in jsonData:
1083  fieldRecalculatorObj.append(dc.EventObjects.FieldRecalculatorObj(""))
1084  fieldRecalculatorObj[i].recalcType = \
1085  self.simpleValueExtractor("recalcType", elem, int(), "FieldRecalculator.recalcType", i)
1086  fieldRecalculatorObj[i].siteId = \
1087  self.simpleValueExtractor("siteId", elem, str(), "FieldRecalculator.siteId", i)
1088  if "criterions" in elem and elem["criterions"] is not None:
1089  if type(elem["criterions"]) is not types.DictType:
1090  raise Exceptions.DeserilizeException("fieldRecalculatorObj.criterions not found in {0} elem".format(i))
1091  if hasattr(fieldRecalculatorObj[i].criterions, '__iter__'):
1092  fieldRecalculatorObj[i].criterions.update(elem["criterions"])
1093  else:
1094  fieldRecalculatorObj[i].criterions = elem["criterions"]
1095  i += 1
1096  else:
1097  raise Exceptions.DeserilizeException("URLPurge not list type")
1098  except (ValueError, TypeError):
1099  raise Exceptions.DeserilizeException("FieldRecalculatorDeserialize, some field has invalid type")
1100  return fieldRecalculatorObj
1101 
1102 
1103  # #URLVerifyDeserialize method, deserializes incoming jsonData to the [URLVerifyDeserialize] object
1104  # jsonData - incoming json string
1105  def URLVerifyDeserialize(self, jsonData):
1106  URLVerifyObj = []
1107  i = 0
1108  try:
1109  if type(jsonData) is types.ListType:
1110  for elem in jsonData:
1111  URLVerifyObj.append(dc.EventObjects.URLVerify("", "", ""))
1112  URLVerifyObj[i].siteId = \
1113  self.simpleValueExtractor("siteId", elem, str(), "URLVErify.siteId", i)
1114  URLVerifyObj[i].urlType = self.simpleValueExtractor("urlType", elem, int(), "URLVErify.urlType", i)
1115  URLVerifyObj[i].url = self.simpleValueExtractor("url", elem, str(), "URLVErify.url", i)
1116  URLVerifyObj[i].dbName = self.simpleValueExtractor("dbName", elem, str(), "URLVErify.dbName", i)
1117  if "criterions" in elem and elem["criterions"] is not None:
1118  if type(elem["criterions"]) is not types.DictType:
1119  raise Exceptions.DeserilizeException("URLVErify.criterions not found in {0} elem".format(i))
1120  if hasattr(URLVerifyObj[i].criterions, '__iter__'):
1121  URLVerifyObj[i].criterions.update(elem["criterions"])
1122  else:
1123  URLVerifyObj[i].criterions = elem["criterions"]
1124  i += 1
1125  else:
1126  raise Exceptions.DeserilizeException("URLVErify not list type")
1127  except (ValueError, TypeError):
1128  raise Exceptions.DeserilizeException("URLVerifyDeserialize, some field has invalid type")
1129  return URLVerifyObj
1130 
1131 
1132  # #URLAgeDeserialize method, deserializes incoming jsonData to the [URLAge] object
1133  # jsonData - incoming json string
1134  def URLAgeDeserialize(self, jsonData):
1135  URLAgeObj = []
1136  i = 0
1137  try:
1138  if type(jsonData) is types.ListType:
1139  for elem in jsonData:
1140  URLAgeObj.append(dc.EventObjects.URLAge(None, None))
1141  if "urlsCriterions" in elem and elem["urlsCriterions"] is not None:
1142  if type(elem["urlsCriterions"]) is not types.DictType:
1143  raise Exceptions.DeserilizeException("URLAgeObj.urlsCriterions not found in {0} elem".format(i))
1144  if hasattr(URLAgeObj[i].urlsCriterions, '__iter__'):
1145  URLAgeObj[i].urlsCriterions.update(elem["urlsCriterions"])
1146  else:
1147  URLAgeObj[i].urlsCriterions = elem["urlsCriterions"]
1148  if "sitesCriterions" in elem and elem["sitesCriterions"] is not None:
1149  if type(elem["sitesCriterions"]) is not types.DictType:
1150  raise Exceptions.DeserilizeException("URLAgeObj.sitesCriterions not found in {0} elem".format(i))
1151  if hasattr(URLAgeObj[i].sitesCriterions, '__iter__'):
1152  URLAgeObj[i].sitesCriterions.update(elem["sitesCriterions"])
1153  else:
1154  URLAgeObj[i].sitesCriterions = elem["sitesCriterions"]
1155  localMaxURLs = self.simpleValueExtractor("maxURLs", elem, int(), "URLAgeObj.maxURLs", i)
1156  if localMaxURLs is not None:
1157  URLAgeObj[i].maxURLs = localMaxURLs
1158  localDelayedType = self.simpleValueExtractor("delayedType", elem, int(), "URLAgeObj.delayedType", i)
1159  if localDelayedType is not None:
1160  URLAgeObj[i].delayedType = localDelayedType
1161  i += 1
1162  else:
1163  raise Exceptions.DeserilizeException("URLAge not list type")
1164  except (ValueError, TypeError):
1165  raise Exceptions.DeserilizeException("URLAgeDeserialize, some field has invalid type")
1166  return URLAgeObj
1167 
1168 
1169  def URLPutDeserializeOneElement(self, jsonData, i):
1170  ret = dc.EventObjects.URLPut(None, None, None)
1171  ret.siteId = self.simpleValueExtractor("siteId", jsonData, str(), "URLPutObj.siteId", i)
1172  ret.urlMd5 = self.simpleValueExtractor("urlMd5", jsonData, str(), "URLPutObj.urlMd5", i)
1173  if "putDict" in jsonData and type(jsonData["putDict"]) is types.DictType:
1174  ret.putDict = jsonData["putDict"]
1175  if "cDate" in ret.putDict and ret.putDict["cDate"] is not None:
1176  ret.putDict["cDate"] = self.timeExtractor(ret.putDict["cDate"])
1177  else:
1178  raise Exceptions.DeserilizeException("Absent mandatory putDict field or it's not dict type")
1179 
1180  if "criterions" in jsonData and jsonData["criterions"] is not None:
1181  if type(jsonData["criterions"]) is not types.DictType:
1182  raise Exceptions.DeserilizeException("URLPutObj.criterions not found in {0} elem".format(i))
1183  if hasattr(ret.criterions, '__iter__'):
1184  ret.criterions.update(jsonData["criterions"])
1185  else:
1186  ret.criterions = jsonData["criterions"]
1187  ret.contentType = self.simpleValueExtractor("contentType", jsonData, int(), "URLPutObj.contentType", i)
1188  if "fileStorageSuffix" in jsonData and jsonData["fileStorageSuffix"] is not None:
1189  ret.fileStorageSuffix = self.simpleValueExtractor("fileStorageSuffix", jsonData, str(),
1190  "URLPutObj.fileStorageSuffix", i)
1191  return ret
1192 
1193 
1194  # #URLPutDeserialize method, deserializes incoming jsonData to the [URLPut] object
1195  # jsonData - incoming json string
1196  def URLPutDeserialize(self, jsonData):
1197  URLPutObjs = []
1198  i = 0
1199  try:
1200  if type(jsonData) is types.ListType:
1201  for elem in jsonData:
1202  URLPutObjs.append(self.URLPutDeserializeOneElement(elem, i))
1203  i += 1
1204  else:
1205  raise Exceptions.DeserilizeException("URLPut not list type")
1206  except (ValueError, TypeError):
1207  raise Exceptions.DeserilizeException("URLPutDeserialize, some field has invalid type")
1208  return URLPutObjs
1209 
1210 
1211  # #URLHistoryDeserialize method, deserializes incoming jsonData to the [URLHistoryRequest] object
1212  # jsonData - incoming json string
1213  def URLHistoryDeserialize(self, jsonData):
1214  URLHistoryRequestObjs = []
1215  i = 0
1216  logger.error(">>> Start URLHistoryDeserialize")
1217  try:
1218  if type(jsonData) is types.ListType:
1219  for elem in jsonData:
1221  ret.siteId = self.simpleValueExtractor("siteId", elem, str(), "URLHistoryRequest.siteId", i)
1222  ret.urlMd5 = self.simpleValueExtractor("urlMd5", elem, str(), "URLHistoryRequest.urlMd5", i)
1223  if "urlCriterions" in elem and elem["urlCriterions"] is not None:
1224  if type(elem["urlCriterions"]) is not types.DictType:
1225  raise Exceptions.DeserilizeException\
1226  ("URLHistoryRequestObjs.urlCriterions not found in {0} elem".format(i))
1227  if hasattr(ret.urlCriterions, '__iter__'):
1228  ret.urlCriterions.update(elem["urlCriterions"])
1229  else:
1230  ret.urlCriterions = elem["urlCriterions"]
1231  if "logCriterions" in elem and elem["logCriterions"] is not None:
1232  if type(elem["logCriterions"]) is not types.DictType:
1233  raise Exceptions.DeserilizeException\
1234  ("URLHistoryRequestObjs.logCriterions not found in {0} elem".format(i))
1235  if hasattr(ret.logCriterions, '__iter__'):
1236  ret.logCriterions.update(elem["logCriterions"])
1237  else:
1238  ret.logCriterions = elem["logCriterions"]
1239  URLHistoryRequestObjs.append(ret)
1240  i += 1
1241  else:
1242  raise Exceptions.DeserilizeException("URLHistoryDeserialize not list type")
1243  except (ValueError, TypeError):
1244  raise Exceptions.DeserilizeException("URLHistoryDeserialize, some field has invalid type")
1245  return URLHistoryRequestObjs
1246 
1247 
1248  # #URLStatsDeserialize method, deserializes incoming jsonData to the [URLStatsRequest] object
1249  # jsonData - incoming json string
1250  def URLStatsDeserialize(self, jsonData):
1251  URLStatsRequestObjs = []
1252  i = 0
1253  logger.error(">>> Start URLStatsRequestObjs")
1254  try:
1255  if type(jsonData) is types.ListType:
1256  for elem in jsonData:
1258  ret.siteId = self.simpleValueExtractor("siteId", elem, str(), "URLHistoryRequest.siteId", i)
1259  ret.urlMd5 = self.simpleValueExtractor("urlMd5", elem, str(), "URLHistoryRequest.urlMd5", i)
1260  if "urlCriterions" in elem and elem["urlCriterions"] is not None:
1261  if type(elem["urlCriterions"]) is not types.DictType:
1262  raise Exceptions.DeserilizeException("URLStatsRequestObjs.urlCriterions not found in {0} elem".format(i))
1263  if hasattr(ret.urlCriterions, '__iter__'):
1264  ret.urlCriterions.update(elem["urlCriterions"])
1265  else:
1266  ret.urlCriterions = elem["urlCriterions"]
1267  if "statsCriterions" in elem and elem["statsCriterions"] is not None:
1268  if type(elem["statsCriterions"]) is not types.DictType:
1269  raise Exceptions.DeserilizeException\
1270  ("URLStatsRequestObjs.statsCriterions not found in {0} elem".format(i))
1271  if hasattr(ret.statsCriterions, '__iter__'):
1272  ret.statsCriterions.update(elem["statsCriterions"])
1273  else:
1274  ret.statsCriterions = elem["statsCriterions"]
1275  URLStatsRequestObjs.append(ret)
1276  i += 1
1277  else:
1278  raise Exceptions.DeserilizeException("URLStatsRequestObjs not list type")
1279  except (ValueError, TypeError):
1280  raise Exceptions.DeserilizeException("URLStatsDeserialize, some field has invalid type")
1281  return URLStatsRequestObjs
1282 
1283 
1284  # #ProxyNewElementDeserialize method, fills data in one ProxyNew elemet
1285  #
1286  # proxyNewObject - filled ProxyNew object
1287  # jsonData - incoming json string
1288  # i - index of current ProxyNew object
1289  def ProxyNewElementDeserialize(self, proxyNewObject, jsonData, i):
1290  proxyNewObject.siteId = self.simpleValueExtractor("siteId", jsonData, str(), "ProxyNew.siteId", i)
1291  proxyNewObject.host = self.simpleValueExtractor("host", jsonData, str(), "ProxyNew.host", i)
1292  if "id" in jsonData and hasattr(proxyNewObject, 'id'):
1293  proxyNewObject.Id = self.simpleValueExtractor("id", jsonData, int(), "ProxyNew.id", i)
1294  if "domains" in jsonData:
1295  proxyNewObject.domains = jsonData["domains"]
1296  if "priority" in jsonData:
1297  proxyNewObject.priority = self.simpleValueExtractor("priority", jsonData, int(), "ProxyNew.priority", i)
1298  if "state" in jsonData:
1299  proxyNewObject.state = self.simpleValueExtractor("state", jsonData, int(), "ProxyNew.state", i)
1300  if "countryCode" in jsonData:
1301  proxyNewObject.countryCode = self.simpleValueExtractor("countryCode", jsonData, str(), "ProxyNew.countryCode", i)
1302  if "countryName" in jsonData:
1303  proxyNewObject.countryName = self.simpleValueExtractor("countryName", jsonData, str(), "ProxyNew.countryName", i)
1304  if "regionCode" in jsonData:
1305  proxyNewObject.regionCode = self.simpleValueExtractor("regionCode", jsonData, int(), "ProxyNew.regionCode", i)
1306  if "regionName" in jsonData:
1307  proxyNewObject.regionName = self.simpleValueExtractor("regionName", jsonData, str(), "ProxyNew.regionName", i)
1308  if "cityName" in jsonData:
1309  proxyNewObject.cityName = self.simpleValueExtractor("cityName", jsonData, str(), "ProxyNew.cityName", i)
1310  if "zipCode" in jsonData:
1311  proxyNewObject.zipCode = self.simpleValueExtractor("zipCode", jsonData, str(), "ProxyNew.zipCode", i)
1312  if "timeZone" in jsonData:
1313  proxyNewObject.timeZone = self.simpleValueExtractor("timeZone", jsonData, str(), "ProxyNew.timeZone", i)
1314  if "latitude" in jsonData:
1315  proxyNewObject.latitude = self.simpleValueExtractor("latitude", jsonData, float(), "ProxyNew.latitude", i)
1316  if "longitude" in jsonData:
1317  proxyNewObject.longitude = self.simpleValueExtractor("longitude", jsonData, float(), "ProxyNew.longitude", i)
1318  if "metroCode" in jsonData:
1319  proxyNewObject.metroCode = self.simpleValueExtractor("metroCode", jsonData, int(), "ProxyNew.metroCode", i)
1320  if "faults" in jsonData:
1321  proxyNewObject.faults = self.simpleValueExtractor("faults", jsonData, int(), "ProxyNew.faults", i)
1322  if "faultsMax" in jsonData:
1323  proxyNewObject.faultsMax = self.simpleValueExtractor("faultsMax", jsonData, int(), "ProxyNew.faultsMax", i)
1324  if "categoryId" in jsonData:
1325  proxyNewObject.categoryId = self.simpleValueExtractor("categoryId", jsonData, int(), "ProxyNew.categoryId", i)
1326  if "limits" in jsonData:
1327  proxyNewObject.limits = jsonData["limits"]
1328  if "description" in jsonData:
1329  proxyNewObject.description = self.simpleValueExtractor("description", jsonData, str(), "ProxyNew.description", i)
1330  if "cDate" in jsonData:
1331  proxyNewObject.cDate = self.timeExtractor(jsonData["cDate"])
1332  if "uDate" in jsonData:
1333  proxyNewObject.uDate = self.timeExtractor(jsonData["uDate"])
1334 
1335 
1336  # #ProxyNewDeserialize method, serializes all ProxyNew elemets from incoming json
1337  #
1338  # jsonData - incoming json string
1339  def ProxyNewDeserialize(self, jsonData):
1340  ProxyNewObjects = []
1341  i = 0
1342  try:
1343  if type(jsonData) is types.ListType:
1344  for elem in jsonData:
1345  ProxyNewObjects.append(dc.EventObjects.Proxy("", ""))
1346  self.ProxyNewElementDeserialize(ProxyNewObjects[i], elem, i)
1347  i += 1
1348  else:
1349  raise Exceptions.DeserilizeException("ProxyNew not list type")
1350  except (ValueError, TypeError):
1351  raise Exceptions.DeserilizeException("ProxyNewDeserialize, some field has invalid type")
1352  return ProxyNewObjects
1353 
1354 
1355  # #ProxyUpdateDeserialize method, serializes all ProxyUpdate elemets from incoming json
1356  #
1357  # jsonData - incoming json string
1358  def ProxyUpdateDeserialize(self, jsonData, siteId='', host=''):
1359  ProxyUpdateObjects = []
1360  i = 0
1361  try:
1362  if type(jsonData) is types.ListType:
1363  for elem in jsonData:
1364  ProxyUpdateObjects.append(dc.EventObjects.ProxyUpdate(siteId, host))
1365  self.ProxyNewElementDeserialize(ProxyUpdateObjects[i], elem, i)
1366  i += 1
1367  else:
1368  raise Exceptions.DeserilizeException("ProxyUpdate not list type")
1369  except (ValueError, TypeError):
1370  raise Exceptions.DeserilizeException("ProxyUpdateDeserialize, some field has invalid type")
1371  return ProxyUpdateObjects
1372 
1373 
1374  # #ProxyUpdateDeserialize method, serializes all ProxyDelete elemets from incoming json
1375  #
1376  # jsonData - incoming json string
1377  def ProxyDeleteDeserialize(self, jsonData):
1378  ProxyDeleteObjects = []
1379  i = 0
1380  try:
1381  if type(jsonData) is types.ListType:
1382  for elem in jsonData:
1383  ProxyDeleteObjects.append(dc.EventObjects.ProxyDelete())
1384  if "siteId" in elem:
1385  ProxyDeleteObjects[i].siteId = self.simpleValueExtractor("siteId", elem, str(), "ProxyDelete.siteId", i)
1386  if "host" in elem:
1387  ProxyDeleteObjects[i].host = self.simpleValueExtractor("host", elem, str(), "ProxyDelete.host", i)
1388  if "criterions" in elem and elem["criterions"] is not None:
1389  if type(elem["criterions"]) is not types.DictType:
1390  raise Exceptions.DeserilizeException(("ProxyDelete.criterions json-field type" +
1391  " not dict in {0} elem").format(i))
1392  if hasattr(ProxyDeleteObjects[i].criterions, '__iter__'):
1393  ProxyDeleteObjects[i].criterions.update(elem["criterions"])
1394  else:
1395  ProxyDeleteObjects[i].criterions = elem["criterions"]
1396  i += 1
1397  else:
1398  raise Exceptions.DeserilizeException("ProxyDelete not list type")
1399  except (ValueError, TypeError):
1400  raise Exceptions.DeserilizeException("ProxyDeleteDeserialize, some field has invalid type")
1401  return ProxyDeleteObjects
1402 
1403 
1404  # #ProxyUpdateDeserialize method, serializes all ProxyStatus elemets from incoming json
1405  #
1406  # jsonData - incoming json string
1407  def ProxyStatusDeserialize(self, jsonData):
1408  ProxyStausObjects = []
1409  i = 0
1410  try:
1411  if type(jsonData) is types.ListType:
1412  for elem in jsonData:
1413  ProxyStausObjects.append(dc.EventObjects.ProxyStatus())
1414  if "siteId" in elem:
1415  ProxyStausObjects[i].siteId = self.simpleValueExtractor("siteId", elem, str(), "ProxyStaus.siteId", i)
1416  if "host" in elem:
1417  ProxyStausObjects[i].host = self.simpleValueExtractor("host", elem, str(), "ProxyStaus.host", i)
1418  if "criterions" in elem and elem["criterions"] is not None:
1419  if type(elem["criterions"]) is not types.DictType:
1420  raise Exceptions.DeserilizeException(("ProxyDelete.criterions json-field type" +
1421  " not dict in {0} elem").format(i))
1422  if hasattr(ProxyStausObjects[i].criterions, '__iter__'):
1423  ProxyStausObjects[i].criterions.update(elem["criterions"])
1424  else:
1425  ProxyStausObjects[i].criterions = elem["criterions"]
1426  i += 1
1427  else:
1428  raise Exceptions.DeserilizeException("ProxyStatus not list type")
1429  except (ValueError, TypeError):
1430  raise Exceptions.DeserilizeException("ProxyStatusDeserialize, some field has invalid type")
1431  return ProxyStausObjects
1432 
1433 
1434  # #ProxyUpdateDeserialize method, serializes all ProxyFind elemets from incoming json
1435  #
1436  # jsonData - incoming json string
1437  # return list of ProxyFind objects
1438  def ProxyFindDeserialize(self, jsonData):
1439  ProxyFindObjects = []
1440  i = 0
1441  try:
1442  if type(jsonData) is types.ListType:
1443  for elem in jsonData:
1444  ProxyFindObjects.append(dc.EventObjects.ProxyFind())
1445  if "siteId" in elem:
1446  ProxyFindObjects[i].siteId = self.simpleValueExtractor("siteId", elem, str(), "ProxyFind.siteId", i)
1447  if "siteCriterions" in elem and elem["siteCriterions"] is not None:
1448  if type(elem["siteCriterions"]) is not types.DictType:
1449  raise Exceptions.DeserilizeException(("ProxyFind.siteCriterions json-field type" +
1450  " not dict in {0} elem").format(i))
1451  if hasattr(ProxyFindObjects[i].siteCriterions, '__iter__'):
1452  ProxyFindObjects[i].siteCriterions.update(elem["siteCriterions"])
1453  else:
1454  ProxyFindObjects[i].siteCriterions = elem["siteCriterions"]
1455  if "criterions" in elem and elem["criterions"] is not None:
1456  if type(elem["criterions"]) is not types.DictType:
1457  raise Exceptions.DeserilizeException(("ProxyFind.criterions json-field type" +
1458  " not dict in {0} elem").format(i))
1459  if hasattr(ProxyFindObjects[i].criterions, '__iter__'):
1460  ProxyFindObjects[i].criterions.update(elem["criterions"])
1461  else:
1462  ProxyFindObjects[i].criterions = elem["criterions"]
1463  i += 1
1464  else:
1465  raise Exceptions.DeserilizeException("ProxyStatus not list type")
1466  except (ValueError, TypeError):
1467  raise Exceptions.DeserilizeException("ProxyFindDeserialize, some field has invalid type")
1468  return ProxyFindObjects
1469 
1470 
1471  # #AttrSetDeserialize method, serializes all ProxyFind elemets from incoming json
1472  #
1473  # jsonData - incoming json string
1474  # return list of Attributes objects
1475  def AttrSetDeserialize(self, jsonData):
1476  attributesObjects = []
1477  i = 0
1478  try:
1479  if type(jsonData) is types.ListType:
1480  for elem in jsonData:
1481  attributesObjects.append(dc.EventObjects.Attribute('', ''))
1482  attributesObjects[i].siteId = self.simpleValueExtractor("siteId", elem, str(), "Attribute.siteId")
1483  attributesObjects[i].name = self.simpleValueExtractor("name", elem, str(), "Attribute.name")
1484  if "urlMd5" in elem:
1485  attributesObjects[i].urlMd5 = self.simpleValueExtractor("urlMd5", elem, str(), "Attribute.urlMd5")
1486  if "value" in elem:
1487  attributesObjects[i].value = self.simpleValueExtractor("value", elem, str(), "Attribute.value")
1488  i += 1
1489  else:
1490  raise Exceptions.DeserilizeException("AttrSet not list type")
1491  except (ValueError, TypeError):
1492  raise Exceptions.DeserilizeException("AttrSetDeserialize, some field has invalid type")
1493  return attributesObjects
1494 
1495 
1496  # #AttrUpdateDeserialize method, serializes all AttributeUpdate elemets from incoming json
1497  #
1498  # jsonData - incoming json string
1499  # return list of AttributeUpdate objects
1500  def AttrUpdateDeserialize(self, jsonData):
1501  attributesUpdateObjects = []
1502  i = 0
1503  try:
1504  if type(jsonData) is types.ListType:
1505  for elem in jsonData:
1506  attributesUpdateObjects.append(dc.EventObjects.AttributeUpdate('', ''))
1507  attributesUpdateObjects[i].siteId = self.simpleValueExtractor("siteId", elem, str(), "AttributeUpdate.siteId")
1508  attributesUpdateObjects[i].name = self.simpleValueExtractor("name", elem, str(), "AttributeUpdate.name")
1509  if "urlMd5" in elem:
1510  attributesUpdateObjects[i].urlMd5 = self.simpleValueExtractor("urlMd5", elem, str(),
1511  "AttributeUpdate.urlMd5")
1512  if "value" in elem:
1513  attributesUpdateObjects[i].value = self.simpleValueExtractor("value", elem, str(), "AttributeUpdate.value")
1514  i += 1
1515  else:
1516  raise Exceptions.DeserilizeException("AttrUpdate not list type")
1517  except (ValueError, TypeError):
1518  raise Exceptions.DeserilizeException("AttrUpdateDeserialize, some field has invalid type")
1519  return attributesUpdateObjects
1520 
1521 
1522  # #AttrDeleteDeserialize method, serializes all AttributeDelete elemets from incoming json
1523  #
1524  # jsonData - incoming json string
1525  # return list of AttributeDelete objects
1526  def AttrDeleteDeserialize(self, jsonData):
1527  attributesDeleteObjects = []
1528  i = 0
1529  try:
1530  if type(jsonData) is types.ListType:
1531  for elem in jsonData:
1532  attributesDeleteObjects.append(dc.EventObjects.AttributeDelete(''))
1533  attributesDeleteObjects[i].siteId = self.simpleValueExtractor("siteId", elem, str(), "AttributeDelete.siteId")
1534  if "name" in elem:
1535  attributesDeleteObjects[i].name = self.simpleValueExtractor("name", elem, str(), "AttributeDelete.name")
1536  if "criterions" in elem and elem["criterions"] is not None:
1537  if type(elem["criterions"]) is not types.DictType:
1538  raise Exceptions.DeserilizeException(("ProxyFind.criterions json-field type" +
1539  " not dict in {0} elem").format(i))
1540  if hasattr(attributesDeleteObjects[i].criterions, '__iter__'):
1541  attributesDeleteObjects[i].criterions.update(elem["criterions"])
1542  else:
1543  attributesDeleteObjects[i].criterions = elem["criterions"]
1544  i += 1
1545  else:
1546  raise Exceptions.DeserilizeException("AttrDelete not list type")
1547  except (ValueError, TypeError):
1548  raise Exceptions.DeserilizeException("AttrDeleteDeserialize, some field has invalid type")
1549  return attributesDeleteObjects
1550 
1551 
1552  # #AttrFetchDeserialize method, serializes all AttributeFetch elemets from incoming json
1553  #
1554  # jsonData - incoming json string
1555  # return list of AttributeFetch objects
1556  def AttrFetchDeserialize(self, jsonData):
1557  attributesFetchObjects = []
1558  i = 0
1559  try:
1560  if type(jsonData) is types.ListType:
1561  for elem in jsonData:
1562  attributesFetchObjects.append(dc.EventObjects.AttributeFetch(''))
1563  attributesFetchObjects[i].siteId = self.simpleValueExtractor("siteId", elem, str(), "AttrFetch.siteId")
1564  if "name" in elem:
1565  attributesFetchObjects[i].name = self.simpleValueExtractor("name", elem, str(), "AttrFetch.name")
1566  if "criterions" in elem and elem["criterions"] is not None:
1567  if type(elem["criterions"]) is not types.DictType:
1568  raise Exceptions.DeserilizeException(("ProxyFind.criterions json-field type" +
1569  " not dict in {0} elem").format(i))
1570  if hasattr(attributesFetchObjects[i].criterions, '__iter__'):
1571  attributesFetchObjects[i].criterions.update(elem["criterions"])
1572  else:
1573  attributesFetchObjects[i].criterions = elem["criterions"]
1574  i += 1
1575  else:
1576  raise Exceptions.DeserilizeException("AttrFetch not list type")
1577  except (ValueError, TypeError):
1578  raise Exceptions.DeserilizeException("AttrFetchDeserialize, some field has invalid type")
1579  return attributesFetchObjects
def fillSiteURLs(self, jsonData, siteId, userId=None, urlNormalizeMask=UrlNormalizator.NORM_NONE)
def simpleValueExtractor(self, fieldName, jsonData, _type, excepStr, i=None)
def ProxyUpdateDeserialize(self, jsonData, siteId='', host='')
def ProxyNewElementDeserialize(self, proxyNewObject, jsonData, i)
def URLNewElementDeserialize(self, urlNewObject, jsonData, i)