2 Created on Apr 11, 2014 6 @link: http://hierarchical-cluster-engine.com/ 7 @copyright: Copyright © 2013-2014 IOIX Ukraine 8 @license: http://hierarchical-cluster-engine.com/license/ 37 if timeField
is not None:
38 if type(timeField)
in types.StringTypes:
41 elif type(timeField)
is types.DictType:
50 for value
in properties.values():
51 if not isinstance(value, str)
and not isinstance(value, unicode)
and not isinstance(value, list):
52 raise Exceptions.DeserilizeException(
"Properties field values can be string or list[] type")
53 elif isinstance(value, list)
and len(value) != 3:
54 raise Exceptions.DeserilizeException(
"Properties field values with list[] type must be len() == 3")
63 if filters
is not None:
65 for localFilter
in filters:
66 if "siteId" in localFilter
and localFilter[
"siteId"]
is not None:
67 siteId = str(localFilter[
"siteId"])
70 if "pattern" not in localFilter
or localFilter[
"pattern"]
is None:
71 raise Exceptions.DeserilizeException(
".filters pattern not found or None")
72 if "type" not in localFilter:
76 if "subject" in localFilter
and localFilter[
"subject"]
is not None:
77 siteFilter.subject = str(localFilter[
"subject"])
78 if "opCode" in localFilter
and localFilter[
"opCode"]
is not None:
79 siteFilter.opCode = int(localFilter[
"opCode"])
80 if "stage" in localFilter
and localFilter[
"stage"]
is not None:
81 siteFilter.stage = int(localFilter[
"stage"])
82 if "action" in localFilter
and localFilter[
"action"]
is not None:
83 siteFilter.action = int(localFilter[
"action"])
84 if "mode" in localFilter
and localFilter[
"mode"]
is not None:
85 siteFilter.mode = int(localFilter[
"mode"])
86 if "state" in localFilter
and localFilter[
"state"]
is not None:
87 siteFilter.state = int(localFilter[
"state"])
88 if "uDate" in localFilter:
90 if "groupId" in localFilter:
91 siteFilter.groupId = int(localFilter[
"groupId"])
92 ret.append(siteFilter)
104 if fieldName
in jsonData:
105 ret = jsonData[fieldName]
107 ret =
type(_type)(ret)
110 raise Exceptions.DeserilizeException(
"%s json-field not found" % excepStr)
112 raise Exceptions.DeserilizeException(
"%s json-field not found in [%s] elem" % (excepStr, str(i)))
120 def fillSiteURLs(self, jsonData, siteId, userId=None, urlNormalizeMask=UrlNormalizator.NORM_NONE):
122 if "url" in jsonData
and jsonData[
"url"]
is not None:
126 if hasattr(ret, key)
and jsonData[key]
is not None and key !=
"url":
127 setattr(ret, key, jsonData[key])
129 logger.error(
">>> Some URL-dict elem doesn't content url field")
139 if hasattr(jsonData,
'__iter__'):
140 for elem
in jsonData:
143 if "urlObj" in elem
and elem[
"urlObj"]
is not None:
147 raise Exceptions.DeserilizeException(
"BatchItem.urlObj json-field not found in {0} elem".
format(i))
150 if "properties" in elem
and elem[
"properties"]
is not None and type(elem[
"properties"])
is types.DictType:
151 localBatchItem.properties = elem[
"properties"]
152 if "siteObj" in elem
and elem[
"siteObj"]
is not None:
156 ret.append(localBatchItem)
167 if "id" in jsonData
and jsonData[
"id"]
is not None:
169 siteObject.id = str(jsonData[
"id"])
171 if "urls" in jsonData
and hasattr(jsonData[
"urls"],
'____')
and \
172 len(jsonData[
"urls"]) > 0
and jsonData[
"urls"][0]
is not None:
175 raise Exceptions.DeserilizeException(
"Site.id json-field not found and Site.urls[0] no found")
176 if "uDate" in jsonData:
179 raise Exceptions.DeserilizeException(
"Site.uDate json-field not found")
180 if "tcDate" in jsonData:
183 raise Exceptions.DeserilizeException(
"Site.tcDate json-field not found")
184 if "cDate" in jsonData:
187 raise Exceptions.DeserilizeException(
"Site.cDate json-field not found")
188 siteObject.resources = self.
simpleValueExtractor(
"resources", jsonData, int(),
"Site.resources")
189 siteObject.iterations = self.
simpleValueExtractor(
"iterations", jsonData, int(),
"Site.iterations")
190 siteObject.description = self.
simpleValueExtractor(
"description", jsonData, str(),
"Site.description")
193 if "urls" in jsonData:
194 if jsonData[
"urls"]
is None:
195 siteObject.urls =
None 196 elif not isinstance(jsonData[
"urls"], list):
197 raise Exceptions.DeserilizeException(
"Site.urls not list type")
199 for jsonUrl
in jsonData[
"urls"]:
201 if isinstance(jsonUrl, dict):
202 localRet = self.
fillSiteURLs(jsonUrl, siteObject.id, siteObject.userId,
203 int(jsonUrl[
"_normalizeMask"])
if "_normalizeMask" in jsonUrl
else \
204 UrlNormalizator.NORM_NONE)
205 elif type(jsonUrl)
in types.StringTypes:
207 normalizeMask=int(jsonData[
"_urlNormalizeMask"])\
208 if "_urlNormalizeMask" in jsonData
else UrlNormalizator.NORM_NONE)
209 localRet.userId = siteObject.userId
210 if localRet
is not None:
211 if siteObject.urls
is None:
213 siteObject.urls.append(localRet)
215 raise Exceptions.DeserilizeException(
"Site.urls json-field not found")
217 if "filters" in jsonData:
218 if jsonData[
"filters"]
is not None and type(jsonData[
"filters"])
is not types.ListType:
219 raise Exceptions.DeserilizeException(
"Site.filters not list type")
220 siteObject.filters = self.
checkFilters(jsonData[
"filters"], siteObject.id)
222 raise Exceptions.DeserilizeException(
"Site.filters json-field not found")
223 if "properties" in jsonData:
224 if jsonData[
"properties"]
is not None and type(jsonData[
"properties"])
is not types.DictType
and \
225 type(jsonData[
"properties"])
is not types.ListType:
226 raise Exceptions.DeserilizeException(
"Site.properties not dict/list type")
227 if type(jsonData[
"properties"])
is not types.ListType:
229 siteObject.properties = jsonData[
"properties"]
231 raise Exceptions.DeserilizeException(
"Site.properties json-field not found")
235 siteObject.maxResources = self.
simpleValueExtractor(
"maxResources", jsonData, int(),
"Site.maxResources")
236 siteObject.maxErrors = self.
simpleValueExtractor(
"maxErrors", jsonData, int(),
"Site.maxErrors")
237 siteObject.maxResourceSize = self.
simpleValueExtractor(
"maxResourceSize", jsonData, int(),
"Site.maxResourceSize")
238 siteObject.requestDelay = self.
simpleValueExtractor(
"requestDelay", jsonData, int(),
"Site.requestDelay")
239 siteObject.httpTimeout = self.
simpleValueExtractor(
"httpTimeout", jsonData, float(),
"Site.httpTimeout")
240 siteObject.errorMask = self.
simpleValueExtractor(
"errorMask", jsonData, int(),
"Site.errorMask")
244 siteObject.processingDelay = self.
simpleValueExtractor(
"processingDelay", jsonData, int(),
"Site.processingDelay")
247 siteObject.avgSpeedCounter = self.
simpleValueExtractor(
"avgSpeedCounter", jsonData, int(),
"Site.avgSpeedCounter")
250 siteObject.maxURLsFromPage = self.
simpleValueExtractor(
"maxURLsFromPage", jsonData, int(),
"Site.maxURLsFromPage")
251 siteObject.recrawlPeriod = self.
simpleValueExtractor(
"recrawlPeriod", jsonData, int(),
"Site.recrawlPeriod")
252 siteObject.maxURLsFromPage = self.
simpleValueExtractor(
"maxURLsFromPage", jsonData, int(),
"Site.maxURLsFromPage")
253 if "recrawlDate" in jsonData:
254 siteObject.recrawlDate = self.
timeExtractor(jsonData[
"recrawlDate"])
256 raise Exceptions.DeserilizeException(
"Site.recrawlDate json-field not found")
257 siteObject.collectedURLs = self.
simpleValueExtractor(
"collectedURLs", jsonData, int(),
"Site.collectedURLs")
258 siteObject.fetchType = self.
simpleValueExtractor(
"fetchType", jsonData, int(),
"Site.fetchType")
259 if "newURLs" in jsonData:
261 if "deletedURLs" in jsonData:
262 siteObject.deletedURLs = self.
simpleValueExtractor(
"deletedURLs", jsonData, int(),
"Site.deletedURLs")
263 if "moveURLs" in jsonData
and jsonData[
"moveURLs"]
is not None:
265 siteObject.moveURLs = bool(siteObject.moveURLs)
266 if "tcDateProcess" in jsonData:
267 siteObject.tcDateProcess = self.
timeExtractor(jsonData[
"tcDateProcess"])
268 if "categoryId" in jsonData:
269 siteObject.categoryId = self.
simpleValueExtractor(
"categoryId", jsonData, int(),
"Site.categoryId")
270 except Exceptions.DeserilizeException, e:
272 except (ValueError, TypeError):
273 raise Exceptions.DeserilizeException(
"siteNewDeserialize, some field has invalid type")
282 siteUpdateObject.updateType = self.
simpleValueExtractor(
"updateType", jsonData, int(),
"SiteUpdate.updateType")
284 if "uDate" in jsonData:
285 siteUpdateObject.uDate = self.
timeExtractor(jsonData[
"uDate"])
287 raise Exceptions.DeserilizeException(
"SiteUpdate.uDate json-field not found")
288 if "tcDate" in jsonData:
289 siteUpdateObject.tcDate = self.
timeExtractor(jsonData[
"tcDate"])
291 raise Exceptions.DeserilizeException(
"SiteUpdate.tcDate json-field not found")
292 if "cDate" in jsonData:
293 siteUpdateObject.cDate = self.
timeExtractor(jsonData[
"cDate"])
295 raise Exceptions.DeserilizeException(
"SiteUpdate.cDate json-field not found")
296 siteUpdateObject.resources = self.
simpleValueExtractor(
"resources", jsonData, int(),
"SiteUpdate.resources")
297 siteUpdateObject.iterations = self.
simpleValueExtractor(
"iterations", jsonData, int(),
"SiteUpdate.iterations")
298 siteUpdateObject.description = self.
simpleValueExtractor(
"description", jsonData, str(),
"SiteUpdate.description")
299 siteUpdateObject.userId = self.
simpleValueExtractor(
"userId", jsonData, int(),
"SiteUpdate.userId")
301 if "urls" in jsonData:
302 if jsonData[
"urls"]
is None:
303 siteUpdateObject.urls =
None 304 elif not isinstance(jsonData[
"urls"], list):
305 raise Exceptions.DeserilizeException(
"SiteUpdate.urls not list type")
307 siteUpdateObject.urls = []
308 for jsonUrl
in jsonData[
"urls"]:
310 if isinstance(jsonUrl, dict):
311 localRet = self.
fillSiteURLs(jsonUrl, siteUpdateObject.id, siteUpdateObject.userId,
312 int(jsonUrl[
"_normalizeMask"])
if "_normalizeMask" in jsonUrl
else \
313 UrlNormalizator.NORM_NONE)
314 elif isinstance(jsonUrl, basestring):
316 normalizeMask=int(jsonData[
"_urlNormalizeMask"])\
317 if "_urlNormalizeMask" in jsonData
else UrlNormalizator.NORM_NONE)
318 localRet.userId = siteUpdateObject.userId
319 if localRet
is not None:
320 siteUpdateObject.urls.append(localRet)
322 if siteUpdateObject.urls
is not None:
323 for url
in siteUpdateObject.urls:
324 url.status = dc.EventObjects.URL.STATUS_PROCESSED
326 raise Exceptions.DeserilizeException(
"SiteUpdate.urls json-field not found")
328 if "filters" in jsonData:
329 if jsonData[
"filters"]
is not None and type(jsonData[
"filters"])
is not types.ListType:
330 raise Exceptions.DeserilizeException(
"SiteUpdate.filters not list type")
331 siteUpdateObject.filters = self.
checkFilters(jsonData[
"filters"], siteUpdateObject.id)
333 raise Exceptions.DeserilizeException(
"SiteUpdate.filters json-field not found")
334 if "properties" in jsonData:
335 if jsonData[
"properties"]
is not None and type(jsonData[
"properties"])
is not types.DictType
and \
336 type(jsonData[
"properties"])
is not types.ListType:
337 raise Exceptions.DeserilizeException(
"SiteUpdate.properties not dict type")
338 if type(jsonData[
"properties"])
is types.DictType:
340 siteUpdateObject.properties = jsonData[
"properties"]
342 raise Exceptions.DeserilizeException(
"SiteUpdate.properties json-field not found")
343 siteUpdateObject.state = self.
simpleValueExtractor(
"state", jsonData, int(),
"SiteUpdate.state")
344 siteUpdateObject.priority = self.
simpleValueExtractor(
"priority", jsonData, int(),
"SiteUpdate.priority")
345 siteUpdateObject.maxURLs = self.
simpleValueExtractor(
"maxURLs", jsonData, int(),
"SiteUpdate.maxURLs")
347 "SiteUpdate.maxResources")
348 siteUpdateObject.maxErrors = self.
simpleValueExtractor(
"maxErrors", jsonData, int(),
"SiteUpdate.maxErrors")
349 siteUpdateObject.maxResourceSize = self.
simpleValueExtractor(
"maxResourceSize", jsonData, int(), \
350 "SiteUpdate.maxResourceSize")
352 "SiteUpdate.requestDelay")
353 siteUpdateObject.httpTimeout = self.
simpleValueExtractor(
"httpTimeout", jsonData, float(),
"SiteUpdate.httpTimeout")
354 siteUpdateObject.errorMask = self.
simpleValueExtractor(
"errorMask", jsonData, int(),
"SiteUpdate.errorMask")
355 siteUpdateObject.errors = self.
simpleValueExtractor(
"errors", jsonData, int(),
"SiteUpdate.errors")
356 siteUpdateObject.urlType = self.
simpleValueExtractor(
"urlType", jsonData, int(),
"SiteUpdate.urlType")
357 siteUpdateObject.contents = self.
simpleValueExtractor(
"contents", jsonData, int(),
"SiteUpdate.contents")
358 siteUpdateObject.processingDelay = self.
simpleValueExtractor(
"processingDelay", jsonData, int(), \
359 "SiteUpdate.processingDelay")
361 siteUpdateObject.avgSpeed = self.
simpleValueExtractor(
"avgSpeed", jsonData, int(),
"SiteUpdate.avgSpeed")
362 siteUpdateObject.avgSpeedCounter = self.
simpleValueExtractor(
"avgSpeedCounter", jsonData, int(), \
363 "SiteUpdate.avgSpeedCounter")
365 "siteUpdateObject.recrawlPeriod")
366 siteUpdateObject.maxURLsFromPage = self.
simpleValueExtractor(
"maxURLsFromPage", jsonData, int(), \
367 "siteUpdateObject.maxURLsFromPage")
368 if "recrawlDate" in jsonData:
369 siteUpdateObject.recrawlDate = self.
timeExtractor(jsonData[
"recrawlDate"])
371 raise Exceptions.DeserilizeException(
"siteUpdateObject.recrawlDate json-field not found")
373 if "criterions" in jsonData
and jsonData[
"criterions"]
is not None:
374 if type(jsonData[
"criterions"])
is not types.DictType:
375 raise Exceptions.DeserilizeException(
"URLCleanup.criterions json-field type")
376 if hasattr(siteUpdateObject.criterions,
'__iter__'):
377 siteUpdateObject.criterions.update(jsonData[
"criterions"])
379 siteUpdateObject.criterions = jsonData[
"criterions"]
381 jsonData, int(),
"siteUpdateObject.collectedURLs")
382 siteUpdateObject.fetchType = self.
simpleValueExtractor(
"fetchType", jsonData, int(),
"siteUpdateObject.fetchType")
383 if "newURLs" in jsonData:
384 siteUpdateObject.newURLs = self.
simpleValueExtractor(
"newURLs", jsonData, int(),
"siteUpdateObject.newURLs")
385 if "deletedURLs" in jsonData:
387 "siteUpdateObject.deletedURLs")
388 if "tcDateProcess" in jsonData:
389 siteUpdateObject.tcDateProcess = self.
timeExtractor(jsonData[
"tcDateProcess"])
390 if "categoryId" in jsonData:
392 "siteUpdateObject.categoryId")
393 except Exceptions.DeserilizeException:
395 except (ValueError, TypeError):
396 raise Exceptions.DeserilizeException(
"siteUpdateDeserialize, some field has invalid type")
397 return siteUpdateObject
405 if "url" in jsonData:
408 siteFindObject.url = jsonData[
"url"]
409 if "criterions" in jsonData
and jsonData[
"criterions"]
is not None:
410 if type(jsonData[
"criterions"])
is not types.DictType:
411 raise Exceptions.DeserilizeException(
"URLFetch.sitesCriterions json-field type not dict")
412 siteFindObject.criterions.update(jsonData[
"criterions"])
413 if "excludeList" in jsonData
and type(jsonData[
"excludeList"])
is types.ListType:
414 siteFindObject.excludeList = jsonData[
"excludeList"]
415 except (ValueError, TypeError):
416 raise Exceptions.DeserilizeException(
"siteFindDeserialize, some field has invalid type")
417 return siteFindObject
426 siteStatusObject.id = jsonData[
"id"]
428 raise Exceptions.DeserilizeException(
"SiteStatus.id json-field not found")
429 if "deleteTaskId" in jsonData:
430 siteStatusObject.deleteTaskId = jsonData[
"deleteTaskId"]
432 raise Exceptions.DeserilizeException(
"SiteStatus.deleteTaskId json-field not found")
433 if "excludeList" in jsonData
and type(jsonData[
"excludeList"])
is types.ListType:
434 siteStatusObject.excludeList = jsonData[
"excludeList"]
435 except (ValueError, TypeError):
436 raise Exceptions.DeserilizeException(
"siteStatusDeserialize, some field has invalid type")
437 return siteStatusObject
443 if type(jsonData)
is types.ListType:
444 siteDeleteObject = []
445 jsonDataElems = jsonData
448 jsonDataElems = [jsonData]
450 for elem
in jsonDataElems:
454 localSiteDeleteObject.id = elem[
"id"]
455 if "taskType" in elem:
456 localSiteDeleteObject.taskType = int(elem[
"taskType"])
458 raise Exceptions.DeserilizeException(
"SiteDelete.taskType json-field not found")
459 if "delayedType" in elem
and elem[
"delayedType"]
is not None:
460 localSiteDeleteObject.delayedType = int(elem[
"delayedType"])
461 if "criterions" in elem:
462 if type(elem[
"criterions"])
is not types.DictType:
463 raise Exceptions.DeserilizeException(
"SiteDelete.criterions not dict type")
464 if hasattr(localSiteDeleteObject.criterions,
'__iter__'):
465 localSiteDeleteObject.criterions.update(elem[
"criterions"])
467 localSiteDeleteObject.criterions = elem[
"criterions"]
469 raise Exceptions.DeserilizeException(
"siteDeleteDeserialize, some field has invalid type")
470 if type(jsonData)
is types.ListType:
471 siteDeleteObject.append(localSiteDeleteObject)
473 siteDeleteObject = localSiteDeleteObject
474 return siteDeleteObject
480 if type(jsonData)
is types.ListType:
481 siteCleanupObject = []
482 jsonDataElems = jsonData
485 jsonDataElems = [jsonData]
487 for elem
in jsonDataElems:
491 localSiteCleanupObject.id = elem[
"id"]
493 raise Exceptions.DeserilizeException(
"SiteCleanup.id json-field not found")
494 if "taskType" in elem:
495 localSiteCleanupObject.taskType = int(elem[
"taskType"])
497 raise Exceptions.DeserilizeException(
"SiteCleanup.taskType json-field not found")
498 if "delayedType" in elem
and elem[
"delayedType"]
is not None:
499 localSiteCleanupObject.delayedType = int(elem[
"delayedType"])
500 if "moveURLs" in elem
and elem[
"moveURLs"]
is not None:
501 localSiteCleanupObject.moveURLs = self.
simpleValueExtractor(
"moveURLs", elem, int(),
"Site.moveURLs")
502 localSiteCleanupObject.moveURLs = bool(localSiteCleanupObject.moveURLs)
503 if "saveRootURLs" in elem
and elem[
"saveRootURLs"]
is not None:
506 localSiteCleanupObject.saveRootUrls = bool(localSiteCleanupObject.saveRootUrls)
507 if "state" in elem
and elem[
"state"]
is not None:
508 localSiteCleanupObject.state = int(elem[
"state"])
509 if "historyCleanUp" in elem
and elem[
"historyCleanUp"]
is not None:
510 localSiteCleanupObject.historyCleanUp = int(elem[
"historyCleanUp"])
511 except (ValueError, TypeError):
512 raise Exceptions.DeserilizeException(
"siteCleanupDeserialize, some field has invalid type")
513 if type(jsonData)
is types.ListType:
514 siteCleanupObject.append(localSiteCleanupObject)
516 siteCleanupObject = localSiteCleanupObject
517 return siteCleanupObject
524 if "url" in jsonData:
525 urlNewObject.url = str(jsonData[
"url"])
527 raise Exceptions.DeserilizeException(
"URLNew.url json-field not found in {0} jsonData".
format(i))
531 urlNewObject.siteSelect = self.
simpleValueExtractor(
"siteSelect", jsonData, int(),
"URLNew.siteSelect", i)
532 urlNewObject.crawled = self.
simpleValueExtractor(
"crawled", jsonData, int(),
"URLNew.crawled", i)
533 urlNewObject.processed = self.
simpleValueExtractor(
"processed", jsonData, int(),
"URLNew.processed", i)
535 if urlNewObject.urlMd5
is None:
536 urlNewObject.fillMD5 = hashlib.md5(urlNewObject.url).hexdigest()
537 urlNewObject.contentType = self.
simpleValueExtractor(
"contentType", jsonData, str(),
"URLNew.contentType", i)
539 "URLNew.requestDelay", i)
541 "URLNew.processingDelay", i)
542 urlNewObject.httpTimeout = self.
simpleValueExtractor(
"httpTimeout", jsonData, float(),
"URLNew.httpTimeout", i)
543 urlNewObject.charset = self.
simpleValueExtractor(
"charset", jsonData, str(),
"URLNew.charset", i)
544 urlNewObject.batchId = self.
simpleValueExtractor(
"batchId", jsonData, int(),
"URLNew.batchId", i)
545 urlNewObject.errorMask = self.
simpleValueExtractor(
"errorMask", jsonData, int(),
"URLNew.errorMask", i)
547 "URLNew.crawlingTime", i)
549 "URLNew.processingTime", i)
550 urlNewObject.totalTime = self.
simpleValueExtractor(
"totalTime", jsonData, int(),
"URLNew.totalTime", i)
551 urlNewObject.httpCode = self.
simpleValueExtractor(
"httpCode", jsonData, int(),
"URLNew.httpCode", i)
552 if "UDate" in jsonData:
555 raise Exceptions.DeserilizeException(
"URLNew.UDate json-field not found in {0} jsonData".
format(i))
556 if "CDate" in jsonData:
559 raise Exceptions.DeserilizeException(
"URLNew.CDate json-field not found in {0} jsonData".
format(i))
560 urlNewObject.httpMethod = self.
simpleValueExtractor(
"httpMethod", jsonData, str(),
"URLNew.httpMethod", i)
567 "URLNew.rawContentMd5", i)
568 urlNewObject.parentMd5 = self.
simpleValueExtractor(
"parentMd5", jsonData, str(),
"URLNew.parentMd5", i)
569 if "lastModified" in jsonData:
570 urlNewObject.lastModified = self.
timeExtractor(jsonData[
"lastModified"])
572 raise Exceptions.DeserilizeException(
"URLNew.lastModified json-field not found in {0} jsonData".
format(i))
575 urlNewObject.mRateCounter = self.
simpleValueExtractor(
"mRateCounter", jsonData, int(),
"URLNew.mRateCounter", i)
577 "URLNew.maxURLsFromPage", i)
578 if "tcDate" in jsonData:
581 raise Exceptions.DeserilizeException(
"URLNew.tcDate json-field not found in {0} jsonData".
format(i))
582 urlNewObject.tagsMask = self.
simpleValueExtractor(
"tagsMask", jsonData, int(),
"URLNew.tagsMask", i)
583 urlNewObject.tagsCount = self.
simpleValueExtractor(
"tagsCount", jsonData, int(),
"URLNew.tagsCount", i)
584 if "pDate" in jsonData:
587 raise Exceptions.DeserilizeException(
"URLNew.pDate json-field not found in {0} jsonData".
format(i))
588 urlNewObject.contentURLMd5 = self.
simpleValueExtractor(
"contentURLMd5", jsonData, str(),
"URLNew.contentURLMd5", i)
589 urlNewObject.priority = self.
simpleValueExtractor(
"priority", jsonData, int(),
"URLNew.priority", i)
590 if "urlUpdate" in jsonData
and jsonData[
"urlUpdate"]
is not None:
592 if "urlPut" in jsonData
and jsonData[
"urlPut"]
is not None:
594 if "chainId" in jsonData:
595 urlNewObject.chainId = self.
simpleValueExtractor(
"chainId", jsonData, int(),
"URLNew.chainId", i)
596 if "classifierMask" in jsonData:
598 "URLNew.classifierMask", i)
599 if "attributes" in jsonData:
600 urlNewObject.attributes = []
601 for attribute
in jsonData[
"attributes"]:
602 if type(attribute)
is types.DictType
and len(attribute) == 1:
604 attrObj.urlMd5 = urlNewObject.urlMd5
605 attrObj.value = attribute[attribute.keys()[0]]
606 urlNewObject.attributes.append(attrObj)
615 if type(jsonData)
is types.ListType:
616 for elem
in jsonData:
621 raise Exceptions.DeserilizeException(
"URLNew not list type")
622 except (ValueError, TypeError)
as excp:
623 raise Exceptions.DeserilizeException(
"URLNewDeserialize, some field has invalid type; =" + str(excp))
633 if type(jsonData)
is types.ListType:
634 for elem
in jsonData:
637 URLStatusObject[i].siteId = str(elem[
"siteId"])
639 raise Exceptions.DeserilizeException(
"URLStatus.siteId json-field not found in {0} elem".
format(i))
641 URLStatusObject[i].url = str(elem[
"url"])
643 raise Exceptions.DeserilizeException(
"URLStatus.url json-field not found in {0} elem".
format(i))
644 if "urlType" in elem:
645 URLStatusObject[i].urlType = int(elem[
"urlType"])
647 raise Exceptions.DeserilizeException(
"URLStatus.urlType json-field not found in {0} elem".
format(i))
650 raise Exceptions.DeserilizeException(
"URLStatus not list type")
651 except (ValueError, TypeError):
652 raise Exceptions.DeserilizeException(
"URLStatusDeserialize, some field has invalid type")
653 return URLStatusObject
661 localObject.siteId = str(elem[
"siteId"])
663 raise Exceptions.DeserilizeException(
"URLUpdate.siteId json-field not found in {0} elem".
format(i))
668 if localObject.urlMd5
is None:
669 localObject.fillMD5(localObject.url, localObject.type)
673 "URLUpdate.siteSelect", i)
675 localObject.processed = self.
simpleValueExtractor(
"processed", elem, int(),
"URLUpdate.processed", i)
677 "URLUpdate.contentType", i)
679 "URLUpdate.requestDelay", i)
681 "URLUpdateObject.processingDelay", i)
683 "URLUpdate.httpTimeout", i)
686 localObject.errorMask = self.
simpleValueExtractor(
"errorMask", elem, int(),
"URLUpdate.errorMask", i)
688 "URLUpdate.crawlingTime", i)
690 "URLUpdate.processingTime", i)
691 localObject.totalTime = self.
simpleValueExtractor(
"totalTime", elem, int(),
"URLUpdate.totalTime", i)
692 localObject.httpCode = self.
simpleValueExtractor(
"httpCode", elem, int(),
"URLUpdate.httpCode", i)
697 raise Exceptions.DeserilizeException(
"URLNew.UDate json-field not found in {0} elem".
format(i))
701 raise Exceptions.DeserilizeException(
"URLNew.CDate json-field not found in {0} elem".
format(i))
703 "URLUpdateObject.httpMethod", i)
705 localObject.linksI = self.
simpleValueExtractor(
"linksI", elem, int(),
"URLUpdateObject.linksI", i)
706 localObject.linksE = self.
simpleValueExtractor(
"linksE", elem, int(),
"URLUpdateObject.linksE", i)
709 localObject.rawContentMd5 = self.
simpleValueExtractor(
"rawContentMd5", elem, str(),
"URLNew.rawContentMd5", i)
710 localObject.parentMd5 = self.
simpleValueExtractor(
"parentMd5", elem, str(),
"URLUpdateObject.parentMd5", i)
711 if "lastModified" in elem:
712 localObject.lastModified = self.
timeExtractor(elem[
"lastModified"])
714 raise Exceptions.DeserilizeException(
"URLUpdateObject.lastModified " +
715 "json-field not found in {0} elem".
format(i))
719 "URLUpdateObject.mRateCounter", i)
721 "URLUpdateObject.maxURLsFromPage", i)
725 raise Exceptions.DeserilizeException(
"URLUpdateObject.tcDate json-field not found in {0} elem".
format(i))
726 localObject.tagsMask = self.
simpleValueExtractor(
"tagsMask", elem, int(),
"URLUpdateObject.tagsMask", i)
727 localObject.tagsCount = self.
simpleValueExtractor(
"tagsCount", elem, int(),
"URLUpdateObject.tagsCount", i)
731 raise Exceptions.DeserilizeException(
"URLUpdateObject.pDate json-field not found in {0} jsonData".
format(i))
733 "URLUpdateObject.contentURLMd5", i)
734 localObject.priority = self.
simpleValueExtractor(
"priority", elem, int(),
"URLUpdateObject.priority", i)
735 if "urlPut" in elem
and elem[
"urlPut"]
is not None:
737 if "chainId" in elem:
738 localObject.chainId = self.
simpleValueExtractor(
"chainId", elem, int(),
"URLUpdateObject.chainId", i)
739 if "classifierMask" in elem:
741 "URLUpdateObject.classifierMask", i)
742 if "attributes" in elem:
743 localObject.attributes = []
744 for attribute
in elem[
"attributes"]:
745 if type(attribute)
is types.DictType
and len(attribute) == 1:
747 attrObj.urlMd5 = localObject.urlMd5
748 attrObj.value = attribute[attribute.keys()[0]]
749 localObject.attributes.append(attrObj)
759 if type(jsonData)
is types.ListType:
760 for elem
in jsonData:
764 raise Exceptions.DeserilizeException(
"URLUpdate not list type")
765 except (ValueError, TypeError):
766 raise Exceptions.DeserilizeException(
"URLUpdateDeserialize, some field has invalid type")
767 return URLUpdateObject
774 if "sitesList" in elem:
775 if type(elem[
"sitesList"])
is not types.ListType:
776 raise Exceptions.DeserilizeException(
"URLFetch.sitesList json-field type not list in {0} elem".
format(i))
777 localObject.sitesList = elem[
"sitesList"]
779 raise Exceptions.DeserilizeException(
"URLFetch.sitesList json-field not found in {0} elem".
format(i))
781 if "sitesCriterions" in elem
and elem[
"sitesCriterions"]
is not None:
782 if type(elem[
"sitesCriterions"])
is not types.DictType:
783 raise Exceptions.DeserilizeException((
"URLFetch.sitesCriterions json-field type" +
784 " not dict in {0} elem").
format(i))
785 if hasattr(localObject.sitesCriterions,
'__iter__'):
786 localObject.sitesCriterions.update(elem[
"sitesCriterions"])
788 localObject.sitesCriterions = elem[
"sitesCriterions"]
790 if "urlsCriterions" in elem
and elem[
"urlsCriterions"]
is not None:
791 if type(elem[
"urlsCriterions"])
is not types.DictType:
792 raise Exceptions.DeserilizeException((
"URLFetch.urlsCriterions json-field type" +
793 " not dict in {0} elem").
format(i))
794 if hasattr(localObject.urlsCriterions,
'__iter__'):
795 localObject.urlsCriterions.update(elem[
"urlsCriterions"])
797 localObject.urlsCriterions = elem[
"urlsCriterions"]
800 if localObject.maxURLs
is None:
801 localObject.maxURLs = dc.EventObjects.URLFetch.DEFAULT_LIMIT
803 localObject.algorithm = self.
simpleValueExtractor(
"algorithm", elem, int(),
"URLFetch.algorithm", i)
805 if localObject.algorithm
is None:
806 localObject.algorithm = dc.EventObjects.URLFetch.DEFAULT_ALGORITHM
808 if "isLocking" in elem
and elem[
"isLocking"]
is not None:
809 localObject.isLocking = bool(self.
simpleValueExtractor(
"isLocking", elem, int(),
"URLFetch.isLocking", i))
811 if "lockIterationTimeout" in elem
and elem[
"lockIterationTimeout"]
is not None:
812 localObject.lockIterationTimeout = self.
simpleValueExtractor(
"lockIterationTimeout", elem, int(),
813 "URLFetch.lockIterationTimeout", i)
814 if "siteUpdate" in elem
and elem[
"siteUpdate"]
is not None:
817 if "attributeNames" in elem:
818 if not isinstance(elem[
"attributeNames"], list):
819 raise Exceptions.DeserilizeException(
"URLFetch.attributeNames json-field type not list in {0} elem".
format(i))
820 localObject.attributeNames = elem[
"attributeNames"]
831 if type(jsonData)
is types.ListType:
832 for elem
in jsonData:
836 raise Exceptions.DeserilizeException(
"URLFetchDeserialize's root element must be an list")
837 except (ValueError, TypeError):
838 raise Exceptions.DeserilizeException(
"URLFetchDeserialize's some field has invalid type")
839 return URLFetchObject
848 if type(jsonData)
is types.ListType:
849 for elem
in jsonData:
852 URLDeleteObject[i].siteId = str(elem[
"siteId"])
854 raise Exceptions.DeserilizeException(
"URLDelete.siteId json-field not found in {0} elem".
format(i))
855 URLDeleteObject[i].url = self.
simpleValueExtractor(
"url", elem, str(),
"URLDeleteObject.url", i)
856 if "urlType" in elem:
857 URLDeleteObject[i].urlType = int(elem[
"urlType"])
859 raise Exceptions.DeserilizeException(
"URLDelete.urlType json-field not found in {0} elem".
format(i))
860 if "criterions" in elem
and elem[
"criterions"]
is not None:
861 if type(elem[
"criterions"])
is not types.DictType:
862 raise Exceptions.DeserilizeException((
"URLDelete.criterions json-field type" +
863 " not dict in {0} elem").
format(i))
864 if hasattr(URLDeleteObject[i].criterions,
'__iter__'):
865 URLDeleteObject[i].criterions.update(elem[
"criterions"])
867 URLDeleteObject[i].criterions = elem[
"criterions"]
868 if "delayedType" in elem
and elem[
"delayedType"]
is not None:
869 URLDeleteObject[i].delayedType = int(elem[
"delayedType"])
872 raise Exceptions.DeserilizeException(
"URLDelete not list type")
873 except (ValueError, TypeError):
874 raise Exceptions.DeserilizeException(
"URLDeleteDeserialize, some field has invalid type")
875 return URLDeleteObject
881 URLCleanupObject = []
884 if type(jsonData)
is types.ListType:
885 for elem
in jsonData:
888 URLCleanupObject[i].siteId = str(elem[
"siteId"])
890 raise Exceptions.DeserilizeException(
"URLCleanupObject.siteId json-field not found in {0} elem".
format(i))
891 URLCleanupObject[i].url = self.
simpleValueExtractor(
"url", elem, str(),
"URLCleanupObject.url", i)
892 if "urlType" in elem:
893 URLCleanupObject[i].urlType = int(elem[
"urlType"])
895 raise Exceptions.DeserilizeException(
"URLCleanup.urlType json-field not found in {0} elem".
format(i))
897 URLCleanupObject[i].state = int(elem[
"state"])
899 raise Exceptions.DeserilizeException(
"URLCleanup.state json-field not found in {0} elem".
format(i))
901 URLCleanupObject[i].status = int(elem[
"status"])
903 raise Exceptions.DeserilizeException(
"URLCleanup.status json-field not found in {0} elem".
format(i))
904 if "criterions" in elem
and elem[
"criterions"]
is not None:
905 if type(elem[
"criterions"])
is not types.DictType:
906 raise Exceptions.DeserilizeException((
"URLCleanup.criterions json-field type" +
907 " not dict in {0} elem").
format(i))
908 if hasattr(URLCleanupObject[i].criterions,
'__iter__'):
909 URLCleanupObject[i].criterions.update(elem[
"criterions"])
911 URLCleanupObject[i].criterions = elem[
"criterions"]
912 if "delayedType" in elem
and elem[
"delayedType"]
is not None:
913 URLCleanupObject[i].delayedType = int(elem[
"delayedType"])
916 raise Exceptions.DeserilizeException(
"URLDelete not list type")
917 except (ValueError, TypeError):
918 raise Exceptions.DeserilizeException(
"URLCleanupDeserialize, some field has invalid type")
919 return URLCleanupObject
925 URLContentObject = []
928 if type(jsonData)
is types.ListType:
929 for elem
in jsonData:
932 URLContentObject[i].siteId = str(elem[
"siteId"])
934 raise Exceptions.DeserilizeException(
"URLContentObject.siteId json-field not found in {0} elem".
format(i))
937 URLContentObject[i].url = str(elem[
"url"])
939 raise Exceptions.DeserilizeException(
"URLContentObject.url json-field not found in {0} elem".
format(i))
941 if "contentTypeMask" in elem:
942 URLContentObject[i].contentTypeMask = int(elem[
"contentTypeMask"])
944 raise Exceptions.DeserilizeException(
"URLContentObject.contentTypeMask json-field not found in {0} elem".\
947 URLContentObject[i].urlMd5 = str(elem[
"urlMd5"])
949 URLContentObject[i].fillMD5(URLContentObject[i].url)
951 if "urlFetch" in elem
and elem[
"urlFetch"]
is not None:
954 if (dc.EventObjects.URLFetch.CRITERION_WHERE
not in URLContentObject[i].urlFetch.sitesCriterions
or \
955 URLContentObject[i].urlFetch.sitesCriterions[dc.EventObjects.URLFetch.CRITERION_WHERE]
is None)
and \
956 URLContentObject[i].siteId
not in URLContentObject[i].urlFetch.sitesList:
957 URLContentObject[i].urlFetch.sitesList.append(URLContentObject[i].siteId)
959 if "attributeNames" in elem:
960 if not isinstance(elem[
"attributeNames"], list):
961 raise Exceptions.DeserilizeException(
"URLContentRequest.attributeNames json-field type not list in \ 963 URLContentObject[i].attributeNames = elem[
"attributeNames"]
965 if "dbFieldsList" in elem:
968 if type(elem[
"dbFieldsList"])
is types.ListType:
969 URLContentObject[i].dbFieldsList = elem[
"dbFieldsList"]
978 raise Exceptions.DeserilizeException(
"URLContentObject not list type")
979 except (ValueError, TypeError):
980 raise Exceptions.DeserilizeException(
"URLContentDeserialize, some field has invalid type")
981 return URLContentObject
990 if "rid" in jsonData
and jsonData[
"rid"]
is not None:
993 raise Exceptions.DeserilizeException(
"mandatory SQLCustom.rid absent or None")
994 if "query" in jsonData
and jsonData[
"query"]
is not None:
997 raise Exceptions.DeserilizeException(
"mandatory SQLCustom.query absent or None")
998 if "dbName" in jsonData
and jsonData[
"dbName"]
is not None:
1001 raise Exceptions.DeserilizeException(
"mandatory SQLCustom.query dbName or None")
1003 if "includeFieldsNames" in jsonData
and jsonData[
"includeFieldsNames"]
is not None:
1004 sqlCustomObject.includeFieldsNames = self.
simpleValueExtractor(
"includeFieldsNames", jsonData, int(),
1005 "SQLCustom.includeFieldsNames")
1006 return sqlCustomObject
1015 batchObject.crawlerType = self.
simpleValueExtractor(
"crawlerType", jsonData, int(),
"Batch.crawlerType")
1016 if "items" in jsonData
and jsonData[
"items"]
is not None:
1018 if "errorMask" in jsonData:
1019 batchObject.errorMask = self.
simpleValueExtractor(
"errorMask", jsonData, int(),
"Batch.errorMask")
1020 if "dbMode" in jsonData:
1022 if "maxIterations" in jsonData:
1023 batchObject.maxIterations = self.
simpleValueExtractor(
"maxIterations", jsonData, int(),
"Batch.maxIterations")
1024 if "maxItems" in jsonData:
1025 batchObject.maxItems = self.
simpleValueExtractor(
"maxItems", jsonData, int(),
"Batch.maxItems")
1026 if "maxExecutionTime" in jsonData:
1028 "Batch.maxExecutionTime")
1029 if "removeUnprocessedItems" in jsonData:
1030 batchObject.removeUnprocessedItems = bool(self.
simpleValueExtractor(
"removeUnprocessedItems", jsonData, int(),
1031 "Batch.removeUnprocessedItems"))
1042 if type(jsonData)
is types.ListType:
1043 for elem
in jsonData:
1045 if "siteId" in elem:
1046 if elem[
"siteId"]
is None:
1047 URLPurgeObject[i].siteId = elem[
"siteId"]
1049 URLPurgeObject[i].siteId = str(elem[
"siteId"])
1051 raise Exceptions.DeserilizeException(
"URLPurge.siteId json-field not found in {0} elem".
format(i))
1053 if "urlType" in elem:
1054 URLPurgeObject[i].urlType = int(elem[
"urlType"])
1056 raise Exceptions.DeserilizeException(
"URLPurge.urlType json-field not found in {0} elem".
format(i))
1057 if "criterions" in elem
and elem[
"criterions"]
is not None:
1058 if type(elem[
"criterions"])
is not types.DictType:
1059 raise Exceptions.DeserilizeException((
"URLPurge.criterions json-field type" +
1060 " not dict in {0} elem").
format(i))
1061 if hasattr(URLPurgeObject[i].criterions,
'__iter__'):
1062 URLPurgeObject[i].criterions.update(elem[
"criterions"])
1064 URLPurgeObject[i].criterions = elem[
"criterions"]
1065 if "siteLimits" in elem
and elem[
"siteLimits"]
is not None:
1066 URLPurgeObject[i].siteLimits = elem[
"siteLimits"]
1069 raise Exceptions.DeserilizeException(
"URLPurge not list type")
1070 except (ValueError, TypeError):
1071 raise Exceptions.DeserilizeException(
"URLPurgeDeserialize, some field has invalid type")
1072 return URLPurgeObject
1078 fieldRecalculatorObj = []
1081 if type(jsonData)
is types.ListType:
1082 for elem
in jsonData:
1084 fieldRecalculatorObj[i].recalcType = \
1086 fieldRecalculatorObj[i].siteId = \
1088 if "criterions" in elem
and elem[
"criterions"]
is not None:
1089 if type(elem[
"criterions"])
is not types.DictType:
1090 raise Exceptions.DeserilizeException(
"fieldRecalculatorObj.criterions not found in {0} elem".
format(i))
1091 if hasattr(fieldRecalculatorObj[i].criterions,
'__iter__'):
1092 fieldRecalculatorObj[i].criterions.update(elem[
"criterions"])
1094 fieldRecalculatorObj[i].criterions = elem[
"criterions"]
1097 raise Exceptions.DeserilizeException(
"URLPurge not list type")
1098 except (ValueError, TypeError):
1099 raise Exceptions.DeserilizeException(
"FieldRecalculatorDeserialize, some field has invalid type")
1100 return fieldRecalculatorObj
1109 if type(jsonData)
is types.ListType:
1110 for elem
in jsonData:
1112 URLVerifyObj[i].siteId = \
1114 URLVerifyObj[i].urlType = self.
simpleValueExtractor(
"urlType", elem, int(),
"URLVErify.urlType", i)
1116 URLVerifyObj[i].dbName = self.
simpleValueExtractor(
"dbName", elem, str(),
"URLVErify.dbName", i)
1117 if "criterions" in elem
and elem[
"criterions"]
is not None:
1118 if type(elem[
"criterions"])
is not types.DictType:
1119 raise Exceptions.DeserilizeException(
"URLVErify.criterions not found in {0} elem".
format(i))
1120 if hasattr(URLVerifyObj[i].criterions,
'__iter__'):
1121 URLVerifyObj[i].criterions.update(elem[
"criterions"])
1123 URLVerifyObj[i].criterions = elem[
"criterions"]
1126 raise Exceptions.DeserilizeException(
"URLVErify not list type")
1127 except (ValueError, TypeError):
1128 raise Exceptions.DeserilizeException(
"URLVerifyDeserialize, some field has invalid type")
1138 if type(jsonData)
is types.ListType:
1139 for elem
in jsonData:
1141 if "urlsCriterions" in elem
and elem[
"urlsCriterions"]
is not None:
1142 if type(elem[
"urlsCriterions"])
is not types.DictType:
1143 raise Exceptions.DeserilizeException(
"URLAgeObj.urlsCriterions not found in {0} elem".
format(i))
1144 if hasattr(URLAgeObj[i].urlsCriterions,
'__iter__'):
1145 URLAgeObj[i].urlsCriterions.update(elem[
"urlsCriterions"])
1147 URLAgeObj[i].urlsCriterions = elem[
"urlsCriterions"]
1148 if "sitesCriterions" in elem
and elem[
"sitesCriterions"]
is not None:
1149 if type(elem[
"sitesCriterions"])
is not types.DictType:
1150 raise Exceptions.DeserilizeException(
"URLAgeObj.sitesCriterions not found in {0} elem".
format(i))
1151 if hasattr(URLAgeObj[i].sitesCriterions,
'__iter__'):
1152 URLAgeObj[i].sitesCriterions.update(elem[
"sitesCriterions"])
1154 URLAgeObj[i].sitesCriterions = elem[
"sitesCriterions"]
1156 if localMaxURLs
is not None:
1157 URLAgeObj[i].maxURLs = localMaxURLs
1158 localDelayedType = self.
simpleValueExtractor(
"delayedType", elem, int(),
"URLAgeObj.delayedType", i)
1159 if localDelayedType
is not None:
1160 URLAgeObj[i].delayedType = localDelayedType
1163 raise Exceptions.DeserilizeException(
"URLAge not list type")
1164 except (ValueError, TypeError):
1165 raise Exceptions.DeserilizeException(
"URLAgeDeserialize, some field has invalid type")
1173 if "putDict" in jsonData
and type(jsonData[
"putDict"])
is types.DictType:
1174 ret.putDict = jsonData[
"putDict"]
1175 if "cDate" in ret.putDict
and ret.putDict[
"cDate"]
is not None:
1176 ret.putDict[
"cDate"] = self.
timeExtractor(ret.putDict[
"cDate"])
1178 raise Exceptions.DeserilizeException(
"Absent mandatory putDict field or it's not dict type")
1180 if "criterions" in jsonData
and jsonData[
"criterions"]
is not None:
1181 if type(jsonData[
"criterions"])
is not types.DictType:
1182 raise Exceptions.DeserilizeException(
"URLPutObj.criterions not found in {0} elem".
format(i))
1183 if hasattr(ret.criterions,
'__iter__'):
1184 ret.criterions.update(jsonData[
"criterions"])
1186 ret.criterions = jsonData[
"criterions"]
1187 ret.contentType = self.
simpleValueExtractor(
"contentType", jsonData, int(),
"URLPutObj.contentType", i)
1188 if "fileStorageSuffix" in jsonData
and jsonData[
"fileStorageSuffix"]
is not None:
1190 "URLPutObj.fileStorageSuffix", i)
1200 if type(jsonData)
is types.ListType:
1201 for elem
in jsonData:
1205 raise Exceptions.DeserilizeException(
"URLPut not list type")
1206 except (ValueError, TypeError):
1207 raise Exceptions.DeserilizeException(
"URLPutDeserialize, some field has invalid type")
1214 URLHistoryRequestObjs = []
1216 logger.error(
">>> Start URLHistoryDeserialize")
1218 if type(jsonData)
is types.ListType:
1219 for elem
in jsonData:
1223 if "urlCriterions" in elem
and elem[
"urlCriterions"]
is not None:
1224 if type(elem[
"urlCriterions"])
is not types.DictType:
1225 raise Exceptions.DeserilizeException\
1226 (
"URLHistoryRequestObjs.urlCriterions not found in {0} elem".
format(i))
1227 if hasattr(ret.urlCriterions,
'__iter__'):
1228 ret.urlCriterions.update(elem[
"urlCriterions"])
1230 ret.urlCriterions = elem[
"urlCriterions"]
1231 if "logCriterions" in elem
and elem[
"logCriterions"]
is not None:
1232 if type(elem[
"logCriterions"])
is not types.DictType:
1233 raise Exceptions.DeserilizeException\
1234 (
"URLHistoryRequestObjs.logCriterions not found in {0} elem".
format(i))
1235 if hasattr(ret.logCriterions,
'__iter__'):
1236 ret.logCriterions.update(elem[
"logCriterions"])
1238 ret.logCriterions = elem[
"logCriterions"]
1239 URLHistoryRequestObjs.append(ret)
1242 raise Exceptions.DeserilizeException(
"URLHistoryDeserialize not list type")
1243 except (ValueError, TypeError):
1244 raise Exceptions.DeserilizeException(
"URLHistoryDeserialize, some field has invalid type")
1245 return URLHistoryRequestObjs
1251 URLStatsRequestObjs = []
1253 logger.error(
">>> Start URLStatsRequestObjs")
1255 if type(jsonData)
is types.ListType:
1256 for elem
in jsonData:
1260 if "urlCriterions" in elem
and elem[
"urlCriterions"]
is not None:
1261 if type(elem[
"urlCriterions"])
is not types.DictType:
1262 raise Exceptions.DeserilizeException(
"URLStatsRequestObjs.urlCriterions not found in {0} elem".
format(i))
1263 if hasattr(ret.urlCriterions,
'__iter__'):
1264 ret.urlCriterions.update(elem[
"urlCriterions"])
1266 ret.urlCriterions = elem[
"urlCriterions"]
1267 if "statsCriterions" in elem
and elem[
"statsCriterions"]
is not None:
1268 if type(elem[
"statsCriterions"])
is not types.DictType:
1269 raise Exceptions.DeserilizeException\
1270 (
"URLStatsRequestObjs.statsCriterions not found in {0} elem".
format(i))
1271 if hasattr(ret.statsCriterions,
'__iter__'):
1272 ret.statsCriterions.update(elem[
"statsCriterions"])
1274 ret.statsCriterions = elem[
"statsCriterions"]
1275 URLStatsRequestObjs.append(ret)
1278 raise Exceptions.DeserilizeException(
"URLStatsRequestObjs not list type")
1279 except (ValueError, TypeError):
1280 raise Exceptions.DeserilizeException(
"URLStatsDeserialize, some field has invalid type")
1281 return URLStatsRequestObjs
1290 proxyNewObject.siteId = self.
simpleValueExtractor(
"siteId", jsonData, str(),
"ProxyNew.siteId", i)
1292 if "id" in jsonData
and hasattr(proxyNewObject,
'id'):
1294 if "domains" in jsonData:
1295 proxyNewObject.domains = jsonData[
"domains"]
1296 if "priority" in jsonData:
1297 proxyNewObject.priority = self.
simpleValueExtractor(
"priority", jsonData, int(),
"ProxyNew.priority", i)
1298 if "state" in jsonData:
1299 proxyNewObject.state = self.
simpleValueExtractor(
"state", jsonData, int(),
"ProxyNew.state", i)
1300 if "countryCode" in jsonData:
1301 proxyNewObject.countryCode = self.
simpleValueExtractor(
"countryCode", jsonData, str(),
"ProxyNew.countryCode", i)
1302 if "countryName" in jsonData:
1303 proxyNewObject.countryName = self.
simpleValueExtractor(
"countryName", jsonData, str(),
"ProxyNew.countryName", i)
1304 if "regionCode" in jsonData:
1305 proxyNewObject.regionCode = self.
simpleValueExtractor(
"regionCode", jsonData, int(),
"ProxyNew.regionCode", i)
1306 if "regionName" in jsonData:
1307 proxyNewObject.regionName = self.
simpleValueExtractor(
"regionName", jsonData, str(),
"ProxyNew.regionName", i)
1308 if "cityName" in jsonData:
1309 proxyNewObject.cityName = self.
simpleValueExtractor(
"cityName", jsonData, str(),
"ProxyNew.cityName", i)
1310 if "zipCode" in jsonData:
1311 proxyNewObject.zipCode = self.
simpleValueExtractor(
"zipCode", jsonData, str(),
"ProxyNew.zipCode", i)
1312 if "timeZone" in jsonData:
1313 proxyNewObject.timeZone = self.
simpleValueExtractor(
"timeZone", jsonData, str(),
"ProxyNew.timeZone", i)
1314 if "latitude" in jsonData:
1315 proxyNewObject.latitude = self.
simpleValueExtractor(
"latitude", jsonData, float(),
"ProxyNew.latitude", i)
1316 if "longitude" in jsonData:
1317 proxyNewObject.longitude = self.
simpleValueExtractor(
"longitude", jsonData, float(),
"ProxyNew.longitude", i)
1318 if "metroCode" in jsonData:
1319 proxyNewObject.metroCode = self.
simpleValueExtractor(
"metroCode", jsonData, int(),
"ProxyNew.metroCode", i)
1320 if "faults" in jsonData:
1321 proxyNewObject.faults = self.
simpleValueExtractor(
"faults", jsonData, int(),
"ProxyNew.faults", i)
1322 if "faultsMax" in jsonData:
1323 proxyNewObject.faultsMax = self.
simpleValueExtractor(
"faultsMax", jsonData, int(),
"ProxyNew.faultsMax", i)
1324 if "categoryId" in jsonData:
1325 proxyNewObject.categoryId = self.
simpleValueExtractor(
"categoryId", jsonData, int(),
"ProxyNew.categoryId", i)
1326 if "limits" in jsonData:
1327 proxyNewObject.limits = jsonData[
"limits"]
1328 if "description" in jsonData:
1329 proxyNewObject.description = self.
simpleValueExtractor(
"description", jsonData, str(),
"ProxyNew.description", i)
1330 if "cDate" in jsonData:
1331 proxyNewObject.cDate = self.
timeExtractor(jsonData[
"cDate"])
1332 if "uDate" in jsonData:
1333 proxyNewObject.uDate = self.
timeExtractor(jsonData[
"uDate"])
1340 ProxyNewObjects = []
1343 if type(jsonData)
is types.ListType:
1344 for elem
in jsonData:
1349 raise Exceptions.DeserilizeException(
"ProxyNew not list type")
1350 except (ValueError, TypeError):
1351 raise Exceptions.DeserilizeException(
"ProxyNewDeserialize, some field has invalid type")
1352 return ProxyNewObjects
1359 ProxyUpdateObjects = []
1362 if type(jsonData)
is types.ListType:
1363 for elem
in jsonData:
1368 raise Exceptions.DeserilizeException(
"ProxyUpdate not list type")
1369 except (ValueError, TypeError):
1370 raise Exceptions.DeserilizeException(
"ProxyUpdateDeserialize, some field has invalid type")
1371 return ProxyUpdateObjects
1378 ProxyDeleteObjects = []
1381 if type(jsonData)
is types.ListType:
1382 for elem
in jsonData:
1384 if "siteId" in elem:
1385 ProxyDeleteObjects[i].siteId = self.
simpleValueExtractor(
"siteId", elem, str(),
"ProxyDelete.siteId", i)
1387 ProxyDeleteObjects[i].host = self.
simpleValueExtractor(
"host", elem, str(),
"ProxyDelete.host", i)
1388 if "criterions" in elem
and elem[
"criterions"]
is not None:
1389 if type(elem[
"criterions"])
is not types.DictType:
1390 raise Exceptions.DeserilizeException((
"ProxyDelete.criterions json-field type" +
1391 " not dict in {0} elem").
format(i))
1392 if hasattr(ProxyDeleteObjects[i].criterions,
'__iter__'):
1393 ProxyDeleteObjects[i].criterions.update(elem[
"criterions"])
1395 ProxyDeleteObjects[i].criterions = elem[
"criterions"]
1398 raise Exceptions.DeserilizeException(
"ProxyDelete not list type")
1399 except (ValueError, TypeError):
1400 raise Exceptions.DeserilizeException(
"ProxyDeleteDeserialize, some field has invalid type")
1401 return ProxyDeleteObjects
1408 ProxyStausObjects = []
1411 if type(jsonData)
is types.ListType:
1412 for elem
in jsonData:
1414 if "siteId" in elem:
1415 ProxyStausObjects[i].siteId = self.
simpleValueExtractor(
"siteId", elem, str(),
"ProxyStaus.siteId", i)
1417 ProxyStausObjects[i].host = self.
simpleValueExtractor(
"host", elem, str(),
"ProxyStaus.host", i)
1418 if "criterions" in elem
and elem[
"criterions"]
is not None:
1419 if type(elem[
"criterions"])
is not types.DictType:
1420 raise Exceptions.DeserilizeException((
"ProxyDelete.criterions json-field type" +
1421 " not dict in {0} elem").
format(i))
1422 if hasattr(ProxyStausObjects[i].criterions,
'__iter__'):
1423 ProxyStausObjects[i].criterions.update(elem[
"criterions"])
1425 ProxyStausObjects[i].criterions = elem[
"criterions"]
1428 raise Exceptions.DeserilizeException(
"ProxyStatus not list type")
1429 except (ValueError, TypeError):
1430 raise Exceptions.DeserilizeException(
"ProxyStatusDeserialize, some field has invalid type")
1431 return ProxyStausObjects
1439 ProxyFindObjects = []
1442 if type(jsonData)
is types.ListType:
1443 for elem
in jsonData:
1445 if "siteId" in elem:
1446 ProxyFindObjects[i].siteId = self.
simpleValueExtractor(
"siteId", elem, str(),
"ProxyFind.siteId", i)
1447 if "siteCriterions" in elem
and elem[
"siteCriterions"]
is not None:
1448 if type(elem[
"siteCriterions"])
is not types.DictType:
1449 raise Exceptions.DeserilizeException((
"ProxyFind.siteCriterions json-field type" +
1450 " not dict in {0} elem").
format(i))
1451 if hasattr(ProxyFindObjects[i].siteCriterions,
'__iter__'):
1452 ProxyFindObjects[i].siteCriterions.update(elem[
"siteCriterions"])
1454 ProxyFindObjects[i].siteCriterions = elem[
"siteCriterions"]
1455 if "criterions" in elem
and elem[
"criterions"]
is not None:
1456 if type(elem[
"criterions"])
is not types.DictType:
1457 raise Exceptions.DeserilizeException((
"ProxyFind.criterions json-field type" +
1458 " not dict in {0} elem").
format(i))
1459 if hasattr(ProxyFindObjects[i].criterions,
'__iter__'):
1460 ProxyFindObjects[i].criterions.update(elem[
"criterions"])
1462 ProxyFindObjects[i].criterions = elem[
"criterions"]
1465 raise Exceptions.DeserilizeException(
"ProxyStatus not list type")
1466 except (ValueError, TypeError):
1467 raise Exceptions.DeserilizeException(
"ProxyFindDeserialize, some field has invalid type")
1468 return ProxyFindObjects
1476 attributesObjects = []
1479 if type(jsonData)
is types.ListType:
1480 for elem
in jsonData:
1482 attributesObjects[i].siteId = self.
simpleValueExtractor(
"siteId", elem, str(),
"Attribute.siteId")
1484 if "urlMd5" in elem:
1485 attributesObjects[i].urlMd5 = self.
simpleValueExtractor(
"urlMd5", elem, str(),
"Attribute.urlMd5")
1487 attributesObjects[i].value = self.
simpleValueExtractor(
"value", elem, str(),
"Attribute.value")
1490 raise Exceptions.DeserilizeException(
"AttrSet not list type")
1491 except (ValueError, TypeError):
1492 raise Exceptions.DeserilizeException(
"AttrSetDeserialize, some field has invalid type")
1493 return attributesObjects
1501 attributesUpdateObjects = []
1504 if type(jsonData)
is types.ListType:
1505 for elem
in jsonData:
1507 attributesUpdateObjects[i].siteId = self.
simpleValueExtractor(
"siteId", elem, str(),
"AttributeUpdate.siteId")
1508 attributesUpdateObjects[i].name = self.
simpleValueExtractor(
"name", elem, str(),
"AttributeUpdate.name")
1509 if "urlMd5" in elem:
1511 "AttributeUpdate.urlMd5")
1513 attributesUpdateObjects[i].value = self.
simpleValueExtractor(
"value", elem, str(),
"AttributeUpdate.value")
1516 raise Exceptions.DeserilizeException(
"AttrUpdate not list type")
1517 except (ValueError, TypeError):
1518 raise Exceptions.DeserilizeException(
"AttrUpdateDeserialize, some field has invalid type")
1519 return attributesUpdateObjects
1527 attributesDeleteObjects = []
1530 if type(jsonData)
is types.ListType:
1531 for elem
in jsonData:
1533 attributesDeleteObjects[i].siteId = self.
simpleValueExtractor(
"siteId", elem, str(),
"AttributeDelete.siteId")
1535 attributesDeleteObjects[i].name = self.
simpleValueExtractor(
"name", elem, str(),
"AttributeDelete.name")
1536 if "criterions" in elem
and elem[
"criterions"]
is not None:
1537 if type(elem[
"criterions"])
is not types.DictType:
1538 raise Exceptions.DeserilizeException((
"ProxyFind.criterions json-field type" +
1539 " not dict in {0} elem").
format(i))
1540 if hasattr(attributesDeleteObjects[i].criterions,
'__iter__'):
1541 attributesDeleteObjects[i].criterions.update(elem[
"criterions"])
1543 attributesDeleteObjects[i].criterions = elem[
"criterions"]
1546 raise Exceptions.DeserilizeException(
"AttrDelete not list type")
1547 except (ValueError, TypeError):
1548 raise Exceptions.DeserilizeException(
"AttrDeleteDeserialize, some field has invalid type")
1549 return attributesDeleteObjects
1557 attributesFetchObjects = []
1560 if type(jsonData)
is types.ListType:
1561 for elem
in jsonData:
1563 attributesFetchObjects[i].siteId = self.
simpleValueExtractor(
"siteId", elem, str(),
"AttrFetch.siteId")
1565 attributesFetchObjects[i].name = self.
simpleValueExtractor(
"name", elem, str(),
"AttrFetch.name")
1566 if "criterions" in elem
and elem[
"criterions"]
is not None:
1567 if type(elem[
"criterions"])
is not types.DictType:
1568 raise Exceptions.DeserilizeException((
"ProxyFind.criterions json-field type" +
1569 " not dict in {0} elem").
format(i))
1570 if hasattr(attributesFetchObjects[i].criterions,
'__iter__'):
1571 attributesFetchObjects[i].criterions.update(elem[
"criterions"])
1573 attributesFetchObjects[i].criterions = elem[
"criterions"]
1576 raise Exceptions.DeserilizeException(
"AttrFetch not list type")
1577 except (ValueError, TypeError):
1578 raise Exceptions.DeserilizeException(
"AttrFetchDeserialize, some field has invalid type")
1579 return attributesFetchObjects
def readBatchItems(self, jsonData)
def URLFetchDeserialize(self, jsonData)
def URLDeleteDeserialize(self, jsonData)
def ProxyFindDeserialize(self, jsonData)
def AttrSetDeserialize(self, jsonData)
def fillSiteURLs(self, jsonData, siteId, userId=None, urlNormalizeMask=UrlNormalizator.NORM_NONE)
def URLPutDeserializeOneElement(self, jsonData, i)
def ProxyStatusDeserialize(self, jsonData)
def URLUpdateDeserialize(self, jsonData)
def siteNewDeserialize(self, jsonData)
def AttrFetchDeserialize(self, jsonData)
def URLContentDeserialize(self, jsonData)
def URLUpdateDeserializeOneElem(self, elem, i)
def BatchDeserialize(self, jsonData)
def simpleValueExtractor(self, fieldName, jsonData, _type, excepStr, i=None)
def siteStatusDeserialize(self, jsonData)
def URLAgeDeserialize(self, jsonData)
def URLPutDeserialize(self, jsonData)
def URLStatsDeserialize(self, jsonData)
def SQLCustomDeserialize(self, jsonData)
def URLStatusDeserialize(self, jsonData)
def URLPurgeDeserialize(self, jsonData)
def checkFilters(self, filters, sid)
def URLNewDeserialize(self, jsonData)
def AttrDeleteDeserialize(self, jsonData)
def ProxyUpdateDeserialize(self, jsonData, siteId='', host='')
def URLFetchDeserializeOneElem(self, elem, i)
def checkProperties(self, properties)
def ProxyDeleteDeserialize(self, jsonData)
def siteUpdateDeserialize(self, jsonData)
def ProxyNewElementDeserialize(self, proxyNewObject, jsonData, i)
def FieldRecalculatorDeserialize(self, jsonData)
def siteFindDeserialize(self, jsonData)
def ProxyNewDeserialize(self, jsonData)
def timeExtractor(self, timeField)
def URLNewElementDeserialize(self, urlNewObject, jsonData, i)
def URLVerifyDeserialize(self, jsonData)
def AttrUpdateDeserialize(self, jsonData)
def siteCleanupDeserialize(self, jsonData)
def URLHistoryDeserialize(self, jsonData)
def siteDeleteDeserialize(self, jsonData)
def URLCleanupDeserialize(self, jsonData)