HCE Project Python language Distributed Tasks Manager Application, Distributed Crawler Application and client API bindings.  2.0.0-chaika
Hierarchical Cluster Engine Python language binding
dc_db.ProcessedContentInternalStruct.ProcessedContentInternalStruct Class Reference
Inheritance diagram for dc_db.ProcessedContentInternalStruct.ProcessedContentInternalStruct:
Collaboration diagram for dc_db.ProcessedContentInternalStruct.ProcessedContentInternalStruct:

Static Public Member Functions

def processDictProcessedContent (cDateValue, contentMask, processedContent)
 
def parseProcessedBuf (buf, cDateValue, contentMask)
 

Static Public Attributes

string DATA_FIELD = "data"
 
string CDATE_FIELD = "CDate"
 

Detailed Description

Definition at line 21 of file ProcessedContentInternalStruct.py.

Member Function Documentation

◆ parseProcessedBuf()

def dc_db.ProcessedContentInternalStruct.ProcessedContentInternalStruct.parseProcessedBuf (   buf,
  cDateValue,
  contentMask 
)
static

Definition at line 98 of file ProcessedContentInternalStruct.py.

98  def parseProcessedBuf(buf, cDateValue, contentMask):
99  ret = []
100  try:
101  processedContent = json.loads(base64.b64decode(buf))
102  except Exception as excp:
103  processedContent = None
104  logger.debug(">>> Wrong something bad with processedContent decode, =" + str(excp))
105  if processedContent is not None:
106  if isinstance(processedContent, types.DictType):
107  if "custom" in processedContent and "internal" in processedContent:
108  ret = ProcessedContentInternalStruct.processDictProcessedContent(cDateValue, contentMask, processedContent)
109  else:
110  logger.debug(">>> Wrong custom or internal not present in processedContent DICT")
111  elif isinstance(processedContent, types.ListType):
112  if contentMask & dc.EventObjects.URLContentRequest.CONTENT_TYPE_PROCESSED:
113  resultElem = json.dumps(processedContent, ensure_ascii=False, encoding='utf-8')
114  resultElem = base64.b64encode(resultElem)
115  content = dc.EventObjects.Content(resultElem, cDateValue, dc.EventObjects.Content.CONTENT_PROCESSOR_CONTENT)
116  ret.append(content)
117  return ret

◆ processDictProcessedContent()

def dc_db.ProcessedContentInternalStruct.ProcessedContentInternalStruct.processDictProcessedContent (   cDateValue,
  contentMask,
  processedContent 
)
static

Definition at line 34 of file ProcessedContentInternalStruct.py.

34  def processDictProcessedContent(cDateValue, contentMask, processedContent):
35  ret = []
36 
37  logger.debug("contentMask: %s, len(processedContent): %s", str(contentMask), str(len(processedContent)))
38  if contentMask & dc.EventObjects.URLContentRequest.CONTENT_TYPE_PROCESSED:
39  if len(processedContent["custom"]) > 0:
40  logger.debug("return item processedContent['custom'][0]")
41  resultElem = base64.b64encode(processedContent["custom"][0])
42  else:
43  logger.debug("return deep copy of processedContent, `internal` and `custom` removed")
44  localProcessedContent = copy.deepcopy(processedContent)
45  del localProcessedContent["internal"]
46  del localProcessedContent["custom"]
47  resultElem = json.dumps(localProcessedContent, ensure_ascii=False, encoding='utf-8')
48  resultElem = base64.b64encode(resultElem)
49  content = dc.EventObjects.Content(resultElem, cDateValue, dc.EventObjects.Content.CONTENT_PROCESSOR_CONTENT)
50  ret.append(content)
51  if contentMask & dc.EventObjects.URLContentRequest.CONTENT_TYPE_PROCESSED_INTERNAL and \
52  contentMask & dc.EventObjects.URLContentRequest.CONTENT_TYPE_PROCESSED_CUSTOM:
53  i = 0
54  for elem in processedContent["internal"]:
55  resultElem = json.dumps(elem, ensure_ascii=False, encoding='utf-8')
56  resultElem = base64.b64encode(resultElem)
57  contentInternal = dc.EventObjects.Content(resultElem, cDateValue,
58  dc.EventObjects.Content.CONTENT_PROCESSOR_CONTENT)
59  if i < len(processedContent["custom"]):
60  elemCustom = processedContent["custom"][i]
61  resultElem = base64.b64encode(elemCustom)
62  contentCustom = dc.EventObjects.Content(resultElem, cDateValue,
63  dc.EventObjects.Content.CONTENT_PROCESSOR_CONTENT)
64  else:
65  contentCustom = None
66  insertTuple = (contentInternal, contentCustom)
67  ret.append(insertTuple)
68  i += 1
69  elif contentMask & dc.EventObjects.URLContentRequest.CONTENT_TYPE_PROCESSED_INTERNAL:
70  for elem in processedContent["internal"]:
71  resultElem = json.dumps(elem, ensure_ascii=False, encoding='utf-8')
72  resultElem = base64.b64encode(resultElem)
73  content = dc.EventObjects.Content(resultElem, cDateValue, dc.EventObjects.Content.CONTENT_PROCESSOR_CONTENT)
74  ret.append(content)
75  if (contentMask & dc.EventObjects.URLContentRequest.CONTENT_TYPE_PROCESSED_ALL) == 0:
76  break
77  elif contentMask & dc.EventObjects.URLContentRequest.CONTENT_TYPE_PROCESSED_CUSTOM:
78  for elem in processedContent["custom"]:
79  resultElem = base64.b64encode(elem)
80  content = dc.EventObjects.Content(resultElem, cDateValue, dc.EventObjects.Content.CONTENT_PROCESSOR_CONTENT)
81  ret.append(content)
82  if (contentMask & dc.EventObjects.URLContentRequest.CONTENT_TYPE_PROCESSED_ALL) == 0:
83  break
84 
85  logger.debug("Return Content instance list: %s", str(len(ret)))
86 
87  return ret
88 
89 

Member Data Documentation

◆ CDATE_FIELD

string dc_db.ProcessedContentInternalStruct.ProcessedContentInternalStruct.CDATE_FIELD = "CDate"
static

Definition at line 24 of file ProcessedContentInternalStruct.py.

◆ DATA_FIELD

string dc_db.ProcessedContentInternalStruct.ProcessedContentInternalStruct.DATA_FIELD = "data"
static

Definition at line 23 of file ProcessedContentInternalStruct.py.


The documentation for this class was generated from the following file: