hce-node application  1.4.3
HCE Hierarchical Cluster Engine node application
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros
proto.h
Go to the documentation of this file.
1 /*
2  * Citrusleaf Aerospike
3  * include/proto.h - wire protocol definition
4  *
5  * Copyright 2008-2012 by Citrusleaf. All rights reserved.
6  * THIS IS UNPUBLISHED PROPRIETARY SOURCE CODE. THE COPYRIGHT NOTICE
7  * ABOVE DOES NOT EVIDENCE ANY ACTUAL OR INTENDED PUBLICATION.
8  */
9 
10 #pragma once
11 
12 #include <stddef.h>
13 #include <stdint.h>
14 
15 #ifdef __cplusplus
16 extern "C" {
17 #endif
18 
19 #define CL_PROTO_RESULT_OK 0
20 #define CL_PROTO_RESULT_FAIL_UNKNOWN 1 // unknown failure
21 #define CL_PROTO_RESULT_FAIL_NOTFOUND 2
22 #define CL_PROTO_RESULT_FAIL_GENERATION 3
23 #define CL_PROTO_RESULT_FAIL_PARAMETER 4
24 #define CL_PROTO_RESULT_FAIL_KEY_EXISTS 5 // if 'WRITE_ADD', could fail because already exists
25 #define CL_PROTO_RESULT_FAIL_BIN_EXISTS 6
26 #define CL_PROTO_RESULT_FAIL_CLUSTER_KEY_MISMATCH 7
27 #define CL_PROTO_RESULT_FAIL_PARTITION_OUT_OF_SPACE 8
28 #define CL_PROTO_RESULT_FAIL_TIMEOUT 9
29 #define CL_PROTO_RESULT_FAIL_NOXDS 10
30 #define CL_PROTO_RESULT_FAIL_UNAVAILABLE 11 // error returned during node down and partition isn't available
31 #define CL_PROTO_RESULT_FAIL_INCOMPATIBLE_TYPE 12 // op and bin type incompatibilty
32 #define CL_PROTO_RESULT_FAIL_RECORD_TOO_BIG 13
33 #define CL_PROTO_RESULT_FAIL_KEY_BUSY 14
34 
35 // Forward definitions
36 struct cl_bin_s;
37 
38 
39 
40 
41 // NOTE! This is the only place where datamodel.h is exported in the external
42 // proto.h. Maybe I should make a different (annotated) proto.h?
43 
44 /* cl_particle_type
45  * Particles are typed, which reflects their contents:
46  * NULL: no associated content (not sure I really need this internally?)
47  * INTEGER: a signed, 64-bit integer
48  * BIGNUM: a big number
49  * STRING: a null-terminated UTF-8 string
50  * BLOB: arbitrary-length binary data
51  * TIMESTAMP: milliseconds since 1 January 1970, 00:00:00 GMT
52  * DIGEST: an internal Aerospike key digest */
53 typedef enum {
54  CL_PARTICLE_TYPE_NULL = 0,
55  CL_PARTICLE_TYPE_INTEGER = 1,
56  CL_PARTICLE_TYPE_FLOAT = 2,
57  CL_PARTICLE_TYPE_STRING = 3,
58  CL_PARTICLE_TYPE_BLOB = 4,
59  CL_PARTICLE_TYPE_TIMESTAMP = 5,
60  CL_PARTICLE_TYPE_DIGEST = 6,
61  CL_PARTICLE_TYPE_JAVA_BLOB = 7,
62  CL_PARTICLE_TYPE_CSHARP_BLOB = 8,
63  CL_PARTICLE_TYPE_PYTHON_BLOB = 9,
64  CL_PARTICLE_TYPE_RUBY_BLOB = 10,
65  CL_PARTICLE_TYPE_MAX = 11
66 } cl_particle_type;
67 
68 
69 /* SYNOPSIS
70  * Aerospike wire protocol
71  *
72  * Version 2
73  *
74  * Aerospike uses a message-oriented wire protocol to transfer information.
75  * Each message consists of a header, which determines the type and the length
76  * to follow. This is called the 'proto_msg'.
77  *
78  * these messages are vectored out to the correct handler. Over TCP, they can be
79  * pipelined (but not out of order). If we wish to support out of order responses,
80  * we should upgrade the protocol.
81  *
82  * the most common type of message is the cl_msg, a message which reads or writes
83  * a single row to the data store.
84  *
85  */
86 
87 
88 #define CL_PROTO_VERSION 2
89 #define CL_PROTO_TYPE_INFO 1 // ascii-format message for determining server info
90 #define CL_PROTO_TYPE_CL_MSG 3
91 #define CL_PROTO_TYPE_CL_MSG_COMPRESSED 4
92 
93 #define CL_RESULT_OK 0
94 #define CL_RESULT_FAIL 1
95 #define CL_RESULT_NOTFOUND 2
96 
97 #ifdef CF_WINDOWS
98 
99 #pragma pack(push, 1) // packing is now 1
100 typedef struct cl_proto_s {
101  uint64_t version :8;
102  uint64_t type :8;
103  uint64_t sz :48;
104 } cl_proto;
105 #pragma pack(pop) // packing is back to what it was
106 
107 #pragma pack(push, 1) // packing is now 1
108 /*
109  * zlib decompression API needs original size of the compressed data.
110  * So we need to transfer it to another end.
111  * This structure packs together -
112  * header + original size of data + compressed data
113  */
114 typedef struct cl_comp_proto_s {
115  cl_proto proto; // Protocol header
116  uint64_t org_sz; // Original size of compressed data hold in 'data'
117  uint8_t data[]; // Compressed data
118 } cl_comp_proto;
119 #pragma pack(pop) // packing is back to what it was
120 
121 
122 /* cl_msg_field
123  * Aerospike message field */
124 
125 #pragma pack(push, 1)
126 typedef struct cl_msg_field_s {
127 #define CL_MSG_FIELD_TYPE_NAMESPACE 0 // UTF8 string
128 #define CL_MSG_FIELD_TYPE_SET 1
129 #define CL_MSG_FIELD_TYPE_KEY 2 // contains a key type
130 #define CL_MSG_FIELD_TYPE_BIN 3 // used for secondary key access - contains a bin, thus a name and value
131 #define CL_MSG_FIELD_TYPE_DIGEST_RIPE 4 // used to send the digest just computed to the server so it doesn't have to
132 #define CL_MSG_FIELD_TYPE_GU_TID 5
133 #define CL_MSG_FIELD_TYPE_DIGEST_RIPE_ARRAY 6
134 #define CL_MSG_FIELD_TYPE_TRID 7
135 #define CL_MSG_FIELD_TYPE_SCAN_OPTIONS 8
136  uint32_t field_sz; // get the data size through the accessor function, don't worry, it's a small macro
137  uint8_t type;
138  uint8_t data[];
139 } cl_msg_field;
140 #pragma pack(pop)
141 
142 #pragma pack(push, 1) // packing is now 1
143 typedef struct cl_msg_op_s {
144  uint32_t op_sz;
145  uint8_t op;
146  uint8_t particle_type;
147  uint8_t version;
148  uint8_t name_sz;
149  uint8_t name[]; // UTF-8
150  // there's also a value here but you can't have two variable size arrays
151 } cl_msg_op;
152 #pragma pack(pop) // packing is back to what it was
153 
154 /* cl_msg_key_s
155 */
156 // Not using it anywhere in libevent2 client
157 // Please be aware when using this for any other client
158 /*
159 #pragma pack(push, 1) // packing is now 1
160 typedef struct cl_msg_key_s {
161  cl_msg_field f;
162  uint8_t key[];
163 } cl_msg_key;
164 #pragma pack(pop) // packing is back to what it was
165 */
166 /* cl_msg_number_s
167 */
168 
169 #pragma pack(push, 1) // packing is now 1
170 typedef struct cl_msg_number_s {
171  uint32_t number;
172  cl_msg_field f;
173 } cl_msg_number;
174 #pragma pack(pop) // packing is back to what it was
175 
176 
177 /* cl_msg
178  * Aerospike message
179  * size: size of the payload, not including the header */
180 
181 #pragma pack(push, 1) // packing is now 1
182 typedef struct cl_msg_s {
183 /*00*/ uint8_t header_sz; // number of uint8_ts in this header
184 /*01*/ uint8_t info1; // bitfield about this request
185 /*02*/ uint8_t info2;
186 /*03*/ uint8_t info3;
187 /*04*/ uint8_t unused;
188 /*05*/ uint8_t result_code;
189 /*06*/ uint32_t generation;
190 /*10*/ uint32_t record_ttl;
191 /*14*/ uint32_t transaction_ttl;
192 /*18*/ uint16_t n_fields; // size in uint8_ts
193 /*20*/ uint16_t n_ops; // number of operations
194 /*22*/ uint8_t data[0]; // data contains first the fields, then the ops
195 } cl_msg;
196 #pragma pack(pop) // packing is back to what it was
197 
198 
199 /* cl_ms
200  * Aerospike message
201  * sz: size of the payload, not including the header */
202 
203 #pragma pack(push, 1) // packing is now 1
204 typedef struct as_msg_s {
205  cl_proto proto;
206  cl_msg m;
207 } as_msg;
208 #pragma pack(pop) // packing is back to what it was
209 
210 
211 #else
212 
213 typedef struct cl_proto_s {
214  uint8_t version;
215  uint8_t type;
216  uint64_t sz:48;
217  uint8_t data[];
218 } __attribute__ ((__packed__)) cl_proto;
219 
220 /*
221  * zlib decompression API needs original size of the compressed data.
222  * So we need to transfer it to another end.
223  * This structure packs together -
224  * header + original size of data + compressed data
225  */
226 typedef struct cl_comp_proto_s {
227  cl_proto proto; // Protocol header
228  uint64_t org_sz; // Original size of compressed data hold in 'data'
229  uint8_t data[]; // Compressed data
230 } cl_comp_proto;
231 
232  /* cl_msg_field
233  * Aerospike message field */
234 typedef struct cl_msg_field_s {
235 #define CL_MSG_FIELD_TYPE_NAMESPACE 0 // UTF8 string
236 #define CL_MSG_FIELD_TYPE_SET 1
237 #define CL_MSG_FIELD_TYPE_KEY 2 // contains a key type
238 #define CL_MSG_FIELD_TYPE_BIN 3 // used for secondary key access - contains a bin, thus a name and value
239 #define CL_MSG_FIELD_TYPE_DIGEST_RIPE 4 // used to send the digest just computed to the server so it doesn't have to
240 #define CL_MSG_FIELD_TYPE_GU_TID 5
241 #define CL_MSG_FIELD_TYPE_DIGEST_RIPE_ARRAY 6
242 #define CL_MSG_FIELD_TYPE_TRID 7
243 #define CL_MSG_FIELD_TYPE_SCAN_OPTIONS 8
244  uint32_t field_sz; // get the data size through the accessor function, don't worry, it's a small macro
245  uint8_t type;
246  uint8_t data[];
247 } __attribute__((__packed__)) cl_msg_field;
248 
249 
250 
251 
252 typedef struct cl_msg_op_s {
253  uint32_t op_sz;
254  uint8_t op;
255  uint8_t particle_type;
256  uint8_t version;
257  uint8_t name_sz;
258  uint8_t name[]; // UTF-8
259  // there's also a value here but you can't have two variable size arrays
260 } __attribute__((__packed__)) cl_msg_op;
261 
262 
263 
264 typedef struct cl_msg_key_s {
265  cl_msg_field f;
266  uint8_t key[];
267 } __attribute__ ((__packed__)) cl_msg_key;
268 
269 typedef struct cl_msg_number_s {
270  cl_msg_field f;
271  uint32_t number;
272 } __attribute__ ((__packed__)) cl_msg_number;
273 
274 
275 
276 /* cl_msg
277  * Aerospike message
278  * size: size of the payload, not including the header */
279 typedef struct cl_msg_s {
280 /*00*/ uint8_t header_sz; // number of uint8_ts in this header
281 /*01*/ uint8_t info1; // bitfield about this request
282 /*02*/ uint8_t info2;
283 /*03*/ uint8_t info3;
284 /*04*/ uint8_t unused;
285 /*05*/ uint8_t result_code;
286 /*06*/ uint32_t generation;
287 /*10*/ uint32_t record_ttl;
288 /*14*/ uint32_t transaction_ttl;
289 /*18*/ uint16_t n_fields; // size in uint8_ts
290 /*20*/ uint16_t n_ops; // number of operations
291 /*22*/ uint8_t data[]; // data contains first the fields, then the ops
292 } __attribute__((__packed__)) cl_msg;
293 
294 /* cl_ms
295  * Aerospike message
296  * sz: size of the payload, not including the header */
297 typedef struct as_msg_s {
298  cl_proto proto;
299  cl_msg m;
300 } __attribute__((__packed__)) as_msg;
301 
302 #endif
303 
304 #define CL_MSG_OP_READ 1 // read the value in question
305 #define CL_MSG_OP_WRITE 2 // write the value in question
306 #define CL_MSG_OP_WRITE_UNIQUE 3 // write a namespace-wide unique value
307 #define CL_MSG_OP_WRITE_NOW 4 // write the server-current time
308 #define CL_MSG_OP_INCR 5
309 #define CL_MSG_OP_APPEND_SEGMENT 6 // Append segment to a particle
310 #define CL_MSG_OP_APPEND_SEGMENT_EXT 7 // Extended append - with parameters
311 #define CL_MSG_OP_APPEND_SEGMENT_QUERY 8 // Query to return subset of segments
312 #define CL_MSG_OP_APPEND 9 // Add to an existing particle
313 #define CL_MSG_OP_PREPEND 10 // Add to the beginning of an existing particle
314 #define CL_MSG_OP_TOUCH 11 // Touch
315 
316 #define CL_MSG_OP_MC_INCR 129 // Memcache-compatible version of the increment command
317 #define CL_MSG_OP_MC_APPEND 130 // Memcache compatible append. Allow appending to ints.
318 #define CL_MSG_OP_MC_PREPEND 131 // Memcache compatile prepend. Allow prepending to ints.
319 #define CL_MSG_OP_MC_TOUCH 132 // Memcache compatible touch - does not change generation count
320 
321 #define CL_MSG_INFO1_READ (1 << 0) // contains a read operation
322 #define CL_MSG_INFO1_GET_ALL (1 << 1) // get all bins, period
323 #define CL_MSG_INFO1_GET_ALL_NODATA (1 << 2) // get all bins WITHOUT data (currently unimplemented)
324 #define CL_MSG_INFO1_VERIFY (1 << 3) // verify is a GET transaction that includes data, and assert if the data aint right
325 #define CL_MSG_INFO1_XDS (1 << 4) // operation is being performed by XDS
326 #define CL_MSG_INFO1_NOBINDATA (1 << 5) // dOBo not read the bin information
327 
328 #define CL_MSG_INFO2_WRITE (1 << 0) // contains a write semantic
329 #define CL_MSG_INFO2_DELETE (1 << 1) // fling a record into the belly of Moloch
330 #define CL_MSG_INFO2_GENERATION (1 << 2) // pay attention to the generation
331 #define CL_MSG_INFO2_GENERATION_GT (1 << 3) // apply write if new generation >= old, good for restore
332 #define CL_MSG_INFO2_GENERATION_DUP (1 << 4) // if a generation collision, create a duplicate
333 #define CL_MSG_INFO2_WRITE_UNIQUE (1 << 5) // write only if it doesn't exist
334 #define CL_MSG_INFO2_WRITE_BINUNIQUE (1 << 6)
335 #define CL_MSG_INFO2_WRITE_MERGE (1 << 7) // merge this with current
336 
337 #define CL_MSG_INFO3_LAST (1 << 0) // this is the last of a multi-part message
338 #define CL_MSG_INFO3_TRACE (1 << 1) // apply server trace logging for this transaction
339 #define CL_MSG_INFO3_TOMBSTONE (1 << 2) // if set on response, a version was a delete tombstone
340 #define CL_MSG_INFO3_REPLACE (1 << 3) // properly a write option, but there are no more bits. Overwrite existing record only; do not create new record
341 
342 
343 static inline uint8_t * cl_msg_op_get_value_p(cl_msg_op *op)
344 {
345  return ( ((uint8_t *)op) + sizeof(cl_msg_op) + op->name_sz);
346 }
347 
348 static inline uint32_t cl_msg_op_get_value_sz(cl_msg_op *op)
349 {
350  return( op->op_sz - (4 + op->name_sz) );
351 }
352 
353 static inline uint32_t cl_msg_field_get_value_sz(cl_msg_field *f)
354 {
355  return( f->field_sz - 1 );
356 }
357 
358 static inline cl_msg_field *
359 cl_msg_field_get_next(cl_msg_field *mf)
360 {
361  return ( (cl_msg_field *) (((uint8_t *)mf) + sizeof(mf->field_sz) + mf->field_sz) );
362 }
363 
364 
365 /* cl_msg_field_get
366  * Retrieve a specific field from a message */
367 static inline cl_msg_field *
368 cl_msg_field_get(cl_msg *msg, uint8_t type)
369 {
370  uint16_t n;
371  cl_msg_field *fp = NULL;
372 
373  fp = (cl_msg_field *)msg->data;
374  for (n = 0; n < msg->n_fields; n++) {
375 
376  if (fp->type == type)
377  break;
378 
379  fp = cl_msg_field_get_next(fp);
380  }
381  if (n == msg->n_fields)
382  return(NULL);
383  else
384  return(fp);
385 }
386 
387 /* cl_msg_field_getnext
388  * iterator for all fields of a particular type
389  * First time through: pass 0 as current, you'll get a field
390  * next time: pass the current as current
391  * you'll get null when there are no more
392  */
393 static inline cl_msg_op *
394 cl_msg_op_get_next(cl_msg_op *op)
395 {
396  return ( (cl_msg_op *) (((uint8_t *) op) + sizeof(op->op_sz) + op->op_sz ) );
397 }
398 
399 
400 static inline cl_msg_op *
401 cl_msg_op_iterate(cl_msg *msg, cl_msg_op *current, int *n)
402 {
403  // skip over the fields the first time
404  if (!current) {
405  if (msg->n_ops == 0) return(0); // short cut
406  cl_msg_field *mf = (cl_msg_field *) msg->data;
407  for (uint32_t i = 0; i < msg->n_fields; i++)
408  mf = cl_msg_field_get_next(mf);
409  current = (cl_msg_op *) mf;
410  *n = 0;
411  return(current);
412  }
413  (*n)++;
414  if (*n >= msg->n_ops) return(0);
415  return ( cl_msg_op_get_next( current ) );
416 
417 }
418 
419 
420 /* cl_msg_size_get
421  * Get the size of a message */
422 static inline size_t
423 cl_proto_size_get(cl_proto *proto)
424 {
425  return( sizeof(cl_proto) + proto->sz);
426 }
427 
428 
429 /* Function declarations */
430 extern void cl_proto_swap(cl_proto *m);
431 extern void cl_msg_swap_header(cl_msg *m);
432 extern void cl_msg_swap_field(cl_msg_field *mf);
433 extern void cl_msg_swap_fields(cl_msg *m);
434 extern void cl_msg_swap_ops(cl_msg *m);
435 extern void cl_msg_swap_op(cl_msg_op *op);
436 extern void cl_msg_swap_fields_and_ops(cl_msg *m);
437 
438 #ifdef __cplusplus
439 } // end extern "C"
440 #endif
441