hce-node application  1.4.3
HCE Hierarchical Cluster Engine node application
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros
citrusleaf.h
Go to the documentation of this file.
1 /*
2  * The Citrusleaf C interface. A good, basic library that many clients can be based on.
3  *
4  * This is the external, public header file
5  *
6  * this code currently assumes that the server is running in an ASCII-7 based
7  * (ie, utf8 or ISO-LATIN-1)
8  * character set, as values coming back from the server are UTF-8. We currently
9  * don't bother to convert to the character set of the machine we're running on
10  * but we advertise these values as 'strings'
11  *
12  * All rights reserved
13  * Brian Bulkowski, 2009
14  * CitrusLeaf
15  */
16 
17 #include <inttypes.h>
18 #include <stdbool.h>
19 #include <netinet/in.h>
20 
21 // do this both the new skool and old skool way which gives the highest correctness,
22 // speed, and compatibility
23 #pragma once
24 
25 #ifndef XDS // Hack for the sake of XDS. XDS includes the main CF libs.
26  // We do not want to include them again from client API
27 #include "citrusleaf/cf_atomic.h"
28 #include "citrusleaf/cf_log.h"
29 #include "citrusleaf/cf_ll.h"
30 #include "citrusleaf/cf_clock.h"
31 #include "citrusleaf/cf_vector.h"
32 #include "citrusleaf/cf_queue.h"
33 #include "citrusleaf/cf_alloc.h"
34 #include "citrusleaf/cf_digest.h"
35 #include "citrusleaf/cf_shash.h"
36 #endif
37 
38 #define STACK_BUF_SZ (1024 * 16) // provide a safe number for your system - linux tends to have 8M stacks these days
39 #define DEFAULT_PROGRESS_TIMEOUT 50
40 #define INFO_TIMEOUT_MS 500
41 
42 #define NODE_NAME_SIZE 20
43 
44 #ifdef __cplusplus
45 extern "C" {
46 #endif
47 
48 //
49 // Return values for the citrusleaf get and put calls
50 //
51 
52 typedef enum cl_rv {
55  CITRUSLEAF_FAIL_CLIENT = -1, // an out of memory or similar locally
57  CITRUSLEAF_FAIL_UNKNOWN = 1, // unknown failure on the server side
59  CITRUSLEAF_FAIL_GENERATION = 3, // likely a CAS write, and the write failed
60  CITRUSLEAF_FAIL_PARAMETER = 4, // you passed in bad parameters
68  CITRUSLEAF_FAIL_INCOMPATIBLE_TYPE = 12, // specified operation cannot be performed on that data type
71 } cl_rv;
72 
73 typedef enum cl_rvclient {
76 } cl_rvclient;
77 
78 
79 // hidden forward reference
80 typedef struct cl_conn_s cl_conn;
81 
82 enum cl_type {
83  CL_NULL = 0x00, CL_INT = 0x01, CL_FLOAT = 2, CL_STR = 0x03, CL_BLOB = 0x04,
86  CL_UNKNOWN = 666666};
87 typedef enum cl_type cl_type;
88 
90 
92 
94 
96 
97 
98 //
99 // An object is the value in a bin, or it is used as a key
100 // The object is typed according to the citrusleaf typing system
101 // These are often stack allocated, and are assigned using the 'wrap' calls
102 //
103 
104 typedef struct cl_object_s {
105  enum cl_type type;
106  size_t sz;
107  union {
108  char *str; // note for str: sz is strlen (not strlen+1
109  void *blob;
110  int64_t i64; // easiest to have one large int type
111  } u;
112 
113  void *free; // if this is set, this must be freed on destructuion
114 
115 } cl_object;
116 
117 
119 
120 // A bin is the bin name, and the value set or gotten
121 
122 typedef struct cl_bin_s {
123  char bin_name[32];
125 } cl_bin;
126 
127 // A record structure containing the most common fileds of a record
128 typedef struct cl_rec {
130  uint32_t generation;
131  uint32_t record_voidtime;
133  int n_bins;
134 } cl_rec;
135 
136 // Structure used by functions which want to return a bunch of records
137 typedef struct cl_batchresult {
138  pthread_mutex_t lock;
139  int numrecs;
142 
143 // An operation is the bin, plus the operator (write, read, add, etc)
144 // This structure is used for the more complex 'operate' call,
145 // which can specify simultaneous operations on multiple bins
146 
147 typedef struct cl_operation_s {
150 }cl_operation;
151 
152 //Structure to map the internal address to the external address
153 typedef struct cl_addrmap {
154  char *orig;
155  char *alt;
156 } cl_addrmap;
157 
158 //
159 // All citrusleaf functions return an integer. This integer is 0 if the
160 // call has succeeded, and a negative number if it has failed.
161 // All returns of pointers and objects are done through the parameters.
162 // (When in C++, use & parameters for return, but we're not there yet)
163 //
164 // 'void' return functions are only used for functions that are syntactically
165 // unable to fail.
166 //
167 
168 typedef void (*cl_async_fail_cb) (void *udata, int rv, uint64_t);
169 typedef void (*cl_async_success_cb) (void *udata, int rv, uint64_t);
170 
171 //
172 // Call this init function sometime early, create our mutexes and a few other things.
173 // We'd prefer if this is only called once
174 //
175 
176 int
177 citrusleaf_init(void);
178 
179 void
181 
182 //
183 // Initialize async queue and async worker threads.
184 //
185 // size_limit: Maximum number of items allowed in queue. Puts are rejected when maximum is reached.
186 //
187 // num_receiver_threads: Number of worker threads to create.
188 // If running in multi-process mode from python or perl, num_receiver_threads should be 1.
189 // The maximum num_receiver_threads is 32.
190 //
191 // fail_cb_fn: Callback for failed transactions. Use null if callback is not desired.
192 // success_cb_fn: Callback for successful transactions. Use null if callback is not desired.
193 //
194 int
195 citrusleaf_async_init(int size_limit, int num_receiver_threads, cl_async_fail_cb fail_cb_fn, cl_async_success_cb success_cb_fn);
196 
197 int
198 citrusleaf_async_reinit(int size_limit, unsigned int num_receiver_threads);
199 
200 void
201 citrusleaf_async_getstats(uint64_t *retries, uint64_t *dropouts, int *workitems);
202 
203 void
204 citrusleaf_async_set_nw_timeout(int nw_timeout);
205 
206 //
207 // If you wish to free up resources used by the citrusleaf client in your process,
208 // call this - all cl_conn will be invalid, and you'll have to call citrusleaf_init
209 // again to do anything
210 //
211 void citrusleaf_shutdown(void);
212 
213 void citrusleaf_set_debug(bool debug_flag);
214 
215 //
216 // This call will print stats to stderr
217 //
218 void citrusleaf_print_stats(void);
219 
220 //
221 // cl_object calls
222 //
223 
224 // fill out the object structure with the string in question - no allocs
226 void citrusleaf_object_init_str(cl_object *o, char const *str);
227 void citrusleaf_object_init_str2(cl_object *o, char const *str, size_t str_len);
228 void citrusleaf_object_init_blob(cl_object *o, void const *buf, size_t buf_len);
229 void citrusleaf_object_init_blob2(cl_object *o, void const *buf, size_t buf_len, cl_type type); // several blob types
230 void citrusleaf_object_init_int(cl_object *o, int64_t i);
233 
234 // frees all the memory in a bin array that would be returned from get_all but not the bin array itself
235 void citrusleaf_bins_free(cl_bin *bins, int n_bins);
236 
237 int citrusleaf_copy_bins(cl_bin **destbins, const cl_bin *srcbins, int n_bins);
238 
239 
240 // use:
241 // Calling this will return an cl_conn, which is a structure internal to the library.
242 // You can use this to do multiple calls to Aerospike instead of returning
243 // and re-getting each time
244 // although if you have an error, it might be because that particular cluster
245 // went down and you might need to get a new connection to the cluster.
246 // (perhaps we'll hide that detail in future APIs)
247 //
248 // You will be required to return your connections! That would be called a leak.
249 //
250 // the host-port combo should be some way to get to the cluster (often a DNS name)
251 // However, it might be done. Using the cluster name allows the clustering
252 // internals to keep track of other host-ports which are the same cluster,
253 // and match your request to other hosts
254 
255 // forward, hidden reference
256 struct cl_cluster_s;
257 typedef struct cl_cluster_s cl_cluster;
258 
261 
263 extern void citrusleaf_cluster_destroy(cl_cluster *asc);
264 extern void citrusleaf_cluster_shutdown(void);
265 
266 extern cl_cluster * citrusleaf_cluster_get_or_create(char *host, short port, int timeout_ms);
268 extern void citrusleaf_cluster_change_tend_speed(struct cl_cluster_s *asc, int secs);
269 extern void citrusleaf_cluster_change_info_timeout(struct cl_cluster_s *asc, int msecs);
270 extern void citrusleaf_cluster_use_nbconnect(struct cl_cluster_s *asc);
271 
272 extern void citrusleaf_cluster_put_compression_stat(cl_cluster *asc, uint64_t actual_sz, uint64_t compressed_sz);
273 extern void citrusleaf_cluster_get_compression_stat(cl_cluster *asc, uint64_t *actual_sz, uint64_t *compressed_sz);
274 
275 // the timeout is how long to wait before the cluster is "settled"
276 // 0 - a sensible default
277 // N - some number of MS
278 // -1 - don't wait this time
279 
280 extern cl_rv citrusleaf_cluster_add_host(cl_cluster *asc, char const *host, short port, int timeout_ms);
281 
282 extern void citrusleaf_cluster_add_addr_map(cl_cluster *asc, char *orig, char *alt);
283 
284 extern bool citrusleaf_cluster_settled(cl_cluster *asc);
285 
287 
288 // must free node_names when done
289 extern void cl_cluster_get_node_names(cl_cluster *asc, int *n_nodes, char **node_names);
290 
291 
292 // in the PHP system, URLs are lingua franca. We expect that
293 // a single cluster will be created with one name - the URL - and
294 // will be used over and over.
295 //
296 // URLs are of the form;
297 // citrusleaf://host:port (or similar)
298 extern cl_cluster *citrusleaf_cluster_get(char const *url);
299 
300 
301 // By default, the C client will "follow" the cluster, that is,
302 // track all the nodes in the cluster and continually update the cluster
303 // members. If, for testing, you wish to disable this feature, set this
304 // flag to false. This must be done before any 'addhost' calls, because
305 // even at the first one, the following of the cluster might start.
306 //
307 // Currently, setting this flags after hosts has an undefined effect.
308 
309 extern void citrusleaf_cluster_follow(cl_cluster *asc, bool flag);
310 
311 //
312 // write info structure
313 // There's a lot of info that can go into a write ---
314 typedef struct {
315  bool unique; // write unique - means success if didn't exist before
316  bool unique_bin; // write unique bin - means success if the bin didn't exist before
317  bool use_generation; // generation must be exact for write to succeed
318  bool use_generation_gt; // generation must be less - good for backup & restore
319  bool use_generation_dup; // on generation collision, create a duplicat
320  uint32_t generation;
322  uint32_t record_ttl; // seconds, from now, when the record would be auto-removed from the DBcd
323  cl_write_policy w_pol;
325 
326 static inline void
327 cl_write_parameters_set_default(cl_write_parameters *cl_w_p)
328 {
329  cl_w_p->unique = false;
330  cl_w_p->unique_bin = false;
331  cl_w_p->use_generation = false;
332  cl_w_p->use_generation_gt = false;
333  cl_w_p->use_generation_dup = false;
334  cl_w_p->timeout_ms = 0;
335  cl_w_p->record_ttl = 0;
336  cl_w_p->w_pol = CL_WRITE_RETRY;
337 }
338 
339 static inline void
340 cl_write_parameters_set_generation( cl_write_parameters *cl_w_p, uint32_t generation) {
341  cl_w_p->generation = generation;
342  cl_w_p->use_generation = true;
343 }
344 
345 static inline void
346 cl_write_parameters_set_generation_gt( cl_write_parameters *cl_w_p, uint32_t generation) {
347  cl_w_p->generation = generation;
348  cl_w_p->use_generation_gt = true;
349 }
350 
351 static inline void
352 cl_write_parameters_set_generation_dup( cl_write_parameters *cl_w_p, uint32_t generation) {
353  cl_w_p->generation = generation;
354  cl_w_p->use_generation_dup = true;
355 }
356 
357 
358 // scan_option info
359 typedef struct cl_scan_parameters_s {
360  bool fail_on_cluster_change; // honored by server: terminate scan if cluster in fluctuating state
361  cl_scan_priority priority; // honored by server: priority of scan
362  bool concurrent_nodes; // honored on client: work on nodes in parallel or serially
363  uint8_t threads_per_node; // honored on client: have multiple threads per node. @TODO
365 
366 static inline void
367 cl_scan_parameters_set_default(cl_scan_parameters *cl_scan_p)
368 {
369  cl_scan_p->fail_on_cluster_change = false;
370  cl_scan_p->concurrent_nodes = false;
371  cl_scan_p->threads_per_node = 1; // not honored currently
372  cl_scan_p->priority = CL_SCAN_PRIORITY_AUTO;
373 }
374 
375 typedef struct cl_node_response_s {
379 //
380 // get-all will malloc an array of values and return all current values for a row.
381 // thus, it is SELECT * in SQL. So pass in a pointer to cl_value to be filled, and a
382 // pointer-to-int to know how many.
383 //
384 // The memory contract with get_all is that the bins pointer (*bins) must be freed by the caller.
385 // The data you've actually retrieved (cl_object->u.str ; cl->object->u.blob) has been allocated using malloc.
386 // You may use it for your own purposes, transfer it to another program, or you must free it.
387 // citrusleaf_object_free() will free the internal memory in these cases,
388 // or you can call citrusleaf_bins_free and it will take care of all memory.
389 //
390 // Note this is different from getting a specific set of bins, where the bin array was passed in.
391 // (the simple 'get') See that call for information there.
392 
393 
394 cl_rv
395 citrusleaf_get_all(cl_cluster *asc, const char *ns, const char *set, const cl_object *key, cl_bin **bins, int *n_bins, int timeout_ms, uint32_t *cl_gen);
396 
397 cl_rv
398 citrusleaf_get_all_digest(cl_cluster *asc, const char *ns, const cf_digest *d, cl_bin **bins, int *n_bins, int timeout_ms, uint32_t *cl_gen);
399 
400 cl_rv
401 citrusleaf_get_all_digest_getsetname(cl_cluster *asc, const char *ns, const cf_digest *d, cl_bin **bins, int *n_bins, int timeout_ms, uint32_t *cl_gen, char *setname);
402 
403 //
404 // Put is like insert. Create a list of bins, and call this function to set them.
405 //
406 
407 cl_rv
408 citrusleaf_put(cl_cluster *asc, const char *ns, const char *set, const cl_object *key, const cl_bin *bins, int n_bins, const cl_write_parameters *cl_w_p);
409 
410 cl_rv
411 citrusleaf_put_digest(cl_cluster *asc, const char *ns, const cf_digest *d, const cl_bin *bins, int n_bins, const cl_write_parameters *cl_w_p);
412 
413 cl_rv
414 citrusleaf_put_replace(cl_cluster *asc, const char *ns, const char *set, const cl_object *key, const cl_bin *values, int n_values, const cl_write_parameters *cl_w_p);
415 
416 cl_rv
417 citrusleaf_restore(cl_cluster *asc, const char *ns, const cf_digest *digest, const char *set, const cl_bin *values, int n_values, const cl_write_parameters *cl_w_p);
418 
419 //
420 // Send asynchronous put request to server and return without waiting for response.
421 // The response is available in callback specified in citrusleaf_async_init().
422 //
423 cl_rv
424 citrusleaf_async_put(cl_cluster *asc, const char *ns, const char *set, const cl_object *key,
425  const cl_bin *bins, int n_bins, const cl_write_parameters *cl_w_p, uint64_t trid, void *udata);
426 
427 //
428 // Send asynchronous put request with digest key to server and return without waiting for response.
429 // The response is available in callback specified in citrusleaf_async_init().
430 //
431 cl_rv
432 citrusleaf_async_put_digest(cl_cluster *asc, const char *ns, const cf_digest *d, const char *set,
433  const cl_bin *bins, int n_bins, const cl_write_parameters *cl_w_p, uint64_t trid, void *udata);
434 
435 //
436 // Send asynchronous put request to server and return without waiting for response.
437 //
438 cl_rv
439 citrusleaf_async_put_forget(cl_cluster *asc, const char *ns, const char *set, const cl_object *key,
440  const cl_bin *bins, int n_bins, const cl_write_parameters *cl_w_p);
441 
442 //
443 // Send asynchronous put request with digest key to server and return without waiting for response.
444 //
445 cl_rv
446 citrusleaf_async_put_digest_forget(cl_cluster *asc, const char *ns, const cf_digest *d, const char *set,
447  const cl_bin *bins, int n_bins, const cl_write_parameters *cl_w_p);
448 
449 cl_rv
450 citrusleaf_async_put_digest_xdr(cl_cluster *asc, const char *ns, const cf_digest *d, char *set,
451  const cl_bin *bins, int n_bins, const cl_write_parameters *cl_w_p, uint64_t trid, void *udata);
452 
455 
456 void
458 
459 /*
460  * Set minimum size of packet, above which packet will be compressed before sending on wire,
461  * provided compression is enabled.
462  */
463 int
465 
466 //
467 // Get is like select in SQL. Create a list of bins to get, and call this function to retrieve
468 // the values.
469 
470 cl_rv
471 citrusleaf_get(cl_cluster *asc, const char *ns, const char *set, const cl_object *key, cl_bin *bins, int n_bins, int timeout_ms, uint32_t *cl_gen);
472 
473 cl_rv
474 citrusleaf_get_digest(cl_cluster *asc, const char *ns, const cf_digest *d, cl_bin *bins, int n_bins, int timeout_ms, uint32_t *cl_gen);
475 
476 //
477 // Delete simply wipes this single key off the face of the earth.
478 //
479 
480 cl_rv
481 citrusleaf_delete(cl_cluster *asc, const char *ns, const char *set, const cl_object *key, const cl_write_parameters *cl_w_p);
482 
483 cl_rv
484 citrusleaf_delete_digest(cl_cluster *asc, const char *ns, const cf_digest *d, const cl_write_parameters *cl_w_p);
485 
486 cl_rv
487 citrusleaf_async_delete_digest_xdr(cl_cluster *asc, const char *ns, const cf_digest *digest, const cl_write_parameters *cl_w_p, void *udata);
488 
489 //
490 // Efficiently determine if the key exists.
491 // (Note: The bins are currently ignored but may be testable in the future.)
492 //
493 
494 cl_rv
495 citrusleaf_exists_key(cl_cluster *asc, const char *ns, const char *set, const cl_object *key, cl_bin *bins, int n_bins, int timeout_ms, uint32_t *cl_gen);
496 
497 cl_rv
498 citrusleaf_exists_digest(cl_cluster *asc, const char *ns, const cf_digest *d, cl_bin *bins, int n_bins, int timeout_ms, uint32_t *cl_gen);
499 
500 //
501 // get many call has the following properties:
502 // you can pass null either with the namespace or the set, and a large iteration will occur
503 // Memory available vanishes after 'return', if you want a copy, make a copy
504 // Non-zero return in the callback aborts the call
505 typedef int (*citrusleaf_get_many_cb) (char *ns, cf_digest *keyd, char *set, uint32_t generation, uint32_t record_ttl,
506  cl_bin *bins, int n_bins, bool is_last, void *udata);
507 
508 cl_rv
509 citrusleaf_scan(cl_cluster *asc, char *ns, char *set, cl_bin *bins, int n_bins, bool get_key, citrusleaf_get_many_cb cb, void *udata, bool nobindata);
510 
511 // response is a vector of cl_node_response
512 cf_vector *
513 citrusleaf_scan_all_nodes (cl_cluster *asc, char *ns, char *set, cl_bin *bins, int n_bins, bool nobindata, uint8_t scan_pct,
514  citrusleaf_get_many_cb cb, void *udata, cl_scan_parameters *scan_p);
515 
516 cl_rv
517 citrusleaf_scan_node (cl_cluster *asc, char *node_name, char *ns, char *set, cl_bin *bins, int n_bins, bool nobindata, uint8_t scan_pct,
518  citrusleaf_get_many_cb cb, void *udata, cl_scan_parameters *scan_p);
519 
520 //
521 // Initialize batch queue and specified number of worker threads (Maximum thread count is 6).
522 //
523 cl_rv
524 citrusleaf_batch_init(int n_threads);
525 
526 //
527 // Get many digest
528 // This version of the call acts exactly as the get digest call, but takes an array
529 // of digests. Those digests will be retrieved with the same API conventions as the
530 // previous ones.
531 // To come: an array of keys, but it'll just be a wrapper on this.
532 //
533 cl_rv
534 citrusleaf_get_many_digest(cl_cluster *asc, char *ns, const cf_digest *digests, int n_digests, cl_bin *bins, int n_bins, bool get_key /*if true, retrieve key instead of simply digest*/,
535  citrusleaf_get_many_cb cb, void *udata);
536 
537 //
538 // Get many digest without a callback
539 // This version of the batch-get call does not need the callback function. It will return an array of records.
540 // The results are returned in an array. No ordering is guaranteed between the input digest array and
541 // the returned rows. If the corresponding records for the digests are not found in the cluster, there wont
542 // be any corresponding entry in the result array indicating that the records are missing. The caller must
543 // call the citrusleaf_free_batchresult() to free the memory allocated during this operation.
544 cl_rv
545 citrusleaf_get_many_digest_direct(cl_cluster *asc, char *ns, const cf_digest *digests, int n_digests, cl_batchresult **br);
546 
547 // Utility function to free the memory allocated by the citrusleaf_get_many_digest_direct() function
548 void
550 
551 //
552 // Key exists many digest
553 // This version of the call acts exactly as the exists digest call, but takes an array
554 // of digests. Those digests will be retrieved with the same API conventions as the
555 // previous ones.
556 // To come: an array of keys, but it'll just be a wrapper on this.
557 //
558 
559 cl_rv
560 citrusleaf_exists_many_digest(cl_cluster *asc, char *ns, const cf_digest *digests, int n_digests, cl_bin *bins, int n_bins, bool get_key /*if true, retrieve key instead of simply digest*/,
561  citrusleaf_get_many_cb cb, void *udata);
562 
563 //
564 // Use the information system to request data from a given node
565 // Pass in a '\n' seperated list of names, or no names at all
566 // [Perhaps a better interface would be an array of pointers-to-names, returning an
567 // array of pointers-to-values?]
568 // Returns a malloc'd string which is the response from the server.
569 //
570 
571 int
572 citrusleaf_info(char *host, short port, char *names, char **values, int timeout_ms);
573 
574 int
575 citrusleaf_info_host(struct sockaddr_in *sa_in, char *names, char **values, int timeout_ms, bool send_asis);
576 
577 // Do a lookup with this name and port, and add the sockaddr to the
578 // vector using the unique lookup
579 int cl_lookup(cl_cluster *asc, char *hostname, short port, cf_vector *sockaddr_in_v);
580 
581 //
582 // This call is good for testing. Call it when you think you know the values. If the key doesn't exist, or
583 // the data is incorrect, then the server that is serving the request will spit a failure, and if you're
584 // running in the right server debug mode you can examine the error in detail.
585 //
586 
587 cl_rv
588 citrusleaf_verify(cl_cluster *asc, const char *ns, const char *set, const cl_object *key, const cl_bin *bins, int n_bins, int timeout_ms, uint32_t *cl_gen);
589 cl_rv
590 citrusleaf_delete_verify(cl_cluster *asc, const char *ns, const char *set, const cl_object *key, const cl_write_parameters *cl_w_p);
591 
592 //
593 // This call allows the caller to specify the operation - read, write, add, etc. Multiple operations
594 // can be specified in a single call.//
595 
596 cl_rv
597 citrusleaf_operate(cl_cluster *asc, const char *ns, const char *set, const cl_object *key, cl_operation *operations, int n_operations, const cl_write_parameters *cl_w_p, int replace, uint32_t *generation);
598 
599 cl_rv
600 citrusleaf_operate_digest(cl_cluster *asc, const char *ns, cf_digest *digest, cl_operation *operations, int n_operations, const cl_write_parameters *cl_w_p, int replace, uint32_t *generation);
601 
602 //
603 // This debugging call can be useful for tracking down errors and coordinating with server failures
604 // gets the digest for a particular set and key
605 int
606 citrusleaf_calculate_digest(const char *set, const cl_object *key, cf_digest *digest);
607 
608 
609 #ifdef __cplusplus
610 } // end extern "C"
611 #endif
612 
613