liberasurecode  1.6.2
Erasure Code API library
erasurecode_helpers.c
Go to the documentation of this file.
1 /*
2  * Copyright 2014 Tushar Gohad, Kevin M Greenan, Eric Lambert
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions are met:
6  *
7  * Redistributions of source code must retain the above copyright notice, this
8  * list of conditions and the following disclaimer.
9  *
10  * Redistributions in binary form must reproduce the above copyright notice, this
11  * list of conditions and the following disclaimer in the documentation and/or
12  * other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY
13  * THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
14  * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
15  * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
16  * EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
17  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
18  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
19  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
20  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
21  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
22  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
23  *
24  * liberasurecode API helpers implementation
25  *
26  * vi: set noai tw=79 ts=4 sw=4:
27  */
28 #include <assert.h>
29 #include <stdio.h>
30 #include <stdarg.h>
31 #include <zlib.h>
32 #include "erasurecode_backend.h"
33 #include "erasurecode_helpers.h"
34 #include "erasurecode_helpers_ext.h"
35 #include "erasurecode_stdinc.h"
36 #include "erasurecode_version.h"
37 
38 #include "alg_sig.h"
39 #include "erasurecode_log.h"
40 
41 /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */
42 
43 static bool is_fragment(char *buf)
44 {
45  fragment_header_t *header = (fragment_header_t *) buf;
46 
47  assert(NULL != header);
48  if (header->magic == LIBERASURECODE_FRAG_HEADER_MAGIC) {
49  return true;
50  }
51 
52  return false;
53 }
54 
61 void *get_aligned_buffer16(int size)
62 {
63  void *buf;
64 
69  if (posix_memalign(&buf, 16, size) != 0) {
70  return NULL;
71  }
72 
73  memset(buf, 0, size);
74 
75  return buf;
76 }
77 
84 void * alloc_zeroed_buffer(int size)
85 {
86  return alloc_and_set_buffer(size, 0);
87 }
88 
97 void * alloc_and_set_buffer(int size, int value) {
98  void * buf = NULL; /* buffer to allocate and return */
99 
100  /* Allocate and zero the buffer, or set the appropriate error */
101  buf = malloc((size_t) size);
102  if (buf) {
103  buf = memset(buf, value, (size_t) size);
104  }
105  return buf;
106 }
107 
116 void * check_and_free_buffer(void * buf)
117 {
118  if (buf)
119  free(buf);
120  return NULL;
121 }
122 
123 char *alloc_fragment_buffer(int size)
124 {
125  char *buf;
126  fragment_header_t *header = NULL;
127 
128  size += sizeof(fragment_header_t);
129  buf = get_aligned_buffer16(size);
130 
131  if (buf) {
132  header = (fragment_header_t *) buf;
133  header->magic = LIBERASURECODE_FRAG_HEADER_MAGIC;
134  }
135 
136  return buf;
137 }
138 
139 int free_fragment_buffer(char *buf)
140 {
141  fragment_header_t *header;
142 
143  if (NULL == buf) {
144  return -1;
145  }
146 
147  buf -= sizeof(fragment_header_t);
148 
149  header = (fragment_header_t *) buf;
150  if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
151  log_error("Invalid fragment header (free fragment)!");
152  return -1;
153  }
154 
155  free(buf);
156  return 0;
157 }
158 
159 /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */
160 
168 uint64_t get_fragment_size(char *buf)
169 {
170 
171  if (NULL == buf)
172  return -1;
173 
174  return get_fragment_buffer_size(buf) + sizeof(fragment_header_t);
175  }
176 
185 int get_aligned_data_size(ec_backend_t instance, int data_len)
186 {
187  int k = instance->args.uargs.k;
188  int w = instance->args.uargs.w;
189  int word_size = w / 8;
190  int alignment_multiple;
191  int aligned_size = 0;
192 
193  /*
194  * For Cauchy reed-solomon align to k*word_size*packet_size
195  * For Vandermonde reed-solomon and flat-XOR, align to k*word_size
196  */
197  if (EC_BACKEND_JERASURE_RS_CAUCHY == instance->common.id) {
198  alignment_multiple = k * w * (sizeof(long) * 128);
199  } else {
200  alignment_multiple = k * word_size;
201  }
202 
203  aligned_size = ((data_len + alignment_multiple - 1) / alignment_multiple)
204  * alignment_multiple;
205 
206  return aligned_size;
207 }
208 
209 /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */
210 
212 {
213  buf += sizeof(fragment_header_t);
214 
215  return buf;
216 }
217 
218 int get_data_ptr_array_from_fragments(char **data_array, char **fragments,
219  int num_fragments)
220 {
221  int i = 0, num = 0;
222  for (i = 0; i < num_fragments; i++) {
223  char *frag = fragments[i];
224  if (frag == NULL) {
225  data_array[i] = NULL;
226  continue;
227  }
228  data_array[i] = get_data_ptr_from_fragment(frag);
229  num++;
230  }
231  return num;
232 }
233 
234 int get_fragment_ptr_array_from_data(char **frag_array, char **data,
235  int num_data)
236 {
237  int i = 0, num = 0;
238  for (i = 0; i < num_data; i++) {
239  char *data_ptr = frag_array[i];
240  if (data_ptr == NULL) {
241  data[i] = NULL;
242  continue;
243  }
244  data[i] = get_fragment_ptr_from_data(data_ptr);
245  num++;
246  }
247  return num;
248 }
249 
251 {
252  buf -= sizeof(fragment_header_t);
253 
254  return buf;
255 }
256 
258 {
259  fragment_header_t *header;
260 
261  buf -= sizeof(fragment_header_t);
262 
263  header = (fragment_header_t *) buf;
264 
265  if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
266  log_error("Invalid fragment header (get header ptr)!\n");
267  return NULL;
268  }
269 
270  return buf;
271 }
272 
273 /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */
274 
275 int set_fragment_idx(char *buf, int idx)
276 {
277  fragment_header_t *header = (fragment_header_t *) buf;
278 
279  assert(NULL != header);
280  if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
281  log_error("Invalid fragment header (idx check)!\n");
282  return -1;
283  }
284 
285  header->meta.idx = idx;
286 
287  return 0;
288 }
289 
290 int get_fragment_idx(char *buf)
291 {
292  fragment_header_t *header = (fragment_header_t *) buf;
293 
294  assert(NULL != header);
295  if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
296  log_error("Invalid fragment header (get idx)!");
297  return -1;
298  }
299 
300  return header->meta.idx;
301 }
302 
303 int set_fragment_payload_size(char *buf, int size)
304 {
305  fragment_header_t *header = (fragment_header_t *) buf;
306 
307  assert(NULL != header);
308  if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
309  log_error("Invalid fragment header (size check)!");
310  return -1;
311  }
312 
313  header->meta.size = size;
314 
315  return 0;
316 }
317 
319 {
320  fragment_header_t *header = (fragment_header_t *) buf;
321 
322  assert(NULL != header);
323  if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
324  log_error("Invalid fragment header (get size)!");
325  return -1;
326  }
327 
328  return header->meta.size;
329 }
330 
331 int set_fragment_backend_metadata_size(char *buf, int size)
332 {
333  fragment_header_t *header = (fragment_header_t *) buf;
334 
335  assert(NULL != header);
336  if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
337  log_error("Invalid fragment header (set fragment backend metadata size)!");
338  return -1;
339  }
340 
341  header->meta.frag_backend_metadata_size = size;
342 
343  return 0;
344 }
345 
347 {
348  fragment_header_t *header = (fragment_header_t *) buf;
349 
350  assert(NULL != header);
351  if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
352  log_error("Invalid fragment header (get fragment backend metadata size)!");
353  return -1;
354  }
355 
356  return header->meta.frag_backend_metadata_size;
357 }
358 
360 {
361  fragment_header_t *header = (fragment_header_t *) buf;
362 
363  assert(NULL != header);
364  if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
365  log_error("Invalid fragment header (get size)!");
366  return -1;
367  }
368 
369  return header->meta.size + header->meta.frag_backend_metadata_size;
370 }
371 
372 int set_orig_data_size(char *buf, int orig_data_size)
373 {
374  fragment_header_t *header = (fragment_header_t *) buf;
375 
376  assert(NULL != header);
377  if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
378  log_error("Invalid fragment header (set orig data check)!");
379  return -1;
380  }
381 
382  header->meta.orig_data_size = orig_data_size;
383 
384  return 0;
385 }
386 
387 int get_orig_data_size(char *buf)
388 {
389  fragment_header_t *header = (fragment_header_t *) buf;
390 
391  assert(NULL != header);
392  if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
393  log_error("Invalid fragment header (get orig data check)!");
394  return -1;
395  }
396 
397  return header->meta.orig_data_size;
398 }
399 
400 int set_libec_version(char *buf)
401 {
402  if (!is_fragment(buf)) {
403  return -1;
404  }
405  fragment_header_t *header = (fragment_header_t *) buf;
406  header->libec_version = (uint32_t)LIBERASURECODE_VERSION;
407  return 0;
408 }
409 
410 int get_libec_version(char *buf, uint32_t *ver)
411 {
412  if (!is_fragment(buf)) {
413  return -1;
414  }
415  fragment_header_t *header = (fragment_header_t *) buf;
416  *ver = header->libec_version;
417  return 0;
418 }
419 
420 int set_backend_id(char *buf, ec_backend_id_t id)
421 {
422  if (!is_fragment(buf)) {
423  return -1;
424  }
425  fragment_header_t *header = (fragment_header_t *) buf;
426  header->meta.backend_id = (uint8_t)id;
427  return 0;
428 }
429 
430 int get_backend_id(char *buf, ec_backend_id_t *id)
431 {
432  if (!is_fragment(buf)) {
433  return -1;
434  }
435  fragment_header_t *header = (fragment_header_t *) buf;
436  *id = header->meta.backend_id;
437  return 0;
438 }
439 
440 int set_backend_version(char *buf, uint32_t version)
441 {
442  if (!is_fragment(buf)) {
443  return -1;
444  }
445  fragment_header_t *header = (fragment_header_t *) buf;
446  header->meta.backend_version = version;
447  return 0;
448 }
449 
450 int get_backend_version(char *buf, uint32_t *version)
451 {
452  if (!is_fragment(buf)) {
453  return -1;
454  }
455  fragment_header_t *header = (fragment_header_t *) buf;
456  *version = header->meta.backend_version;
457  return 0;
458 }
459 
460 /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */
461 
462 inline int set_checksum(ec_checksum_type_t ct, char *buf, int blocksize)
463 {
464  fragment_header_t* header = (fragment_header_t*) buf;
465  char *data = get_data_ptr_from_fragment(buf);
466  char *flag;
467 
468  assert(NULL != header);
469  if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
470  log_error("Invalid fragment header (set chksum)!\n");
471  return -1;
472  }
473 
474  header->meta.chksum_type = ct;
475  header->meta.chksum_mismatch = 0;
476 
477  switch(header->meta.chksum_type) {
478  case CHKSUM_CRC32:
479  flag = getenv("LIBERASURECODE_WRITE_LEGACY_CRC");
480  if (flag && !(flag[0] == '\0' || (flag[0] == '0' && flag[1] == '\0'))) {
481  header->meta.chksum[0] = liberasurecode_crc32_alt(0, data, blocksize);
482  } else {
483  header->meta.chksum[0] = crc32(0, (unsigned char *) data, blocksize);
484  }
485  break;
486  case CHKSUM_MD5:
487  break;
488  case CHKSUM_NONE:
489  default:
490  break;
491  }
492 
493  return 0;
494 }
495 
496 /* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */
set_backend_version
int set_backend_version(char *buf, uint32_t version)
Definition: erasurecode_helpers.c:440
get_fragment_buffer_size
int get_fragment_buffer_size(char *buf)
Definition: erasurecode_helpers.c:359
get_aligned_buffer16
void * get_aligned_buffer16(int size)
Memory Management Methods.
Definition: erasurecode_helpers.c:61
set_fragment_backend_metadata_size
int set_fragment_backend_metadata_size(char *buf, int size)
Definition: erasurecode_helpers.c:331
free_fragment_buffer
int free_fragment_buffer(char *buf)
Definition: erasurecode_helpers.c:139
get_data_ptr_from_fragment
char * get_data_ptr_from_fragment(char *buf)
Definition: erasurecode_helpers.c:211
get_backend_version
int get_backend_version(char *buf, uint32_t *version)
Definition: erasurecode_helpers.c:450
get_data_ptr_array_from_fragments
int get_data_ptr_array_from_fragments(char **data_array, char **fragments, int num_fragments)
Definition: erasurecode_helpers.c:218
set_orig_data_size
int set_orig_data_size(char *buf, int orig_data_size)
Definition: erasurecode_helpers.c:372
get_fragment_size
uint64_t get_fragment_size(char *buf)
Return total fragment length (on-disk, on-wire)
Definition: erasurecode_helpers.c:168
check_and_free_buffer
void * check_and_free_buffer(void *buf)
Deallocate memory buffer if it's not NULL.
Definition: erasurecode_helpers.c:116
get_aligned_data_size
int get_aligned_data_size(ec_backend_t instance, int data_len)
Compute a size aligned to the number of data and the underlying wordsize of the EC algorithm.
Definition: erasurecode_helpers.c:185
get_libec_version
int get_libec_version(char *buf, uint32_t *ver)
Definition: erasurecode_helpers.c:410
liberasurecode_crc32_alt
int liberasurecode_crc32_alt(int crc, const void *buf, size_t size)
Definition: crc32.c:92
get_fragment_idx
int get_fragment_idx(char *buf)
Definition: erasurecode_helpers.c:290
alloc_zeroed_buffer
void * alloc_zeroed_buffer(int size)
Allocate a zero-ed buffer of a specific size.
Definition: erasurecode_helpers.c:84
set_libec_version
int set_libec_version(char *buf)
Definition: erasurecode_helpers.c:400
get_fragment_ptr_from_data
char * get_fragment_ptr_from_data(char *buf)
Definition: erasurecode_helpers.c:257
get_backend_id
int get_backend_id(char *buf, ec_backend_id_t *id)
Definition: erasurecode_helpers.c:430
get_orig_data_size
int get_orig_data_size(char *buf)
Definition: erasurecode_helpers.c:387
set_backend_id
int set_backend_id(char *buf, ec_backend_id_t id)
Definition: erasurecode_helpers.c:420
get_fragment_payload_size
int get_fragment_payload_size(char *buf)
Definition: erasurecode_helpers.c:318
get_fragment_ptr_array_from_data
int get_fragment_ptr_array_from_data(char **frag_array, char **data, int num_data)
Definition: erasurecode_helpers.c:234
is_fragment
static bool is_fragment(char *buf)
Definition: erasurecode_helpers.c:43
alloc_and_set_buffer
void * alloc_and_set_buffer(int size, int value)
Allocate a buffer of a specific size and set its' contents to the specified value.
Definition: erasurecode_helpers.c:97
set_checksum
int set_checksum(ec_checksum_type_t ct, char *buf, int blocksize)
Definition: erasurecode_helpers.c:462
set_fragment_idx
int set_fragment_idx(char *buf, int idx)
Definition: erasurecode_helpers.c:275
set_fragment_payload_size
int set_fragment_payload_size(char *buf, int size)
Definition: erasurecode_helpers.c:303
alloc_fragment_buffer
char * alloc_fragment_buffer(int size)
Definition: erasurecode_helpers.c:123
get_fragment_ptr_from_data_novalidate
char * get_fragment_ptr_from_data_novalidate(char *buf)
Definition: erasurecode_helpers.c:250
get_fragment_backend_metadata_size
int get_fragment_backend_metadata_size(char *buf)
Definition: erasurecode_helpers.c:346