-
-
Notifications
You must be signed in to change notification settings - Fork 85
/
Copy pathblosc2.h
2608 lines (2362 loc) · 95.8 KB
/
blosc2.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/*********************************************************************
Blosc - Blocked Shuffling and Compression Library
Copyright (c) 2021 Blosc Development Team <[email protected]>
https://blosc.org
License: BSD 3-Clause (see LICENSE.txt)
See LICENSE.txt for details about copyright and rights to use.
**********************************************************************/
/*********************************************************************
@file blosc2.h
@brief Blosc2 header file.
This file contains Blosc2 public API and the structures needed to use it.
@author Blosc Development Team <[email protected]>
**********************************************************************/
#ifndef BLOSC_BLOSC2_H
#define BLOSC_BLOSC2_H
#include "blosc2/blosc2-export.h"
#include "blosc2/blosc2-common.h"
#include "blosc2/blosc2-stdio.h"
#if defined(_WIN32) && !defined(__MINGW32__)
#include <windows.h>
#include <malloc.h>
#include <process.h>
#define getpid _getpid
#endif
#include <limits.h>
#include <stdlib.h>
#include <stdint.h>
#include <stdbool.h>
#include <stdio.h>
#ifdef __cplusplus
extern "C" {
#endif
// For compatibility with the Blosc 1.x series
#ifdef BLOSC1_COMPAT
// Blosc2 symbols that should be accessible from Blosc 1.x API
#define BLOSC_VERSION_MAJOR BLOSC2_VERSION_MAJOR
#define BLOSC_VERSION_MINOR BLOSC2_VERSION_MINOR
#define BLOSC_VERSION_RELEASE BLOSC2_VERSION_RELEASE
#define BLOSC_VERSION_STRING BLOSC2_VERSION_STRING
#define BLOSC_VERSION_DATE BLOSC2_VERSION_DATE
#define BLOSC_MAX_OVERHEAD BLOSC2_MAX_OVERHEAD
#define BLOSC_MAX_BUFFERSIZE BLOSC2_MAX_BUFFERSIZE
// API that changed to blosc1_ prefix
#define blosc_compress blosc1_compress
#define blosc_decompress blosc1_decompress
#define blosc_getitem blosc1_getitem
#define blosc_get_compressor blosc1_get_compressor
#define blosc_set_compressor blosc1_set_compressor
#define blosc_cbuffer_sizes blosc1_cbuffer_sizes
#define blosc_cbuffer_validate blosc1_cbuffer_validate
#define blosc_cbuffer_metainfo blosc1_cbuffer_metainfo
#define blosc_get_blocksize blosc1_get_blocksize
#define blosc_set_blocksize blosc1_set_blocksize
#define blosc_set_splitmode blosc1_set_splitmode
// API that changed to blosc2_ prefix
#define blosc_init blosc2_init
#define blosc_destroy blosc2_destroy
#define blosc_free_resources blosc2_free_resources
#define blosc_get_nthreads blosc2_get_nthreads
#define blosc_set_nthreads blosc2_set_nthreads
#define blosc_compcode_to_compname blosc2_compcode_to_compname
#define blosc_compname_to_compcode blosc2_compname_to_compcode
#define blosc_list_compressors blosc2_list_compressors
#define blosc_get_version_string blosc2_get_version_string
#define blosc_get_complib_info blosc2_get_complib_info
#define blosc_cbuffer_versions blosc2_cbuffer_versions
#define blosc_cbuffer_complib blosc2_cbuffer_complib
#endif
/* Version numbers */
#define BLOSC2_VERSION_MAJOR 2 /* for major interface/format changes */
#define BLOSC2_VERSION_MINOR 15 /* for minor interface/format changes */
#define BLOSC2_VERSION_RELEASE 3 /* for tweaks, bug-fixes, or development */
#define BLOSC2_VERSION_STRING "2.15.3.dev" /* string version. Sync with above! */
#define BLOSC2_VERSION_DATE "$Date:: 2024-11-28 #$" /* date version year-month-day */
/* The maximum number of dimensions for Blosc2 NDim arrays */
#define BLOSC2_MAX_DIM 8
/* Tracing macros */
#define BLOSC_TRACE_ERROR(msg, ...) BLOSC_TRACE(error, msg, ##__VA_ARGS__)
#define BLOSC_TRACE_WARNING(msg, ...) BLOSC_TRACE(warning, msg, ##__VA_ARGS__)
#define BLOSC_TRACE_INFO(msg, ...) BLOSC_TRACE(info, msg, ##__VA_ARGS__)
#define BLOSC_TRACE(cat, msg, ...) \
do { \
const char *__e = getenv("BLOSC_TRACE"); \
if (!__e) { break; } \
fprintf(stderr, "[%s] - " msg " (%s:%d)\n", #cat, ##__VA_ARGS__, __FILE__, __LINE__); \
} while(0)
#define BLOSC_ERROR_NULL(pointer, rc) \
do { \
if ((pointer) == NULL) { \
BLOSC_TRACE_ERROR("Pointer is null"); \
return (rc); \
} \
} while (0)
#define BLOSC_ERROR(rc) \
do { \
int rc_ = (rc); \
if (rc_ < BLOSC2_ERROR_SUCCESS) { \
char *error_msg = print_error(rc_); \
BLOSC_TRACE_ERROR("%s", error_msg); \
return rc_; \
} \
} while (0)
#define BLOSC_INFO(msg, ...) \
do { \
const char *__e = getenv("BLOSC_INFO"); \
if (!__e) { break; } \
fprintf(stderr, "[INFO] - " msg "\n", ##__VA_ARGS__); \
} while(0)
/* The VERSION_FORMAT symbols below should be just 1-byte long */
enum {
/* Blosc format version, starting at 1
1 -> Blosc pre-1.0
2 -> Blosc 1.x stable series
3 -> Blosc 2-alpha.x series
4 -> Blosc 2.x beta.1 series
5 -> Blosc 2.x stable series
*/
BLOSC1_VERSION_FORMAT_PRE1 = 1,
BLOSC1_VERSION_FORMAT = 2,
BLOSC2_VERSION_FORMAT_ALPHA = 3,
BLOSC2_VERSION_FORMAT_BETA1 = 4,
BLOSC2_VERSION_FORMAT_STABLE = 5,
BLOSC2_VERSION_FORMAT = BLOSC2_VERSION_FORMAT_STABLE,
};
/* The FRAME_FORMAT_VERSION symbols below should be just 4-bit long */
enum {
/* Blosc format version
* 1 -> First version (introduced in beta.2)
* 2 -> Second version (introduced in rc.1)
*
*/
BLOSC2_VERSION_FRAME_FORMAT_BETA2 = 1, // for 2.0.0-beta2 and after
BLOSC2_VERSION_FRAME_FORMAT_RC1 = 2, // for 2.0.0-rc1 and after
BLOSC2_VERSION_FRAME_FORMAT = BLOSC2_VERSION_FRAME_FORMAT_RC1,
};
//!< Struct for storing data from instrumentation of codecs
// This can be flexible because it is typically used mainly for development
typedef struct {
float cratio;
float cspeed;
float filter_speed;
//float memory;
//float power;
uint8_t flags[4];
} blosc2_instr;
enum {
#ifndef BLOSC_H
BLOSC_MIN_HEADER_LENGTH = 16,
//!< Minimum header length (Blosc1)
#endif // BLOSC_H
BLOSC_EXTENDED_HEADER_LENGTH = 32,
//!< Extended header length (Blosc2, see README_HEADER)
BLOSC2_MAX_OVERHEAD = BLOSC_EXTENDED_HEADER_LENGTH,
//!< The maximum overhead during compression in bytes. This equals
//!< to @ref BLOSC_EXTENDED_HEADER_LENGTH now, but can be higher in future
//!< implementations.
BLOSC2_MAX_BUFFERSIZE = (INT_MAX - BLOSC2_MAX_OVERHEAD),
//!< Maximum source buffer size to be compressed
#ifndef BLOSC_H
BLOSC_MAX_TYPESIZE = UINT8_MAX,
//!< Maximum typesize before considering source buffer as a stream of bytes.
//!< Cannot be larger than 255.
#endif // BLOSC_H
BLOSC_MIN_BUFFERSIZE = 32,
//!< Minimum buffer size to be compressed.
};
enum {
BLOSC2_DEFINED_TUNER_START = 0,
BLOSC2_DEFINED_TUNER_STOP = 31,
//!< Blosc-defined tuners must be between 0 - 31.
BLOSC2_GLOBAL_REGISTERED_TUNER_START = 32,
BLOSC2_GLOBAL_REGISTERED_TUNER_STOP = 159,
//!< Blosc-registered tuners must be between 31 - 159.
BLOSC2_GLOBAL_REGISTERED_TUNERS = 0,
//!< Number of Blosc-registered tuners at the moment.
BLOSC2_USER_REGISTERED_TUNER_START = 160,
BLOSC2_USER_REGISTERED_TUNER_STOP = 255,
//!< User-defined tuners must be between 160 - 255.
};
/**
* @brief Codes for the different tuners shipped with Blosc
*/
enum {
BLOSC_STUNE = 0,
BLOSC_LAST_TUNER = 1,
//!< Determine the last tuner defined by Blosc.
BLOSC_LAST_REGISTERED_TUNE = BLOSC2_GLOBAL_REGISTERED_TUNER_START + BLOSC2_GLOBAL_REGISTERED_TUNERS - 1,
//!< Determine the last registered tuner. It is used to check if a tuner is registered or not.
};
enum {
BLOSC2_DEFINED_FILTERS_START = 0,
BLOSC2_DEFINED_FILTERS_STOP = 31,
//!< Blosc-defined filters must be between 0 - 31.
BLOSC2_GLOBAL_REGISTERED_FILTERS_START = 32,
BLOSC2_GLOBAL_REGISTERED_FILTERS_STOP = 159,
//!< Blosc-registered filters must be between 32 - 159.
BLOSC2_GLOBAL_REGISTERED_FILTERS = 5,
//!< Number of Blosc-registered filters at the moment.
BLOSC2_USER_REGISTERED_FILTERS_START = 160,
BLOSC2_USER_REGISTERED_FILTERS_STOP = 255,
//!< User-defined filters must be between 128 - 255.
BLOSC2_MAX_FILTERS = 6,
//!< Maximum number of filters in the filter pipeline
BLOSC2_MAX_UDFILTERS = 16,
//!< Maximum number of filters that a user can register.
};
/**
* @brief Codes for filters.
*
* @sa #blosc1_compress
*/
enum {
#ifndef BLOSC_H
BLOSC_NOSHUFFLE = 0,
//!< No shuffle (for compatibility with Blosc1).
BLOSC_NOFILTER = 0,
//!< No filter.
BLOSC_SHUFFLE = 1,
//!< Byte-wise shuffle. `filters_meta` does not have any effect here.
BLOSC_BITSHUFFLE = 2,
//!< Bit-wise shuffle. `filters_meta` does not have any effect here.
#endif // BLOSC_H
BLOSC_DELTA = 3,
//!< Delta filter. `filters_meta` does not have any effect here.
BLOSC_TRUNC_PREC = 4,
//!< Truncate mantissa precision.
//!< Positive values in `filters_meta` will keep bits; negative values will zero bits.
BLOSC_LAST_FILTER = 5,
//!< sentinel
BLOSC_LAST_REGISTERED_FILTER = BLOSC2_GLOBAL_REGISTERED_FILTERS_START + BLOSC2_GLOBAL_REGISTERED_FILTERS - 1,
//!< Determine the last registered filter. It is used to check if a filter is registered or not.
};
/**
* @brief Codes for internal flags (see blosc1_cbuffer_metainfo)
*/
enum {
#ifndef BLOSC_H
BLOSC_DOSHUFFLE = 0x1, //!< byte-wise shuffle
BLOSC_MEMCPYED = 0x2, //!< plain copy
BLOSC_DOBITSHUFFLE = 0x4, //!< bit-wise shuffle
#endif // BLOSC_H
BLOSC_DODELTA = 0x8, //!< delta coding
};
/**
* @brief Codes for new internal flags in Blosc2
*/
enum {
BLOSC2_USEDICT = 0x1, //!< use dictionaries with codec
BLOSC2_BIGENDIAN = 0x2, //!< data is in big-endian ordering
BLOSC2_INSTR_CODEC = 0x80, //!< codec is instrumented (mainly for development)
};
/**
* @brief Values for different Blosc2 capabilities
*/
enum {
BLOSC2_MAXDICTSIZE = 128 * 1024, //!< maximum size for compression dicts
BLOSC2_MAXBLOCKSIZE = 536866816 //!< maximum size for blocks
};
enum {
BLOSC2_DEFINED_CODECS_START = 0,
BLOSC2_DEFINED_CODECS_STOP = 31,
//!< Blosc-defined codecs must be between 0 - 31.
BLOSC2_GLOBAL_REGISTERED_CODECS_START = 32,
BLOSC2_GLOBAL_REGISTERED_CODECS_STOP = 159,
//!< Blosc-registered codecs must be between 31 - 159.
BLOSC2_GLOBAL_REGISTERED_CODECS = 5,
//!< Number of Blosc-registered codecs at the moment.
BLOSC2_USER_REGISTERED_CODECS_START = 160,
BLOSC2_USER_REGISTERED_CODECS_STOP = 255,
//!< User-defined codecs must be between 160 - 255.
};
/**
* @brief Codes for the different compressors shipped with Blosc
*/
enum {
#ifndef BLOSC_H
BLOSC_BLOSCLZ = 0,
BLOSC_LZ4 = 1,
BLOSC_LZ4HC = 2,
BLOSC_ZLIB = 4,
BLOSC_ZSTD = 5,
#endif // BLOSC_H
BLOSC_LAST_CODEC = 6,
//!< Determine the last codec defined by Blosc.
BLOSC_LAST_REGISTERED_CODEC = BLOSC2_GLOBAL_REGISTERED_CODECS_START + BLOSC2_GLOBAL_REGISTERED_CODECS - 1,
//!< Determine the last registered codec. It is used to check if a codec is registered or not.
};
// Names for the different compressors shipped with Blosc
#ifndef BLOSC_H
#define BLOSC_BLOSCLZ_COMPNAME "blosclz"
#define BLOSC_LZ4_COMPNAME "lz4"
#define BLOSC_LZ4HC_COMPNAME "lz4hc"
#define BLOSC_ZLIB_COMPNAME "zlib"
#define BLOSC_ZSTD_COMPNAME "zstd"
#endif // BLOSC_H
/**
* @brief Codes for compression libraries shipped with Blosc (code must be < 8)
*/
enum {
#ifndef BLOSC_H
BLOSC_BLOSCLZ_LIB = 0,
BLOSC_LZ4_LIB = 1,
BLOSC_ZLIB_LIB = 3,
BLOSC_ZSTD_LIB = 4,
#endif // BLOSC_H
BLOSC_UDCODEC_LIB = 6,
BLOSC_SCHUNK_LIB = 7, //!< compressor library in super-chunk header
};
/**
* @brief Names for the different compression libraries shipped with Blosc
*/
#ifndef BLOSC_H
#define BLOSC_BLOSCLZ_LIBNAME "BloscLZ"
#define BLOSC_LZ4_LIBNAME "LZ4"
#define BLOSC_ZLIB_LIBNAME "Zlib"
#define BLOSC_ZSTD_LIBNAME "Zstd"
#endif // BLOSC_H
/**
* @brief The codes for compressor formats shipped with Blosc
*/
enum {
#ifndef BLOSC_H
BLOSC_BLOSCLZ_FORMAT = BLOSC_BLOSCLZ_LIB,
BLOSC_LZ4_FORMAT = BLOSC_LZ4_LIB,
//!< LZ4HC and LZ4 share the same format
BLOSC_LZ4HC_FORMAT = BLOSC_LZ4_LIB,
BLOSC_ZLIB_FORMAT = BLOSC_ZLIB_LIB,
BLOSC_ZSTD_FORMAT = BLOSC_ZSTD_LIB,
#endif // BLOSC_H
BLOSC_UDCODEC_FORMAT = BLOSC_UDCODEC_LIB,
};
/**
* @brief The version formats for compressors shipped with Blosc.
* All versions here starts at 1
*/
enum {
#ifndef BLOSC_H
BLOSC_BLOSCLZ_VERSION_FORMAT = 1,
BLOSC_LZ4_VERSION_FORMAT = 1,
BLOSC_LZ4HC_VERSION_FORMAT = 1, /* LZ4HC and LZ4 share the same format */
BLOSC_ZLIB_VERSION_FORMAT = 1,
BLOSC_ZSTD_VERSION_FORMAT = 1,
#endif // BLOSC_H
BLOSC_UDCODEC_VERSION_FORMAT = 1,
};
/**
* @brief Split mode for blocks.
* NEVER and ALWAYS are for experimenting with compression ratio.
* AUTO for nearly optimal behaviour (based on heuristics).
* FORWARD_COMPAT provides best forward compatibility (default).
*/
#ifndef BLOSC_H
enum {
BLOSC_ALWAYS_SPLIT = 1,
BLOSC_NEVER_SPLIT = 2,
BLOSC_AUTO_SPLIT = 3,
BLOSC_FORWARD_COMPAT_SPLIT = 4,
};
#endif // BLOSC_H
/**
* @brief Offsets for fields in Blosc2 chunk header.
*/
enum {
BLOSC2_CHUNK_VERSION = 0x0, //!< the version for the chunk format
BLOSC2_CHUNK_VERSIONLZ = 0x1, //!< the version for the format of internal codec
BLOSC2_CHUNK_FLAGS = 0x2, //!< flags and codec info
BLOSC2_CHUNK_TYPESIZE = 0x3, //!< (uint8) the number of bytes of the atomic type
BLOSC2_CHUNK_NBYTES = 0x4, //!< (int32) uncompressed size of the buffer (this header is not included)
BLOSC2_CHUNK_BLOCKSIZE = 0x8, //!< (int32) size of internal blocks
BLOSC2_CHUNK_CBYTES = 0xc, //!< (int32) compressed size of the buffer (including this header)
BLOSC2_CHUNK_FILTER_CODES = 0x10, //!< the codecs for the filter pipeline (1 byte per code)
BLOSC2_CHUNK_FILTER_META = 0x18, //!< meta info for the filter pipeline (1 byte per code)
BLOSC2_CHUNK_BLOSC2_FLAGS = 0x1F, //!< flags specific for Blosc2 functionality
};
/**
* @brief Run lengths for special values for chunks/frames
*/
enum {
BLOSC2_NO_SPECIAL = 0x0, //!< no special value
BLOSC2_SPECIAL_ZERO = 0x1, //!< zero special value
BLOSC2_SPECIAL_NAN = 0x2, //!< NaN special value
BLOSC2_SPECIAL_VALUE = 0x3, //!< generic special value
BLOSC2_SPECIAL_UNINIT = 0x4, //!< non initialized values
BLOSC2_SPECIAL_LASTID = 0x4, //!< last valid ID for special value (update this adequately)
BLOSC2_SPECIAL_MASK = 0x7 //!< special value mask (prev IDs cannot be larger than this)
};
/**
* @brief Error codes
* Each time an error code is added here, its corresponding message error should be added in
* print_error()
*/
enum {
BLOSC2_ERROR_SUCCESS = 0, //<! Success
BLOSC2_ERROR_FAILURE = -1, //<! Generic failure
BLOSC2_ERROR_STREAM = -2, //<! Bad stream
BLOSC2_ERROR_DATA = -3, //<! Invalid data
BLOSC2_ERROR_MEMORY_ALLOC = -4, //<! Memory alloc/realloc failure
BLOSC2_ERROR_READ_BUFFER = -5, //!< Not enough space to read
BLOSC2_ERROR_WRITE_BUFFER = -6, //!< Not enough space to write
BLOSC2_ERROR_CODEC_SUPPORT = -7, //!< Codec not supported
BLOSC2_ERROR_CODEC_PARAM = -8, //!< Invalid parameter supplied to codec
BLOSC2_ERROR_CODEC_DICT = -9, //!< Codec dictionary error
BLOSC2_ERROR_VERSION_SUPPORT = -10, //!< Version not supported
BLOSC2_ERROR_INVALID_HEADER = -11, //!< Invalid value in header
BLOSC2_ERROR_INVALID_PARAM = -12, //!< Invalid parameter supplied to function
BLOSC2_ERROR_FILE_READ = -13, //!< File read failure
BLOSC2_ERROR_FILE_WRITE = -14, //!< File write failure
BLOSC2_ERROR_FILE_OPEN = -15, //!< File open failure
BLOSC2_ERROR_NOT_FOUND = -16, //!< Not found
BLOSC2_ERROR_RUN_LENGTH = -17, //!< Bad run length encoding
BLOSC2_ERROR_FILTER_PIPELINE = -18, //!< Filter pipeline error
BLOSC2_ERROR_CHUNK_INSERT = -19, //!< Chunk insert failure
BLOSC2_ERROR_CHUNK_APPEND = -20, //!< Chunk append failure
BLOSC2_ERROR_CHUNK_UPDATE = -21, //!< Chunk update failure
BLOSC2_ERROR_2GB_LIMIT = -22, //!< Sizes larger than 2gb not supported
BLOSC2_ERROR_SCHUNK_COPY = -23, //!< Super-chunk copy failure
BLOSC2_ERROR_FRAME_TYPE = -24, //!< Wrong type for frame
BLOSC2_ERROR_FILE_TRUNCATE = -25, //!< File truncate failure
BLOSC2_ERROR_THREAD_CREATE = -26, //!< Thread or thread context creation failure
BLOSC2_ERROR_POSTFILTER = -27, //!< Postfilter failure
BLOSC2_ERROR_FRAME_SPECIAL = -28, //!< Special frame failure
BLOSC2_ERROR_SCHUNK_SPECIAL = -29, //!< Special super-chunk failure
BLOSC2_ERROR_PLUGIN_IO = -30, //!< IO plugin error
BLOSC2_ERROR_FILE_REMOVE = -31, //!< Remove file failure
BLOSC2_ERROR_NULL_POINTER = -32, //!< Pointer is null
BLOSC2_ERROR_INVALID_INDEX = -33, //!< Invalid index
BLOSC2_ERROR_METALAYER_NOT_FOUND = -34, //!< Metalayer has not been found
BLOSC2_ERROR_MAX_BUFSIZE_EXCEEDED = -35, //!< Max buffer size exceeded
BLOSC2_ERROR_TUNER = -36, //!< Tuner failure
};
#ifdef __GNUC__
#define BLOSC_ATTRIBUTE_UNUSED __attribute__((unused))
#else
#define BLOSC_ATTRIBUTE_UNUSED
#endif
static char *print_error(int rc) BLOSC_ATTRIBUTE_UNUSED;
static char *print_error(int rc) {
switch (rc) {
case BLOSC2_ERROR_FAILURE:
return (char *) "Generic failure";
case BLOSC2_ERROR_STREAM:
return (char *) "Bad stream";
case BLOSC2_ERROR_DATA:
return (char *) "Invalid data";
case BLOSC2_ERROR_MEMORY_ALLOC:
return (char *) "Memory alloc/realloc failure";
case BLOSC2_ERROR_READ_BUFFER:
return (char *) "Not enough space to read";
case BLOSC2_ERROR_WRITE_BUFFER:
return (char *) "Not enough space to write";
case BLOSC2_ERROR_CODEC_SUPPORT:
return (char *) "Codec not supported";
case BLOSC2_ERROR_CODEC_PARAM:
return (char *) "Invalid parameter supplied to codec";
case BLOSC2_ERROR_CODEC_DICT:
return (char *) "Codec dictionary error";
case BLOSC2_ERROR_VERSION_SUPPORT:
return (char *) "Version not supported";
case BLOSC2_ERROR_INVALID_HEADER:
return (char *) "Invalid value in header";
case BLOSC2_ERROR_INVALID_PARAM:
return (char *) "Invalid parameter supplied to function";
case BLOSC2_ERROR_FILE_READ:
return (char *) "File read failure";
case BLOSC2_ERROR_FILE_WRITE:
return (char *) "File write failure";
case BLOSC2_ERROR_FILE_OPEN:
return (char *) "File open failure";
case BLOSC2_ERROR_NOT_FOUND:
return (char *) "Not found";
case BLOSC2_ERROR_RUN_LENGTH:
return (char *) "Bad run length encoding";
case BLOSC2_ERROR_FILTER_PIPELINE:
return (char *) "Filter pipeline error";
case BLOSC2_ERROR_CHUNK_INSERT:
return (char *) "Chunk insert failure";
case BLOSC2_ERROR_CHUNK_APPEND:
return (char *) "Chunk append failure";
case BLOSC2_ERROR_CHUNK_UPDATE:
return (char *) "Chunk update failure";
case BLOSC2_ERROR_2GB_LIMIT:
return (char *) "Sizes larger than 2gb not supported";
case BLOSC2_ERROR_SCHUNK_COPY:
return (char *) "Super-chunk copy failure";
case BLOSC2_ERROR_FRAME_TYPE:
return (char *) "Wrong type for frame";
case BLOSC2_ERROR_FILE_TRUNCATE:
return (char *) "File truncate failure";
case BLOSC2_ERROR_THREAD_CREATE:
return (char *) "Thread or thread context creation failure";
case BLOSC2_ERROR_POSTFILTER:
return (char *) "Postfilter failure";
case BLOSC2_ERROR_FRAME_SPECIAL:
return (char *) "Special frame failure";
case BLOSC2_ERROR_SCHUNK_SPECIAL:
return (char *) "Special super-chunk failure";
case BLOSC2_ERROR_PLUGIN_IO:
return (char *) "IO plugin error";
case BLOSC2_ERROR_FILE_REMOVE:
return (char *) "Remove file failure";
case BLOSC2_ERROR_NULL_POINTER:
return (char *) "Pointer is null";
case BLOSC2_ERROR_INVALID_INDEX:
return (char *) "Invalid index";
case BLOSC2_ERROR_METALAYER_NOT_FOUND:
return (char *) "Metalayer has not been found";
case BLOSC2_ERROR_MAX_BUFSIZE_EXCEEDED:
return (char *) "Maximum buffersize exceeded";
default:
return (char *) "Unknown error";
}
}
/**
* @brief Initialize the Blosc library environment.
*
* You must call this previous to any other Blosc call, unless you want
* Blosc to be used simultaneously in a multi-threaded environment, in
* which case you can use the #blosc2_compress_ctx #blosc2_decompress_ctx pair.
*
* @sa #blosc2_destroy
*/
BLOSC_EXPORT void blosc2_init(void);
/**
* @brief Destroy the Blosc library environment.
*
* You must call this after to you are done with all the Blosc calls,
* unless you have not used blosc2_init() before.
*
* @sa #blosc2_init
*/
BLOSC_EXPORT void blosc2_destroy(void);
/**
* @brief Compress a block of data in the @p src buffer and returns the size of
* compressed block.
*
* @remark Compression is memory safe and guaranteed not to write @p dest
* more than what is specified in @p destsize.
* There is not a minimum for @p src buffer size @p nbytes.
* Equivalent to #blosc2_compress.
*
* @warning The @p src buffer and the @p dest buffer can not overlap.
*
* @param clevel The desired compression level and must be a number
* between 0 (no compression) and 9 (maximum compression).
* @param doshuffle Specifies whether the shuffle compression preconditioner
* should be applied or not. #BLOSC_NOFILTER means not applying filters,
* #BLOSC_SHUFFLE means applying shuffle at a byte level and
* #BLOSC_BITSHUFFLE at a bit level (slower but *may* achieve better
* compression).
* @param typesize Is the number of bytes for the atomic type in binary
* @p src buffer. This is mainly useful for the shuffle preconditioner.
* For implementation reasons, only a 1 < typesize < 256 will allow the
* shuffle filter to work. When typesize is not in this range, shuffle
* will be silently disabled.
* @param nbytes The number of bytes to compress in the @p src buffer.
* @param src The buffer containing the data to compress.
* @param dest The buffer where the compressed data will be put,
* must have at least the size of @p destsize.
* @param destsize The size of the dest buffer. Blosc
* guarantees that if you set @p destsize to, at least,
* (@p nbytes + #BLOSC2_MAX_OVERHEAD), the compression will always succeed.
*
* @return The number of bytes compressed.
* If @p src buffer cannot be compressed into @p destsize, the return
* value is zero and you should discard the contents of the @p dest
* buffer. A negative return value means that an internal error happened. This
* should never happen. If you see this, please report it back
* together with the buffer data causing this and compression settings.
*
*
* @par Environment variables
* @parblock
*
* This function honors different environment variables to control
* internal parameters without the need of doing that programmatically.
* Here are the ones supported:
*
* * **BLOSC_CLEVEL=(INTEGER)**: This will overwrite the @p clevel parameter
* before the compression process starts.
*
* * **BLOSC_SHUFFLE=[NOSHUFFLE | SHUFFLE | BITSHUFFLE]**: This will
* overwrite the @p doshuffle parameter before the compression process
* starts.
*
* * **BLOSC_DELTA=(1|0)**: This will call #blosc2_set_delta() before the
* compression process starts.
*
* * **BLOSC_TYPESIZE=(INTEGER)**: This will overwrite the @p typesize
* parameter before the compression process starts.
*
* * **BLOSC_COMPRESSOR=[BLOSCLZ | LZ4 | LZ4HC | ZLIB | ZSTD]**:
* This will call #blosc1_set_compressor before the compression process starts.
*
* * **BLOSC_NTHREADS=(INTEGER)**: This will call
* #blosc2_set_nthreads before the compression process starts.
*
* * **BLOSC_SPLITMODE=(ALWAYS | NEVER | AUTO | FORWARD_COMPAT)**:
* This will call #blosc1_set_splitmode() before the compression process starts.
*
* * **BLOSC_BLOCKSIZE=(INTEGER)**: This will call
* #blosc1_set_blocksize before the compression process starts.
* *NOTE:* The *blocksize* is a critical parameter with
* important restrictions in the allowed values, so use this with care.
*
* * **BLOSC_NOLOCK=(ANY VALUE)**: This will call #blosc2_compress_ctx under
* the hood, with the *compressor*, *blocksize* and
* *numinternalthreads* parameters set to the same as the last calls to
* #blosc1_set_compressor, #blosc1_set_blocksize and
* #blosc2_set_nthreads. *BLOSC_CLEVEL*, *BLOSC_SHUFFLE*, *BLOSC_DELTA* and
* *BLOSC_TYPESIZE* environment vars will also be honored.
*
* @endparblock
*
* @sa #blosc1_decompress
*/
BLOSC_EXPORT int blosc1_compress(int clevel, int doshuffle, size_t typesize,
size_t nbytes, const void* src, void* dest,
size_t destsize);
/**
* @brief Decompress a block of compressed data in @p src, put the result in
* @p dest and returns the size of the decompressed block.
*
* @warning The @p src buffer and the @p dest buffer can not overlap.
*
* @remark Decompression is memory safe and guaranteed not to write the @p dest
* buffer more than what is specified in @p destsize.
* Similar to #blosc2_decompress.
*
* @remark In case you want to keep under control the number of bytes read from
* source, you can call #blosc1_cbuffer_sizes first to check whether the
* @p nbytes (i.e. the number of bytes to be read from @p src buffer by this
* function) in the compressed buffer is ok with you.
*
* @param src The buffer to be decompressed.
* @param dest The buffer where the decompressed data will be put.
* @param destsize The size of the @p dest buffer.
*
* @return The number of bytes decompressed.
* If an error occurs, e.g. the compressed data is corrupted or the
* output buffer is not large enough, then a negative value
* will be returned instead.
*
* @par Environment variables
* @parblock
* This function honors different environment variables to control
* internal parameters without the need of doing that programmatically.
* Here are the ones supported:
*
* * **BLOSC_NTHREADS=(INTEGER)**: This will call
* #blosc2_set_nthreads before the proper decompression
* process starts.
*
* * **BLOSC_NOLOCK=(ANY VALUE)**: This will call #blosc2_decompress_ctx
* under the hood, with the *numinternalthreads* parameter set to the
* same value as the last call to #blosc2_set_nthreads.
*
* @endparblock
*
* @sa #blosc1_compress
*/
BLOSC_EXPORT int blosc1_decompress(const void* src, void* dest, size_t destsize);
/**
* @brief Get @p nitems (of @p typesize size) in @p src buffer starting in @p start.
* The items are returned in @p dest buffer, which has to have enough
* space for storing all items.
*
* @remark The function #blosc2_getitem is a more complete and secure version.
*
* @param src The compressed buffer from data will be decompressed.
* @param start The position of the first item (of @p typesize size) from where data
* will be retrieved.
* @param nitems The number of items (of @p typesize size) that will be retrieved.
* @param dest The buffer where the decompressed data retrieved will be put.
*
* @return The number of bytes copied to @p dest or a negative value if
* some error happens.
*/
BLOSC_EXPORT int blosc1_getitem(const void* src, int start, int nitems, void* dest);
/**
* @brief Get @p nitems (of @p typesize size) in @p src buffer starting in @p start.
* The items are returned in @p dest buffer. The dest buffer should have enough space
* for storing all items. This function is a more secure version of #blosc1_getitem.
*
* @param src The compressed buffer holding the data to be retrieved.
* @param srcsize Size of the compressed buffer.
* @param start The position of the first item (of @p typesize size) from where data
* will be retrieved.
* @param nitems The number of items (of @p typesize size) that will be retrieved.
* @param dest The buffer where the retrieved data will be stored decompressed.
* @param destsize Size of the buffer where retrieved data will be stored.
*
* @return The number of bytes copied to @p dest or a negative value if
* some error happens.
*/
BLOSC_EXPORT int blosc2_getitem(const void* src, int32_t srcsize, int start, int nitems,
void* dest, int32_t destsize);
/**
Pointer to a callback function that executes `dojob(jobdata + i*jobdata_elsize)` for `i = 0 to numjobs-1`,
possibly in parallel threads (but not returning until all `dojob` calls have returned). This allows the
caller to provide a custom threading backend as an alternative to the default Blosc-managed threads.
`callback_data` is passed through from `blosc2_set_threads_callback`.
*/
typedef void (*blosc_threads_callback)(void *callback_data, void (*dojob)(void *), int numjobs, size_t jobdata_elsize, void *jobdata);
/**
* @brief Set the threading backend for parallel compression/decompression to use @p callback to execute work
* instead of using the Blosc-managed threads. The @p callback_data argument is passed through to the callback.
* @param callback: the callback to use. Passing `NULL` uses the default Blosc threading backend.
* @param callback_data: the callback data.
*
* @warning This function is *not* thread-safe and should be called
* before any other Blosc function: it affects all Blosc contexts.
* @sa https://github.com/Blosc/c-blosc2/pull/81
*/
BLOSC_EXPORT void blosc2_set_threads_callback(blosc_threads_callback callback, void *callback_data);
/**
* @brief Returns the current number of threads that are used for
* compression/decompression.
*/
BLOSC_EXPORT int16_t blosc2_get_nthreads(void);
/**
* @brief Initialize a pool of threads for compression/decompression. If
* @p nthreads is 1, then the serial version is chosen and a possible
* previous existing pool is ended. If this is not called, @p nthreads
* is set to 1 internally.
*
* @param nthreads The number of threads to use.
*
* @return The previous number of threads.
*/
BLOSC_EXPORT int16_t blosc2_set_nthreads(int16_t nthreads);
/**
* @brief Get the current compressor that is used for compression.
*
* @return The string identifying the compressor being used.
*/
BLOSC_EXPORT const char* blosc1_get_compressor(void);
/**
* @brief Select the compressor to be used. The supported ones are "blosclz",
* "lz4", "lz4hc", "zlib" and "ztsd". If this function is not
* called, then "blosclz" will be used.
*
* @param compname The name identifier of the compressor to be set.
*
* @return The code for the compressor (>=0). In case the compressor
* is not recognized, or there is not support for it in this build,
* it returns a -1.
*/
BLOSC_EXPORT int blosc1_set_compressor(const char* compname);
/**
* @brief Select the delta coding filter to be used.
*
* @param dodelta A value >0 will activate the delta filter.
* If 0, it will be de-activated
*
* This call should always succeed.
*/
BLOSC_EXPORT void blosc2_set_delta(int dodelta);
/**
* @brief Get the compressor name associated with the compressor code.
*
* @param compcode The code identifying the compressor
* @param compname The pointer to a string where the compressor name will be put.
*
* @return The compressor code. If the compressor code is not recognized,
* or there is not support for it in this build, -1 is returned.
*/
BLOSC_EXPORT int blosc2_compcode_to_compname(int compcode, const char** compname);
/**
* @brief Get the compressor code associated with the compressor name.
*
* @param compname The string containing the compressor name.
*
* @return The compressor code. If the compressor name is not recognized,
* or there is not support for it in this build, -1 is returned instead.
*/
BLOSC_EXPORT int blosc2_compname_to_compcode(const char* compname);
/**
* @brief Get a list of compressors supported in the current build.
*
* @return The comma separated string with the list of compressor names
* supported.
*
* This function does not leak, so you should not free() the returned
* list.
*
* This function should always succeed.
*/
BLOSC_EXPORT const char* blosc2_list_compressors(void);
/**
* @brief Get the version of Blosc in string format.
*
* @return The string with the current Blosc version.
* Useful for dynamic libraries.
*/
BLOSC_EXPORT const char* blosc2_get_version_string(void);
/**
* @brief Get info from compression libraries included in the current build.
*
* @param compname The compressor name that you want info from.
* @param complib The pointer to a string where the
* compression library name, if available, will be put.
* @param version The pointer to a string where the
* compression library version, if available, will be put.
*
* @warning You are in charge of the @p complib and @p version strings,
* you should free() them so as to avoid leaks.
*
* @return The code for the compression library (>=0). If it is not supported,
* this function returns -1.
*/
BLOSC_EXPORT int blosc2_get_complib_info(const char* compname, char** complib,
char** version);
/**
* @brief Free possible memory temporaries and thread resources. Use this
* when you are not going to use Blosc for a long while.
*
* @return A 0 if succeeds, in case of problems releasing the resources,
* it returns a negative number.
*/
BLOSC_EXPORT int blosc2_free_resources(void);
/**
* @brief Get information about a compressed buffer, namely the number of
* uncompressed bytes (@p nbytes) and compressed (@p cbytes). It also
* returns the @p blocksize (which is used internally for doing the
* compression by blocks).
*
* @remark Equivalent to function #blosc2_cbuffer_sizes.
*
* @param cbuffer The buffer of compressed data.
* @param nbytes The pointer where the number of uncompressed bytes will be put.
* @param cbytes The pointer where the number of compressed bytes will be put.
* @param blocksize The pointer where the block size will be put.
*
* You only need to pass the first BLOSC_MIN_HEADER_LENGTH bytes of a
* compressed buffer for this call to work.
*
* This function should always succeed.
*/
BLOSC_EXPORT void blosc1_cbuffer_sizes(const void* cbuffer, size_t* nbytes,
size_t* cbytes, size_t* blocksize);
/**
* @brief Get information about a compressed buffer, namely the number of
* uncompressed bytes (@p nbytes) and compressed (@p cbytes). It also
* returns the @p blocksize (which is used internally for doing the
* compression by blocks).
*
* @param cbuffer The buffer of compressed data.
* @param nbytes The pointer where the number of uncompressed bytes will be put.
* @param cbytes The pointer where the number of compressed bytes will be put.
* @param blocksize The pointer where the block size will be put.
*
* @note: if any of the nbytes, cbytes or blocksize is NULL, it will not be returned.
*
* You only need to pass the first BLOSC_MIN_HEADER_LENGTH bytes of a
* compressed buffer for this call to work.
*
* @return On failure, returns negative value.
*/
BLOSC_EXPORT int blosc2_cbuffer_sizes(const void* cbuffer, int32_t* nbytes,
int32_t* cbytes, int32_t* blocksize);
/**
* @brief Checks that the compressed buffer starting at @p cbuffer of length @p cbytes
* may contain valid blosc compressed data, and that it is safe to call
* blosc1_decompress/blosc1_getitem.
* On success, returns 0 and sets @p nbytes to the size of the uncompressed data.
* This does not guarantee that the decompression function won't return an error,
* but does guarantee that it is safe to attempt decompression.
*
* @param cbuffer The buffer of compressed data.
* @param cbytes The number of compressed bytes.
* @param nbytes The pointer where the number of uncompressed bytes will be put.
*
* @return On failure, returns negative value.
*/
BLOSC_EXPORT int blosc1_cbuffer_validate(const void* cbuffer, size_t cbytes,
size_t* nbytes);
/**
* @brief Get information about a compressed buffer, namely the type size
* (@p typesize), as well as some internal @p flags.
*
* @param cbuffer The buffer of compressed data.
* @param typesize The pointer where the type size will be put.
* @param flags The pointer of the integer where the additional info is encoded.
* The @p flags is a set of bits, where the currently used ones are:
* * bit 0: whether the shuffle filter has been applied or not
* * bit 1: whether the internal buffer is a pure memcpy or not
* * bit 2: whether the bitshuffle filter has been applied or not
* * bit 3: whether the delta coding filter has been applied or not
*
* You can use the @p BLOSC_DOSHUFFLE, @p BLOSC_DOBITSHUFFLE, @p BLOSC_DODELTA
* and @p BLOSC_MEMCPYED symbols for extracting the interesting bits
* (e.g. @p flags & @p BLOSC_DOSHUFFLE says whether the buffer is byte-shuffled
* or not).
*
* This function should always succeed.
*/
BLOSC_EXPORT void blosc1_cbuffer_metainfo(const void* cbuffer, size_t* typesize,
int* flags);
/**
* @brief Get information about a compressed buffer, namely the internal
* Blosc format version (@p version) and the format for the internal
* Lempel-Ziv compressor used (@p versionlz).
*
* @param cbuffer The buffer of compressed data.