From 99614718ebb56e2d4e5b3de3620ba1e384b528dd Mon Sep 17 00:00:00 2001 From: Vivian Nguyen Date: Tue, 9 Apr 2024 10:54:41 -0500 Subject: [PATCH 01/10] [c++] Addition of ArrowSchema to TileDB ArraySchema Converter --- .../external/include/nanoarrow/nanoarrow.h | 1274 ++++++++++------- libtiledbsoma/src/utils/arrow_adapter.cc | 238 ++- libtiledbsoma/src/utils/arrow_adapter.h | 59 +- 3 files changed, 962 insertions(+), 609 deletions(-) diff --git a/libtiledbsoma/src/external/include/nanoarrow/nanoarrow.h b/libtiledbsoma/src/external/include/nanoarrow/nanoarrow.h index e338560f1a..fa99f12f25 100644 --- a/libtiledbsoma/src/external/include/nanoarrow/nanoarrow.h +++ b/libtiledbsoma/src/external/include/nanoarrow/nanoarrow.h @@ -53,8 +53,6 @@ #include #include - - #if defined(NANOARROW_DEBUG) && !defined(NANOARROW_PRINT_AND_DIE) #include #include @@ -70,11 +68,11 @@ extern "C" { /// \defgroup nanoarrow-arrow-cdata Arrow C Data interface /// /// The Arrow C Data (https://arrow.apache.org/docs/format/CDataInterface.html) -/// and Arrow C Stream (https://arrow.apache.org/docs/format/CStreamInterface.html) -/// interfaces are part of the -/// Arrow Columnar Format specification -/// (https://arrow.apache.org/docs/format/Columnar.html). See the Arrow documentation for -/// documentation of these structures. +/// and Arrow C Stream +/// (https://arrow.apache.org/docs/format/CStreamInterface.html) interfaces are +/// part of the Arrow Columnar Format specification +/// (https://arrow.apache.org/docs/format/Columnar.html). See the Arrow +/// documentation for documentation of these structures. /// /// @{ @@ -129,7 +127,8 @@ struct ArrowArrayStream { // // Return value: 0 if successful, an `errno`-compatible error code otherwise. // - // If successful, the ArrowSchema must be released independently from the stream. + // If successful, the ArrowSchema must be released independently from the + // stream. int (*get_schema)(struct ArrowArrayStream*, struct ArrowSchema* out); // Callback to get the next array @@ -137,7 +136,8 @@ struct ArrowArrayStream { // // Return value: 0 if successful, an `errno`-compatible error code otherwise. // - // If successful, the ArrowArray must be released independently from the stream. + // If successful, the ArrowArray must be released independently from the + // stream. int (*get_next)(struct ArrowArrayStream*, struct ArrowArray* out); // Callback to get optional detailed error information. @@ -171,7 +171,8 @@ struct ArrowArrayStream { #define _NANOARROW_RETURN_NOT_OK_IMPL(NAME, EXPR) \ do { \ const int NAME = (EXPR); \ - if (NAME) return NAME; \ + if (NAME) \ + return NAME; \ } while (0) #define _NANOARROW_CHECK_RANGE(x_, min_, max_) \ @@ -181,29 +182,38 @@ struct ArrowArrayStream { NANOARROW_RETURN_NOT_OK((x_ <= max_) ? NANOARROW_OK : EINVAL) #if defined(NANOARROW_DEBUG) -#define _NANOARROW_RETURN_NOT_OK_WITH_ERROR_IMPL(NAME, EXPR, ERROR_PTR_EXPR, EXPR_STR) \ - do { \ - const int NAME = (EXPR); \ - if (NAME) { \ - ArrowErrorSet((ERROR_PTR_EXPR), "%s failed with errno %d\n* %s:%d", EXPR_STR, \ - NAME, __FILE__, __LINE__); \ - return NAME; \ - } \ +#define _NANOARROW_RETURN_NOT_OK_WITH_ERROR_IMPL( \ + NAME, EXPR, ERROR_PTR_EXPR, EXPR_STR) \ + do { \ + const int NAME = (EXPR); \ + if (NAME) { \ + ArrowErrorSet( \ + (ERROR_PTR_EXPR), \ + "%s failed with errno %d\n* %s:%d", \ + EXPR_STR, \ + NAME, \ + __FILE__, \ + __LINE__); \ + return NAME; \ + } \ } while (0) #else -#define _NANOARROW_RETURN_NOT_OK_WITH_ERROR_IMPL(NAME, EXPR, ERROR_PTR_EXPR, EXPR_STR) \ - do { \ - const int NAME = (EXPR); \ - if (NAME) { \ - ArrowErrorSet((ERROR_PTR_EXPR), "%s failed with errno %d", EXPR_STR, NAME); \ - return NAME; \ - } \ +#define _NANOARROW_RETURN_NOT_OK_WITH_ERROR_IMPL( \ + NAME, EXPR, ERROR_PTR_EXPR, EXPR_STR) \ + do { \ + const int NAME = (EXPR); \ + if (NAME) { \ + ArrowErrorSet( \ + (ERROR_PTR_EXPR), "%s failed with errno %d", EXPR_STR, NAME); \ + return NAME; \ + } \ } while (0) #endif #if defined(NANOARROW_DEBUG) // For checking ArrowErrorSet() calls for valid printf format strings/arguments -// If using mingw's c99-compliant printf, we need a different format-checking attribute +// If using mingw's c99-compliant printf, we need a different format-checking +// attribute #if defined(__USE_MINGW_ANSI_STDIO) && defined(__MINGW_PRINTF_FORMAT) #define NANOARROW_CHECK_PRINTF_ATTRIBUTE \ __attribute__((format(__MINGW_PRINTF_FORMAT, 2, 3))) @@ -243,8 +253,9 @@ typedef int ArrowErrorCode; /// \brief Flags supported by ArrowSchemaViewInit() /// \ingroup nanoarrow-schema-view -#define NANOARROW_FLAG_ALL_SUPPORTED \ - (ARROW_FLAG_DICTIONARY_ORDERED | ARROW_FLAG_NULLABLE | ARROW_FLAG_MAP_KEYS_SORTED) +#define NANOARROW_FLAG_ALL_SUPPORTED \ + (ARROW_FLAG_DICTIONARY_ORDERED | ARROW_FLAG_NULLABLE | \ + ARROW_FLAG_MAP_KEYS_SORTED) /// \brief Error type containing a UTF-8 encoded message. /// \ingroup nanoarrow-errors @@ -253,8 +264,8 @@ struct ArrowError { char message[1024]; }; -/// \brief Ensure an ArrowError is null-terminated by zeroing the first character. -/// \ingroup nanoarrow-errors +/// \brief Ensure an ArrowError is null-terminated by zeroing the first +/// character. \ingroup nanoarrow-errors /// /// If error is NULL, this function does nothing. static inline void ArrowErrorInit(struct ArrowError* error) { @@ -280,7 +291,8 @@ static inline const char* ArrowErrorMessage(struct ArrowError* error) { /// \ingroup nanoarrow-errors /// /// If error is NULL, this function does nothing. -static inline void ArrowErrorSetString(struct ArrowError* error, const char* src) { +static inline void ArrowErrorSetString( + struct ArrowError* error, const char* src) { if (error == NULL) { return; } @@ -298,25 +310,34 @@ static inline void ArrowErrorSetString(struct ArrowError* error, const char* src /// \brief Check the result of an expression and return it if not NANOARROW_OK /// \ingroup nanoarrow-errors #define NANOARROW_RETURN_NOT_OK(EXPR) \ - _NANOARROW_RETURN_NOT_OK_IMPL(_NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), EXPR) + _NANOARROW_RETURN_NOT_OK_IMPL( \ + _NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), EXPR) /// \brief Check the result of an expression and return it if not NANOARROW_OK, /// adding an auto-generated message to an ArrowError. /// \ingroup nanoarrow-errors /// /// This macro is used to ensure that functions that accept an ArrowError -/// as input always set its message when returning an error code (e.g., when calling -/// a nanoarrow function that does *not* accept ArrowError). +/// as input always set its message when returning an error code (e.g., when +/// calling a nanoarrow function that does *not* accept ArrowError). #define NANOARROW_RETURN_NOT_OK_WITH_ERROR(EXPR, ERROR_EXPR) \ _NANOARROW_RETURN_NOT_OK_WITH_ERROR_IMPL( \ - _NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), EXPR, ERROR_EXPR, #EXPR) + _NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), \ + EXPR, \ + ERROR_EXPR, \ + #EXPR) #if defined(NANOARROW_DEBUG) && !defined(NANOARROW_PRINT_AND_DIE) -#define NANOARROW_PRINT_AND_DIE(VALUE, EXPR_STR) \ - do { \ - fprintf(stderr, "%s failed with code %d\n* %s:%d\n", EXPR_STR, (int)(VALUE), \ - __FILE__, (int)__LINE__); \ - abort(); \ +#define NANOARROW_PRINT_AND_DIE(VALUE, EXPR_STR) \ + do { \ + fprintf( \ + stderr, \ + "%s failed with code %d\n* %s:%d\n", \ + EXPR_STR, \ + (int)(VALUE), \ + __FILE__, \ + (int)__LINE__); \ + abort(); \ } while (0) #endif @@ -324,23 +345,27 @@ static inline void ArrowErrorSetString(struct ArrowError* error, const char* src #define _NANOARROW_ASSERT_OK_IMPL(NAME, EXPR, EXPR_STR) \ do { \ const int NAME = (EXPR); \ - if (NAME) NANOARROW_PRINT_AND_DIE(NAME, EXPR_STR); \ + if (NAME) \ + NANOARROW_PRINT_AND_DIE(NAME, EXPR_STR); \ } while (0) /// \brief Assert that an expression's value is NANOARROW_OK /// \ingroup nanoarrow-errors /// -/// If nanoarrow was built in debug mode (i.e., defined(NANOARROW_DEBUG) is true), -/// print a message to stderr and abort. If nanoarrow was built in release mode, -/// this statement has no effect. You can customize fatal error behaviour -/// be defining the NANOARROW_PRINT_AND_DIE macro before including nanoarrow.h -/// This macro is provided as a convenience for users and is not used internally. +/// If nanoarrow was built in debug mode (i.e., defined(NANOARROW_DEBUG) is +/// true), print a message to stderr and abort. If nanoarrow was built in +/// release mode, this statement has no effect. You can customize fatal error +/// behaviour be defining the NANOARROW_PRINT_AND_DIE macro before including +/// nanoarrow.h This macro is provided as a convenience for users and is not +/// used internally. #define NANOARROW_ASSERT_OK(EXPR) \ - _NANOARROW_ASSERT_OK_IMPL(_NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), EXPR, #EXPR) + _NANOARROW_ASSERT_OK_IMPL( \ + _NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), EXPR, #EXPR) -#define _NANOARROW_DCHECK_IMPL(EXPR, EXPR_STR) \ - do { \ - if (!(EXPR)) NANOARROW_PRINT_AND_DIE(-1, EXPR_STR); \ +#define _NANOARROW_DCHECK_IMPL(EXPR, EXPR_STR) \ + do { \ + if (!(EXPR)) \ + NANOARROW_PRINT_AND_DIE(-1, EXPR_STR); \ } while (0) #define NANOARROW_DCHECK(EXPR) _NANOARROW_DCHECK_IMPL(EXPR, #EXPR) @@ -349,7 +374,8 @@ static inline void ArrowErrorSetString(struct ArrowError* error, const char* src #define NANOARROW_DCHECK(EXPR) #endif -static inline void ArrowSchemaMove(struct ArrowSchema* src, struct ArrowSchema* dst) { +static inline void ArrowSchemaMove( + struct ArrowSchema* src, struct ArrowSchema* dst) { NANOARROW_DCHECK(src != NULL); NANOARROW_DCHECK(dst != NULL); @@ -363,7 +389,8 @@ static inline void ArrowSchemaRelease(struct ArrowSchema* schema) { NANOARROW_DCHECK(schema->release == NULL); } -static inline void ArrowArrayMove(struct ArrowArray* src, struct ArrowArray* dst) { +static inline void ArrowArrayMove( + struct ArrowArray* src, struct ArrowArray* dst) { NANOARROW_DCHECK(src != NULL); NANOARROW_DCHECK(dst != NULL); @@ -377,8 +404,8 @@ static inline void ArrowArrayRelease(struct ArrowArray* array) { NANOARROW_DCHECK(array->release == NULL); } -static inline void ArrowArrayStreamMove(struct ArrowArrayStream* src, - struct ArrowArrayStream* dst) { +static inline void ArrowArrayStreamMove( + struct ArrowArrayStream* src, struct ArrowArrayStream* dst) { NANOARROW_DCHECK(src != NULL); NANOARROW_DCHECK(dst != NULL); @@ -399,7 +426,8 @@ static inline const char* ArrowArrayStreamGetLastError( } static inline ArrowErrorCode ArrowArrayStreamGetSchema( - struct ArrowArrayStream* array_stream, struct ArrowSchema* out, + struct ArrowArrayStream* array_stream, + struct ArrowSchema* out, struct ArrowError* error) { NANOARROW_DCHECK(array_stream != NULL); @@ -412,7 +440,8 @@ static inline ArrowErrorCode ArrowArrayStreamGetSchema( } static inline ArrowErrorCode ArrowArrayStreamGetNext( - struct ArrowArrayStream* array_stream, struct ArrowArray* out, + struct ArrowArrayStream* array_stream, + struct ArrowArray* out, struct ArrowError* error) { NANOARROW_DCHECK(array_stream != NULL); @@ -424,7 +453,8 @@ static inline ArrowErrorCode ArrowArrayStreamGetNext( return result; } -static inline void ArrowArrayStreamRelease(struct ArrowArrayStream* array_stream) { +static inline void ArrowArrayStreamRelease( + struct ArrowArrayStream* array_stream) { NANOARROW_DCHECK(array_stream != NULL); array_stream->release(array_stream); NANOARROW_DCHECK(array_stream->release == NULL); @@ -592,16 +622,21 @@ enum ArrowValidationLevel { /// \brief Do not validate buffer sizes or content. NANOARROW_VALIDATION_LEVEL_NONE = 0, - /// \brief Validate buffer sizes that depend on array length but do not validate buffer + /// \brief Validate buffer sizes that depend on array length but do not + /// validate buffer /// sizes that depend on buffer data access. NANOARROW_VALIDATION_LEVEL_MINIMAL = 1, - /// \brief Validate all buffer sizes, including those that require buffer data access, - /// but do not perform any checks that are O(1) along the length of the buffers. + /// \brief Validate all buffer sizes, including those that require buffer data + /// access, + /// but do not perform any checks that are O(1) along the length of the + /// buffers. NANOARROW_VALIDATION_LEVEL_DEFAULT = 2, - /// \brief Validate all buffer sizes and all buffer content. This is useful in the - /// context of untrusted input or input that may have been corrupted in transit. + /// \brief Validate all buffer sizes and all buffer content. This is useful in + /// the + /// context of untrusted input or input that may have been corrupted in + /// transit. NANOARROW_VALIDATION_LEVEL_FULL = 3 }; @@ -626,8 +661,8 @@ static inline const char* ArrowTimeUnitString(enum ArrowTimeUnit time_unit) { } } -/// \brief Functional types of buffers as described in the Arrow Columnar Specification -/// \ingroup nanoarrow-array-view +/// \brief Functional types of buffers as described in the Arrow Columnar +/// Specification \ingroup nanoarrow-array-view enum ArrowBufferType { NANOARROW_BUFFER_TYPE_NONE, NANOARROW_BUFFER_TYPE_VALIDITY, @@ -642,7 +677,8 @@ enum ArrowBufferType { /// /// All currently supported types have 3 buffers or fewer; however, future types /// may involve a variable number of buffers (e.g., string view). These buffers -/// will be represented by separate members of the ArrowArrayView or ArrowLayout. +/// will be represented by separate members of the ArrowArrayView or +/// ArrowLayout. #define NANOARROW_MAX_FIXED_BUFFERS 3 /// \brief An non-owning view of a string @@ -711,11 +747,15 @@ struct ArrowBufferView { /// an ArrowArray. struct ArrowBufferAllocator { /// \brief Reallocate a buffer or return NULL if it cannot be reallocated - uint8_t* (*reallocate)(struct ArrowBufferAllocator* allocator, uint8_t* ptr, - int64_t old_size, int64_t new_size); + uint8_t* (*reallocate)( + struct ArrowBufferAllocator* allocator, + uint8_t* ptr, + int64_t old_size, + int64_t new_size); /// \brief Deallocate a buffer allocated by this allocator - void (*free)(struct ArrowBufferAllocator* allocator, uint8_t* ptr, int64_t size); + void (*free)( + struct ArrowBufferAllocator* allocator, uint8_t* ptr, int64_t size); /// \brief Opaque data specific to the allocator void* private_data; @@ -735,7 +775,8 @@ struct ArrowBuffer { /// \brief The capacity of the buffer in bytes int64_t capacity_bytes; - /// \brief The allocator that will be used to reallocate and/or free the buffer + /// \brief The allocator that will be used to reallocate and/or free the + /// buffer struct ArrowBufferAllocator allocator; }; @@ -762,11 +803,12 @@ struct ArrowLayout { /// \brief The data type of each buffer enum ArrowType buffer_data_type[NANOARROW_MAX_FIXED_BUFFERS]; - /// \brief The size of an element each buffer or 0 if this size is variable or unknown + /// \brief The size of an element each buffer or 0 if this size is variable or + /// unknown int64_t element_size_bits[NANOARROW_MAX_FIXED_BUFFERS]; - /// \brief The number of elements in the child array per element in this array for a - /// fixed-size list + /// \brief The number of elements in the child array per element in this array + /// for a fixed-size list int64_t child_size_elements; }; @@ -869,8 +911,8 @@ struct ArrowInterval { /// \brief Zero initialize an Interval with a given unit /// \ingroup nanoarrow-utils -static inline void ArrowIntervalInit(struct ArrowInterval* interval, - enum ArrowType type) { +static inline void ArrowIntervalInit( + struct ArrowInterval* interval, enum ArrowType type) { memset(interval, 0, sizeof(struct ArrowInterval)); interval->type = type; } @@ -882,7 +924,8 @@ static inline void ArrowIntervalInit(struct ArrowInterval* interval, /// values set using ArrowDecimalSetInt(), ArrowDecimalSetBytes128(), /// or ArrowDecimalSetBytes256(). struct ArrowDecimal { - /// \brief An array of 64-bit integers of n_words length defined in native-endian order + /// \brief An array of 64-bit integers of n_words length defined in + /// native-endian order uint64_t words[4]; /// \brief The number of significant digits this decimal number can represent @@ -903,8 +946,11 @@ struct ArrowDecimal { /// \brief Initialize a decimal with a given set of type parameters /// \ingroup nanoarrow-utils -static inline void ArrowDecimalInit(struct ArrowDecimal* decimal, int32_t bitwidth, - int32_t precision, int32_t scale) { +static inline void ArrowDecimalInit( + struct ArrowDecimal* decimal, + int32_t bitwidth, + int32_t precision, + int32_t scale) { memset(decimal->words, 0, sizeof(decimal->words)); decimal->precision = precision; decimal->scale = scale; @@ -924,14 +970,15 @@ static inline void ArrowDecimalInit(struct ArrowDecimal* decimal, int32_t bitwid /// This does not check if the decimal's precision sufficiently small to fit /// within the signed 64-bit integer range (A precision less than or equal /// to 18 is sufficiently small). -static inline int64_t ArrowDecimalGetIntUnsafe(const struct ArrowDecimal* decimal) { +static inline int64_t ArrowDecimalGetIntUnsafe( + const struct ArrowDecimal* decimal) { return (int64_t)decimal->words[decimal->low_word_index]; } /// \brief Copy the bytes of this decimal into a sufficiently large buffer /// \ingroup nanoarrow-utils -static inline void ArrowDecimalGetBytes(const struct ArrowDecimal* decimal, - uint8_t* out) { +static inline void ArrowDecimalGetBytes( + const struct ArrowDecimal* decimal, uint8_t* out) { memcpy(out, decimal->words, decimal->n_words * sizeof(uint64_t)); } @@ -943,7 +990,8 @@ static inline int64_t ArrowDecimalSign(const struct ArrowDecimal* decimal) { /// \brief Sets the integer value of this decimal /// \ingroup nanoarrow-utils -static inline void ArrowDecimalSetInt(struct ArrowDecimal* decimal, int64_t value) { +static inline void ArrowDecimalSetInt( + struct ArrowDecimal* decimal, int64_t value) { if (value < 0) { memset(decimal->words, 0xff, decimal->n_words * sizeof(uint64_t)); } else { @@ -977,8 +1025,8 @@ static inline void ArrowDecimalNegate(struct ArrowDecimal* decimal) { /// \brief Copy bytes from a buffer into this decimal /// \ingroup nanoarrow-utils -static inline void ArrowDecimalSetBytes(struct ArrowDecimal* decimal, - const uint8_t* value) { +static inline void ArrowDecimalSetBytes( + struct ArrowDecimal* decimal, const uint8_t* value) { memcpy(decimal->words, value, decimal->n_words * sizeof(uint64_t)); } @@ -1011,11 +1059,9 @@ static inline void ArrowDecimalSetBytes(struct ArrowDecimal* decimal, #include #include - - -// If using CMake, optionally pass -DNANOARROW_NAMESPACE=MyNamespace which will set this -// define in nanoarrow_config.h. If not, you can optionally #define NANOARROW_NAMESPACE -// MyNamespace here. +// If using CMake, optionally pass -DNANOARROW_NAMESPACE=MyNamespace which will +// set this define in nanoarrow_config.h. If not, you can optionally #define +// NANOARROW_NAMESPACE MyNamespace here. // This section remaps the non-prefixed symbols to the prefixed symbols so that // code written against this build can be used independent of the value of @@ -1024,7 +1070,8 @@ static inline void ArrowDecimalSetBytes(struct ArrowDecimal* decimal, #define NANOARROW_CAT(A, B) A##B #define NANOARROW_SYMBOL(A, B) NANOARROW_CAT(A, B) -#define ArrowNanoarrowVersion NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowNanoarrowVersion) +#define ArrowNanoarrowVersion \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowNanoarrowVersion) #define ArrowNanoarrowVersionInt \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowNanoarrowVersionInt) #define ArrowMalloc NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMalloc) @@ -1036,13 +1083,15 @@ static inline void ArrowDecimalSetBytes(struct ArrowDecimal* decimal, NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowBufferDeallocator) #define ArrowErrorSet NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowErrorSet) #define ArrowLayoutInit NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowLayoutInit) -#define ArrowDecimalSetDigits NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowDecimalSetDigits) +#define ArrowDecimalSetDigits \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowDecimalSetDigits) #define ArrowDecimalAppendDigitsToBuffer \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowDecimalAppendDigitsToBuffer) #define ArrowSchemaInit NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaInit) #define ArrowSchemaInitFromType \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaInitFromType) -#define ArrowSchemaSetType NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetType) +#define ArrowSchemaSetType \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetType) #define ArrowSchemaSetTypeStruct \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetTypeStruct) #define ArrowSchemaSetTypeFixedSize \ @@ -1053,9 +1102,12 @@ static inline void ArrowDecimalSetBytes(struct ArrowDecimal* decimal, NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetTypeDateTime) #define ArrowSchemaSetTypeUnion \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetTypeUnion) -#define ArrowSchemaDeepCopy NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaDeepCopy) -#define ArrowSchemaSetFormat NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetFormat) -#define ArrowSchemaSetName NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetName) +#define ArrowSchemaDeepCopy \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaDeepCopy) +#define ArrowSchemaSetFormat \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetFormat) +#define ArrowSchemaSetName \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetName) #define ArrowSchemaSetMetadata \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetMetadata) #define ArrowSchemaAllocateChildren \ @@ -1066,9 +1118,12 @@ static inline void ArrowDecimalSetBytes(struct ArrowDecimal* decimal, NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataReaderInit) #define ArrowMetadataReaderRead \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataReaderRead) -#define ArrowMetadataSizeOf NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataSizeOf) -#define ArrowMetadataHasKey NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataHasKey) -#define ArrowMetadataGetValue NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataGetValue) +#define ArrowMetadataSizeOf \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataSizeOf) +#define ArrowMetadataHasKey \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataHasKey) +#define ArrowMetadataGetValue \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataGetValue) #define ArrowMetadataBuilderInit \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataBuilderInit) #define ArrowMetadataBuilderAppend \ @@ -1077,8 +1132,10 @@ static inline void ArrowDecimalSetBytes(struct ArrowDecimal* decimal, NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataBuilderSet) #define ArrowMetadataBuilderRemove \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataBuilderRemove) -#define ArrowSchemaViewInit NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaViewInit) -#define ArrowSchemaToString NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaToString) +#define ArrowSchemaViewInit \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaViewInit) +#define ArrowSchemaToString \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaToString) #define ArrowArrayInitFromType \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayInitFromType) #define ArrowArrayInitFromSchema \ @@ -1093,8 +1150,10 @@ static inline void ArrowDecimalSetBytes(struct ArrowDecimal* decimal, NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayAllocateDictionary) #define ArrowArraySetValidityBitmap \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArraySetValidityBitmap) -#define ArrowArraySetBuffer NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArraySetBuffer) -#define ArrowArrayReserve NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayReserve) +#define ArrowArraySetBuffer \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArraySetBuffer) +#define ArrowArrayReserve \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayReserve) #define ArrowArrayFinishBuilding \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayFinishBuilding) #define ArrowArrayFinishBuildingDefault \ @@ -1115,7 +1174,8 @@ static inline void ArrowDecimalSetBytes(struct ArrowDecimal* decimal, NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewSetArrayMinimal) #define ArrowArrayViewValidate \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewValidate) -#define ArrowArrayViewReset NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewReset) +#define ArrowArrayViewReset \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewReset) #define ArrowBasicArrayStreamInit \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowBasicArrayStreamInit) #define ArrowBasicArrayStreamSetArray \ @@ -1169,31 +1229,33 @@ struct ArrowBufferAllocator ArrowBufferAllocatorDefault(void); /// avoid copying an existing buffer that was not allocated using the /// infrastructure provided here (e.g., by an R or Python object). struct ArrowBufferAllocator ArrowBufferDeallocator( - void (*custom_free)(struct ArrowBufferAllocator* allocator, uint8_t* ptr, - int64_t size), + void (*custom_free)( + struct ArrowBufferAllocator* allocator, uint8_t* ptr, int64_t size), void* private_data); /// @} -/// \brief Move the contents of an src ArrowSchema into dst and set src->release to NULL -/// \ingroup nanoarrow-arrow-cdata -static inline void ArrowSchemaMove(struct ArrowSchema* src, struct ArrowSchema* dst); +/// \brief Move the contents of an src ArrowSchema into dst and set src->release +/// to NULL \ingroup nanoarrow-arrow-cdata +static inline void ArrowSchemaMove( + struct ArrowSchema* src, struct ArrowSchema* dst); /// \brief Call the release callback of an ArrowSchema /// \ingroup nanoarrow-arrow-cdata static inline void ArrowSchemaRelease(struct ArrowSchema* schema); -/// \brief Move the contents of an src ArrowArray into dst and set src->release to NULL -/// \ingroup nanoarrow-arrow-cdata -static inline void ArrowArrayMove(struct ArrowArray* src, struct ArrowArray* dst); +/// \brief Move the contents of an src ArrowArray into dst and set src->release +/// to NULL \ingroup nanoarrow-arrow-cdata +static inline void ArrowArrayMove( + struct ArrowArray* src, struct ArrowArray* dst); /// \brief Call the release callback of an ArrowArray static inline void ArrowArrayRelease(struct ArrowArray* array); -/// \brief Move the contents of an src ArrowArrayStream into dst and set src->release to -/// NULL \ingroup nanoarrow-arrow-cdata -static inline void ArrowArrayStreamMove(struct ArrowArrayStream* src, - struct ArrowArrayStream* dst); +/// \brief Move the contents of an src ArrowArrayStream into dst and set +/// src->release to NULL \ingroup nanoarrow-arrow-cdata +static inline void ArrowArrayStreamMove( + struct ArrowArrayStream* src, struct ArrowArrayStream* dst); /// \brief Call the get_schema callback of an ArrowArrayStream /// \ingroup nanoarrow-arrow-cdata @@ -1203,7 +1265,8 @@ static inline void ArrowArrayStreamMove(struct ArrowArrayStream* src, /// makes it significantly less verbose to iterate over array streams /// using NANOARROW_RETURN_NOT_OK()-style error handling. static inline ArrowErrorCode ArrowArrayStreamGetSchema( - struct ArrowArrayStream* array_stream, struct ArrowSchema* out, + struct ArrowArrayStream* array_stream, + struct ArrowSchema* out, struct ArrowError* error); /// \brief Call the get_schema callback of an ArrowArrayStream @@ -1214,20 +1277,22 @@ static inline ArrowErrorCode ArrowArrayStreamGetSchema( /// makes it significantly less verbose to iterate over array streams /// using NANOARROW_RETURN_NOT_OK()-style error handling. static inline ArrowErrorCode ArrowArrayStreamGetNext( - struct ArrowArrayStream* array_stream, struct ArrowArray* out, + struct ArrowArrayStream* array_stream, + struct ArrowArray* out, struct ArrowError* error); /// \brief Call the get_next callback of an ArrowArrayStream /// \ingroup nanoarrow-arrow-cdata /// /// Unlike the get_next callback, this function never returns NULL (i.e., its -/// result is safe to use in printf-style error formatters). Null values from the -/// original callback are reported as "". +/// result is safe to use in printf-style error formatters). Null values from +/// the original callback are reported as "". static inline const char* ArrowArrayStreamGetLastError( struct ArrowArrayStream* array_stream); /// \brief Call the release callback of an ArrowArrayStream -static inline void ArrowArrayStreamRelease(struct ArrowArrayStream* array_stream); +static inline void ArrowArrayStreamRelease( + struct ArrowArrayStream* array_stream); /// \defgroup nanoarrow-errors Error handling /// @@ -1235,24 +1300,24 @@ static inline void ArrowArrayStreamRelease(struct ArrowArrayStream* array_stream /// need to communicate more verbose error information accept a pointer /// to an ArrowError. This can be stack or statically allocated. The /// content of the message is undefined unless an error code has been -/// returned. If a nanoarrow function is passed a non-null ArrowError pointer, the -/// ArrowError pointed to by the argument will be propagated with a +/// returned. If a nanoarrow function is passed a non-null ArrowError pointer, +/// the ArrowError pointed to by the argument will be propagated with a /// null-terminated error message. It is safe to pass a NULL ArrowError anywhere /// in the nanoarrow API. /// /// Except where documented, it is generally not safe to continue after a -/// function has returned a non-zero ArrowErrorCode. The NANOARROW_RETURN_NOT_OK and -/// NANOARROW_ASSERT_OK macros are provided to help propagate errors. C++ clients can use -/// the helpers provided in the nanoarrow.hpp header to facilitate using C++ idioms -/// for memory management and error propgagtion. +/// function has returned a non-zero ArrowErrorCode. The NANOARROW_RETURN_NOT_OK +/// and NANOARROW_ASSERT_OK macros are provided to help propagate errors. C++ +/// clients can use the helpers provided in the nanoarrow.hpp header to +/// facilitate using C++ idioms for memory management and error propgagtion. /// /// @{ /// \brief Set the contents of an error using printf syntax. /// /// If error is NULL, this function does nothing and returns NANOARROW_OK. -NANOARROW_CHECK_PRINTF_ATTRIBUTE int ArrowErrorSet(struct ArrowError* error, - const char* fmt, ...); +NANOARROW_CHECK_PRINTF_ATTRIBUTE int ArrowErrorSet( + struct ArrowError* error, const char* fmt, ...); /// @} @@ -1273,12 +1338,12 @@ void ArrowLayoutInit(struct ArrowLayout* layout, enum ArrowType storage_type); static inline struct ArrowStringView ArrowCharView(const char* value); /// \brief Sets the integer value of an ArrowDecimal from a string -ArrowErrorCode ArrowDecimalSetDigits(struct ArrowDecimal* decimal, - struct ArrowStringView value); +ArrowErrorCode ArrowDecimalSetDigits( + struct ArrowDecimal* decimal, struct ArrowStringView value); /// \brief Get the integer value of an ArrowDecimal as string -ArrowErrorCode ArrowDecimalAppendDigitsToBuffer(const struct ArrowDecimal* decimal, - struct ArrowBuffer* buffer); +ArrowErrorCode ArrowDecimalAppendDigitsToBuffer( + const struct ArrowDecimal* decimal, struct ArrowBuffer* buffer); /// @} @@ -1299,9 +1364,10 @@ void ArrowSchemaInit(struct ArrowSchema* schema); /// /// A convenience constructor for that calls ArrowSchemaInit() and /// ArrowSchemaSetType() for the common case of constructing an -/// unparameterized type. The caller is responsible for calling the schema->release -/// callback if NANOARROW_OK is returned. -ArrowErrorCode ArrowSchemaInitFromType(struct ArrowSchema* schema, enum ArrowType type); +/// unparameterized type. The caller is responsible for calling the +/// schema->release callback if NANOARROW_OK is returned. +ArrowErrorCode ArrowSchemaInitFromType( + struct ArrowSchema* schema, enum ArrowType type); /// \brief Get a human-readable summary of a Schema /// @@ -1309,8 +1375,8 @@ ArrowErrorCode ArrowSchemaInitFromType(struct ArrowSchema* schema, enum ArrowTyp /// and returns the number of characters required for the output if /// n were sufficiently large. If recursive is non-zero, the result will /// also include children. -int64_t ArrowSchemaToString(const struct ArrowSchema* schema, char* out, int64_t n, - char recursive); +int64_t ArrowSchemaToString( + const struct ArrowSchema* schema, char* out, int64_t n, char recursive); /// \brief Set the format field of a schema from an ArrowType /// @@ -1318,16 +1384,19 @@ int64_t ArrowSchemaToString(const struct ArrowSchema* schema, char* out, int64_t /// NANOARROW_TYPE_LIST, NANOARROW_TYPE_LARGE_LIST, and /// NANOARROW_TYPE_MAP, the appropriate number of children are /// allocated, initialized, and named; however, the caller must -/// ArrowSchemaSetType() on the preinitialized children. Schema must have been initialized -/// using ArrowSchemaInit() or ArrowSchemaDeepCopy(). -ArrowErrorCode ArrowSchemaSetType(struct ArrowSchema* schema, enum ArrowType type); +/// ArrowSchemaSetType() on the preinitialized children. Schema must have been +/// initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy(). +ArrowErrorCode ArrowSchemaSetType( + struct ArrowSchema* schema, enum ArrowType type); /// \brief Set the format field and initialize children of a struct schema /// -/// The specified number of children are initialized; however, the caller is responsible -/// for calling ArrowSchemaSetType() and ArrowSchemaSetName() on each child. -/// Schema must have been initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy(). -ArrowErrorCode ArrowSchemaSetTypeStruct(struct ArrowSchema* schema, int64_t n_children); +/// The specified number of children are initialized; however, the caller is +/// responsible for calling ArrowSchemaSetType() and ArrowSchemaSetName() on +/// each child. Schema must have been initialized using ArrowSchemaInit() or +/// ArrowSchemaDeepCopy(). +ArrowErrorCode ArrowSchemaSetTypeStruct( + struct ArrowSchema* schema, int64_t n_children); /// \brief Set the format field of a fixed-size schema /// @@ -1335,50 +1404,55 @@ ArrowErrorCode ArrowSchemaSetTypeStruct(struct ArrowSchema* schema, int64_t n_ch /// NANOARROW_TYPE_FIXED_SIZE_BINARY or NANOARROW_TYPE_FIXED_SIZE_LIST. /// For NANOARROW_TYPE_FIXED_SIZE_LIST, the appropriate number of children are /// allocated, initialized, and named; however, the caller must -/// ArrowSchemaSetType() the first child. Schema must have been initialized using -/// ArrowSchemaInit() or ArrowSchemaDeepCopy(). -ArrowErrorCode ArrowSchemaSetTypeFixedSize(struct ArrowSchema* schema, - enum ArrowType type, int32_t fixed_size); +/// ArrowSchemaSetType() the first child. Schema must have been initialized +/// using ArrowSchemaInit() or ArrowSchemaDeepCopy(). +ArrowErrorCode ArrowSchemaSetTypeFixedSize( + struct ArrowSchema* schema, enum ArrowType type, int32_t fixed_size); /// \brief Set the format field of a decimal schema /// /// Returns EINVAL for scale <= 0 or for type that is not -/// NANOARROW_TYPE_DECIMAL128 or NANOARROW_TYPE_DECIMAL256. Schema must have been -/// initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy(). -ArrowErrorCode ArrowSchemaSetTypeDecimal(struct ArrowSchema* schema, enum ArrowType type, - int32_t decimal_precision, - int32_t decimal_scale); +/// NANOARROW_TYPE_DECIMAL128 or NANOARROW_TYPE_DECIMAL256. Schema must have +/// been initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy(). +ArrowErrorCode ArrowSchemaSetTypeDecimal( + struct ArrowSchema* schema, + enum ArrowType type, + int32_t decimal_precision, + int32_t decimal_scale); /// \brief Set the format field of a time, timestamp, or duration schema /// /// Returns EINVAL for type that is not /// NANOARROW_TYPE_TIME32, NANOARROW_TYPE_TIME64, /// NANOARROW_TYPE_TIMESTAMP, or NANOARROW_TYPE_DURATION. The -/// timezone parameter must be NULL for a non-timestamp type. Schema must have been -/// initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy(). -ArrowErrorCode ArrowSchemaSetTypeDateTime(struct ArrowSchema* schema, enum ArrowType type, - enum ArrowTimeUnit time_unit, - const char* timezone); +/// timezone parameter must be NULL for a non-timestamp type. Schema must have +/// been initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy(). +ArrowErrorCode ArrowSchemaSetTypeDateTime( + struct ArrowSchema* schema, + enum ArrowType type, + enum ArrowTimeUnit time_unit, + const char* timezone); /// \brief Seet the format field of a union schema /// /// Returns EINVAL for a type that is not NANOARROW_TYPE_DENSE_UNION /// or NANOARROW_TYPE_SPARSE_UNION. The specified number of children are /// allocated, and initialized. -ArrowErrorCode ArrowSchemaSetTypeUnion(struct ArrowSchema* schema, enum ArrowType type, - int64_t n_children); +ArrowErrorCode ArrowSchemaSetTypeUnion( + struct ArrowSchema* schema, enum ArrowType type, int64_t n_children); /// \brief Make a (recursive) copy of a schema /// /// Allocates and copies fields of schema into schema_out. -ArrowErrorCode ArrowSchemaDeepCopy(const struct ArrowSchema* schema, - struct ArrowSchema* schema_out); +ArrowErrorCode ArrowSchemaDeepCopy( + const struct ArrowSchema* schema, struct ArrowSchema* schema_out); /// \brief Copy format into schema->format /// /// schema must have been allocated using ArrowSchemaInitFromType() or /// ArrowSchemaDeepCopy(). -ArrowErrorCode ArrowSchemaSetFormat(struct ArrowSchema* schema, const char* format); +ArrowErrorCode ArrowSchemaSetFormat( + struct ArrowSchema* schema, const char* format); /// \brief Copy name into schema->name /// @@ -1390,15 +1464,16 @@ ArrowErrorCode ArrowSchemaSetName(struct ArrowSchema* schema, const char* name); /// /// schema must have been allocated using ArrowSchemaInitFromType() or /// ArrowSchemaDeepCopy. -ArrowErrorCode ArrowSchemaSetMetadata(struct ArrowSchema* schema, const char* metadata); +ArrowErrorCode ArrowSchemaSetMetadata( + struct ArrowSchema* schema, const char* metadata); /// \brief Allocate the schema->children array /// /// Includes the memory for each child struct ArrowSchema. /// schema must have been allocated using ArrowSchemaInitFromType() or /// ArrowSchemaDeepCopy(). -ArrowErrorCode ArrowSchemaAllocateChildren(struct ArrowSchema* schema, - int64_t n_children); +ArrowErrorCode ArrowSchemaAllocateChildren( + struct ArrowSchema* schema, int64_t n_children); /// \brief Allocate the schema->dictionary member /// @@ -1428,13 +1503,14 @@ struct ArrowMetadataReader { }; /// \brief Initialize an ArrowMetadataReader -ArrowErrorCode ArrowMetadataReaderInit(struct ArrowMetadataReader* reader, - const char* metadata); +ArrowErrorCode ArrowMetadataReaderInit( + struct ArrowMetadataReader* reader, const char* metadata); /// \brief Read the next key/value pair from an ArrowMetadataReader -ArrowErrorCode ArrowMetadataReaderRead(struct ArrowMetadataReader* reader, - struct ArrowStringView* key_out, - struct ArrowStringView* value_out); +ArrowErrorCode ArrowMetadataReaderRead( + struct ArrowMetadataReader* reader, + struct ArrowStringView* key_out, + struct ArrowStringView* value_out); /// \brief The number of bytes in in a key/value metadata string int64_t ArrowMetadataSizeOf(const char* metadata); @@ -1445,32 +1521,37 @@ char ArrowMetadataHasKey(const char* metadata, struct ArrowStringView key); /// \brief Extract a value from schema metadata /// /// If key does not exist in metadata, value_out is unmodified -ArrowErrorCode ArrowMetadataGetValue(const char* metadata, struct ArrowStringView key, - struct ArrowStringView* value_out); +ArrowErrorCode ArrowMetadataGetValue( + const char* metadata, + struct ArrowStringView key, + struct ArrowStringView* value_out); /// \brief Initialize a builder for schema metadata from key/value pairs /// /// metadata can be an existing metadata string or NULL to initialize /// an empty metadata string. -ArrowErrorCode ArrowMetadataBuilderInit(struct ArrowBuffer* buffer, const char* metadata); +ArrowErrorCode ArrowMetadataBuilderInit( + struct ArrowBuffer* buffer, const char* metadata); /// \brief Append a key/value pair to a buffer containing serialized metadata -ArrowErrorCode ArrowMetadataBuilderAppend(struct ArrowBuffer* buffer, - struct ArrowStringView key, - struct ArrowStringView value); +ArrowErrorCode ArrowMetadataBuilderAppend( + struct ArrowBuffer* buffer, + struct ArrowStringView key, + struct ArrowStringView value); /// \brief Set a key/value pair to a buffer containing serialized metadata /// /// Ensures that the only entry for key in the metadata is set to value. /// This function maintains the existing position of (the first instance of) /// key if present in the data. -ArrowErrorCode ArrowMetadataBuilderSet(struct ArrowBuffer* buffer, - struct ArrowStringView key, - struct ArrowStringView value); +ArrowErrorCode ArrowMetadataBuilderSet( + struct ArrowBuffer* buffer, + struct ArrowStringView key, + struct ArrowStringView value); /// \brief Remove a key from a buffer containing serialized metadata -ArrowErrorCode ArrowMetadataBuilderRemove(struct ArrowBuffer* buffer, - struct ArrowStringView key); +ArrowErrorCode ArrowMetadataBuilderRemove( + struct ArrowBuffer* buffer, struct ArrowStringView key); /// @} @@ -1499,9 +1580,9 @@ struct ArrowSchemaView { /// \brief The storage data type represented by the schema /// - /// This value will never be NANOARROW_TYPE_DICTIONARY, NANOARROW_TYPE_EXTENSION - /// or any datetime type. This value represents only the type required to - /// interpret the buffers in the array. + /// This value will never be NANOARROW_TYPE_DICTIONARY, + /// NANOARROW_TYPE_EXTENSION or any datetime type. This value represents only + /// the type required to interpret the buffers in the array. enum ArrowType storage_type; /// \brief The storage layout represented by the schema @@ -1568,9 +1649,10 @@ struct ArrowSchemaView { }; /// \brief Initialize an ArrowSchemaView -ArrowErrorCode ArrowSchemaViewInit(struct ArrowSchemaView* schema_view, - const struct ArrowSchema* schema, - struct ArrowError* error); +ArrowErrorCode ArrowSchemaViewInit( + struct ArrowSchemaView* schema_view, + const struct ArrowSchema* schema, + struct ArrowError* error); /// @} @@ -1601,7 +1683,8 @@ static inline void ArrowBufferReset(struct ArrowBuffer* buffer); /// /// Transfers the buffer data and lifecycle management to another /// address and resets buffer. -static inline void ArrowBufferMove(struct ArrowBuffer* src, struct ArrowBuffer* dst); +static inline void ArrowBufferMove( + struct ArrowBuffer* src, struct ArrowBuffer* dst); /// \brief Grow or shrink a buffer to a given capacity /// @@ -1609,85 +1692,84 @@ static inline void ArrowBufferMove(struct ArrowBuffer* src, struct ArrowBuffer* /// if shrink_to_fit is non-zero. Calling ArrowBufferResize() does not /// adjust the buffer's size member except to ensure that the invariant /// capacity >= size remains true. -static inline ArrowErrorCode ArrowBufferResize(struct ArrowBuffer* buffer, - int64_t new_capacity_bytes, - char shrink_to_fit); +static inline ArrowErrorCode ArrowBufferResize( + struct ArrowBuffer* buffer, int64_t new_capacity_bytes, char shrink_to_fit); /// \brief Ensure a buffer has at least a given additional capacity /// /// Ensures that the buffer has space to append at least /// additional_size_bytes, overallocating when required. -static inline ArrowErrorCode ArrowBufferReserve(struct ArrowBuffer* buffer, - int64_t additional_size_bytes); +static inline ArrowErrorCode ArrowBufferReserve( + struct ArrowBuffer* buffer, int64_t additional_size_bytes); /// \brief Write data to buffer and increment the buffer size /// /// This function does not check that buffer has the required capacity -static inline void ArrowBufferAppendUnsafe(struct ArrowBuffer* buffer, const void* data, - int64_t size_bytes); +static inline void ArrowBufferAppendUnsafe( + struct ArrowBuffer* buffer, const void* data, int64_t size_bytes); /// \brief Write data to buffer and increment the buffer size /// /// This function writes and ensures that the buffer has the required capacity, /// possibly by reallocating the buffer. Like ArrowBufferReserve, this will /// overallocate when reallocation is required. -static inline ArrowErrorCode ArrowBufferAppend(struct ArrowBuffer* buffer, - const void* data, int64_t size_bytes); +static inline ArrowErrorCode ArrowBufferAppend( + struct ArrowBuffer* buffer, const void* data, int64_t size_bytes); /// \brief Write fill to buffer and increment the buffer size /// /// This function writes the specified number of fill bytes and /// ensures that the buffer has the required capacity, -static inline ArrowErrorCode ArrowBufferAppendFill(struct ArrowBuffer* buffer, - uint8_t value, int64_t size_bytes); +static inline ArrowErrorCode ArrowBufferAppendFill( + struct ArrowBuffer* buffer, uint8_t value, int64_t size_bytes); /// \brief Write an 8-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendInt8(struct ArrowBuffer* buffer, - int8_t value); +static inline ArrowErrorCode ArrowBufferAppendInt8( + struct ArrowBuffer* buffer, int8_t value); /// \brief Write an unsigned 8-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendUInt8(struct ArrowBuffer* buffer, - uint8_t value); +static inline ArrowErrorCode ArrowBufferAppendUInt8( + struct ArrowBuffer* buffer, uint8_t value); /// \brief Write a 16-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendInt16(struct ArrowBuffer* buffer, - int16_t value); +static inline ArrowErrorCode ArrowBufferAppendInt16( + struct ArrowBuffer* buffer, int16_t value); /// \brief Write an unsigned 16-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendUInt16(struct ArrowBuffer* buffer, - uint16_t value); +static inline ArrowErrorCode ArrowBufferAppendUInt16( + struct ArrowBuffer* buffer, uint16_t value); /// \brief Write a 32-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendInt32(struct ArrowBuffer* buffer, - int32_t value); +static inline ArrowErrorCode ArrowBufferAppendInt32( + struct ArrowBuffer* buffer, int32_t value); /// \brief Write an unsigned 32-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendUInt32(struct ArrowBuffer* buffer, - uint32_t value); +static inline ArrowErrorCode ArrowBufferAppendUInt32( + struct ArrowBuffer* buffer, uint32_t value); /// \brief Write a 64-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendInt64(struct ArrowBuffer* buffer, - int64_t value); +static inline ArrowErrorCode ArrowBufferAppendInt64( + struct ArrowBuffer* buffer, int64_t value); /// \brief Write an unsigned 64-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendUInt64(struct ArrowBuffer* buffer, - uint64_t value); +static inline ArrowErrorCode ArrowBufferAppendUInt64( + struct ArrowBuffer* buffer, uint64_t value); /// \brief Write a double to a buffer -static inline ArrowErrorCode ArrowBufferAppendDouble(struct ArrowBuffer* buffer, - double value); +static inline ArrowErrorCode ArrowBufferAppendDouble( + struct ArrowBuffer* buffer, double value); /// \brief Write a float to a buffer -static inline ArrowErrorCode ArrowBufferAppendFloat(struct ArrowBuffer* buffer, - float value); +static inline ArrowErrorCode ArrowBufferAppendFloat( + struct ArrowBuffer* buffer, float value); /// \brief Write an ArrowStringView to a buffer -static inline ArrowErrorCode ArrowBufferAppendStringView(struct ArrowBuffer* buffer, - struct ArrowStringView value); +static inline ArrowErrorCode ArrowBufferAppendStringView( + struct ArrowBuffer* buffer, struct ArrowStringView value); /// \brief Write an ArrowBufferView to a buffer -static inline ArrowErrorCode ArrowBufferAppendBufferView(struct ArrowBuffer* buffer, - struct ArrowBufferView value); +static inline ArrowErrorCode ArrowBufferAppendBufferView( + struct ArrowBuffer* buffer, struct ArrowBufferView value); /// @} @@ -1708,19 +1790,20 @@ static inline void ArrowBitClear(uint8_t* bits, int64_t i); static inline void ArrowBitSetTo(uint8_t* bits, int64_t i, uint8_t value); /// \brief Set a boolean value to a range in a bitmap -static inline void ArrowBitsSetTo(uint8_t* bits, int64_t start_offset, int64_t length, - uint8_t bits_are_set); +static inline void ArrowBitsSetTo( + uint8_t* bits, int64_t start_offset, int64_t length, uint8_t bits_are_set); /// \brief Count true values in a bitmap -static inline int64_t ArrowBitCountSet(const uint8_t* bits, int64_t i_from, int64_t i_to); +static inline int64_t ArrowBitCountSet( + const uint8_t* bits, int64_t i_from, int64_t i_to); /// \brief Extract int8 boolean values from a range in a bitmap -static inline void ArrowBitsUnpackInt8(const uint8_t* bits, int64_t start_offset, - int64_t length, int8_t* out); +static inline void ArrowBitsUnpackInt8( + const uint8_t* bits, int64_t start_offset, int64_t length, int8_t* out); /// \brief Extract int32 boolean values from a range in a bitmap -static inline void ArrowBitsUnpackInt32(const uint8_t* bits, int64_t start_offset, - int64_t length, int32_t* out); +static inline void ArrowBitsUnpackInt32( + const uint8_t* bits, int64_t start_offset, int64_t length, int32_t* out); /// \brief Initialize an ArrowBitmap /// @@ -1731,14 +1814,15 @@ static inline void ArrowBitmapInit(struct ArrowBitmap* bitmap); /// /// Transfers the underlying buffer data and lifecycle management to another /// address and resets the bitmap. -static inline void ArrowBitmapMove(struct ArrowBitmap* src, struct ArrowBitmap* dst); +static inline void ArrowBitmapMove( + struct ArrowBitmap* src, struct ArrowBitmap* dst); /// \brief Ensure a bitmap builder has at least a given additional capacity /// /// Ensures that the buffer has space to append at least /// additional_size_bits, overallocating when required. -static inline ArrowErrorCode ArrowBitmapReserve(struct ArrowBitmap* bitmap, - int64_t additional_size_bits); +static inline ArrowErrorCode ArrowBitmapReserve( + struct ArrowBitmap* bitmap, int64_t additional_size_bits); /// \brief Grow or shrink a bitmap to a given capacity /// @@ -1746,33 +1830,34 @@ static inline ArrowErrorCode ArrowBitmapReserve(struct ArrowBitmap* bitmap, /// if shrink_to_fit is non-zero. Calling ArrowBitmapResize() does not /// adjust the buffer's size member except when shrinking new_capacity_bits /// to a value less than the current number of bits in the bitmap. -static inline ArrowErrorCode ArrowBitmapResize(struct ArrowBitmap* bitmap, - int64_t new_capacity_bits, - char shrink_to_fit); +static inline ArrowErrorCode ArrowBitmapResize( + struct ArrowBitmap* bitmap, int64_t new_capacity_bits, char shrink_to_fit); -/// \brief Reserve space for and append zero or more of the same boolean value to a bitmap -static inline ArrowErrorCode ArrowBitmapAppend(struct ArrowBitmap* bitmap, - uint8_t bits_are_set, int64_t length); +/// \brief Reserve space for and append zero or more of the same boolean value +/// to a bitmap +static inline ArrowErrorCode ArrowBitmapAppend( + struct ArrowBitmap* bitmap, uint8_t bits_are_set, int64_t length); /// \brief Append zero or more of the same boolean value to a bitmap -static inline void ArrowBitmapAppendUnsafe(struct ArrowBitmap* bitmap, - uint8_t bits_are_set, int64_t length); +static inline void ArrowBitmapAppendUnsafe( + struct ArrowBitmap* bitmap, uint8_t bits_are_set, int64_t length); /// \brief Append boolean values encoded as int8_t to a bitmap /// /// The values must all be 0 or 1. -static inline void ArrowBitmapAppendInt8Unsafe(struct ArrowBitmap* bitmap, - const int8_t* values, int64_t n_values); +static inline void ArrowBitmapAppendInt8Unsafe( + struct ArrowBitmap* bitmap, const int8_t* values, int64_t n_values); /// \brief Append boolean values encoded as int32_t to a bitmap /// /// The values must all be 0 or 1. -static inline void ArrowBitmapAppendInt32Unsafe(struct ArrowBitmap* bitmap, - const int32_t* values, int64_t n_values); +static inline void ArrowBitmapAppendInt32Unsafe( + struct ArrowBitmap* bitmap, const int32_t* values, int64_t n_values); /// \brief Reset a bitmap builder /// -/// Releases any memory held by buffer, empties the cache, and resets the size to zero +/// Releases any memory held by buffer, empties the cache, and resets the size +/// to zero static inline void ArrowBitmapReset(struct ArrowBitmap* bitmap); /// @} @@ -1791,24 +1876,26 @@ static inline void ArrowBitmapReset(struct ArrowBitmap* bitmap); /// Initializes the fields and release callback of array. Caller /// is responsible for calling the array->release callback if /// NANOARROW_OK is returned. -ArrowErrorCode ArrowArrayInitFromType(struct ArrowArray* array, - enum ArrowType storage_type); +ArrowErrorCode ArrowArrayInitFromType( + struct ArrowArray* array, enum ArrowType storage_type); /// \brief Initialize the contents of an ArrowArray from an ArrowSchema /// /// Caller is responsible for calling the array->release callback if /// NANOARROW_OK is returned. -ArrowErrorCode ArrowArrayInitFromSchema(struct ArrowArray* array, - const struct ArrowSchema* schema, - struct ArrowError* error); +ArrowErrorCode ArrowArrayInitFromSchema( + struct ArrowArray* array, + const struct ArrowSchema* schema, + struct ArrowError* error); /// \brief Initialize the contents of an ArrowArray from an ArrowArrayView /// /// Caller is responsible for calling the array->release callback if /// NANOARROW_OK is returned. -ArrowErrorCode ArrowArrayInitFromArrayView(struct ArrowArray* array, - const struct ArrowArrayView* array_view, - struct ArrowError* error); +ArrowErrorCode ArrowArrayInitFromArrayView( + struct ArrowArray* array, + const struct ArrowArrayView* array_view, + struct ArrowError* error); /// \brief Allocate the array->children array /// @@ -1816,7 +1903,8 @@ ArrowErrorCode ArrowArrayInitFromArrayView(struct ArrowArray* array, /// whose members are marked as released and may be subsequently initialized /// with ArrowArrayInitFromType() or moved from an existing ArrowArray. /// schema must have been allocated using ArrowArrayInitFromType(). -ArrowErrorCode ArrowArrayAllocateChildren(struct ArrowArray* array, int64_t n_children); +ArrowErrorCode ArrowArrayAllocateChildren( + struct ArrowArray* array, int64_t n_children); /// \brief Allocate the array->dictionary member /// @@ -1829,30 +1917,33 @@ ArrowErrorCode ArrowArrayAllocateDictionary(struct ArrowArray* array); /// \brief Set the validity bitmap of an ArrowArray /// /// array must have been allocated using ArrowArrayInitFromType() -void ArrowArraySetValidityBitmap(struct ArrowArray* array, struct ArrowBitmap* bitmap); +void ArrowArraySetValidityBitmap( + struct ArrowArray* array, struct ArrowBitmap* bitmap); /// \brief Set a buffer of an ArrowArray /// /// array must have been allocated using ArrowArrayInitFromType() -ArrowErrorCode ArrowArraySetBuffer(struct ArrowArray* array, int64_t i, - struct ArrowBuffer* buffer); +ArrowErrorCode ArrowArraySetBuffer( + struct ArrowArray* array, int64_t i, struct ArrowBuffer* buffer); /// \brief Get the validity bitmap of an ArrowArray /// /// array must have been allocated using ArrowArrayInitFromType() -static inline struct ArrowBitmap* ArrowArrayValidityBitmap(struct ArrowArray* array); +static inline struct ArrowBitmap* ArrowArrayValidityBitmap( + struct ArrowArray* array); /// \brief Get a buffer of an ArrowArray /// /// array must have been allocated using ArrowArrayInitFromType() -static inline struct ArrowBuffer* ArrowArrayBuffer(struct ArrowArray* array, int64_t i); +static inline struct ArrowBuffer* ArrowArrayBuffer( + struct ArrowArray* array, int64_t i); /// \brief Start element-wise appending to an ArrowArray /// /// Initializes any values needed to use ArrowArrayAppend*() functions. -/// All element-wise appenders append by value and return EINVAL if the exact value -/// cannot be represented by the underlying storage type. -/// array must have been allocated using ArrowArrayInitFromType() +/// All element-wise appenders append by value and return EINVAL if the exact +/// value cannot be represented by the underlying storage type. array must have +/// been allocated using ArrowArrayInitFromType() static inline ArrowErrorCode ArrowArrayStartAppending(struct ArrowArray* array); /// \brief Reserve space for future appends @@ -1861,29 +1952,32 @@ static inline ArrowErrorCode ArrowArrayStartAppending(struct ArrowArray* array); /// child array sizes for non-fixed-size arrays), recursively reserve space for /// additional elements. This is useful for reducing the number of reallocations /// that occur using the item-wise appenders. -ArrowErrorCode ArrowArrayReserve(struct ArrowArray* array, - int64_t additional_size_elements); +ArrowErrorCode ArrowArrayReserve( + struct ArrowArray* array, int64_t additional_size_elements); /// \brief Append a null value to an array -static inline ArrowErrorCode ArrowArrayAppendNull(struct ArrowArray* array, int64_t n); +static inline ArrowErrorCode ArrowArrayAppendNull( + struct ArrowArray* array, int64_t n); /// \brief Append an empty, non-null value to an array -static inline ArrowErrorCode ArrowArrayAppendEmpty(struct ArrowArray* array, int64_t n); +static inline ArrowErrorCode ArrowArrayAppendEmpty( + struct ArrowArray* array, int64_t n); /// \brief Append a signed integer value to an array /// /// Returns NANOARROW_OK if value can be exactly represented by /// the underlying storage type or EINVAL otherwise (e.g., value /// is outside the valid array range). -static inline ArrowErrorCode ArrowArrayAppendInt(struct ArrowArray* array, int64_t value); +static inline ArrowErrorCode ArrowArrayAppendInt( + struct ArrowArray* array, int64_t value); /// \brief Append an unsigned integer value to an array /// /// Returns NANOARROW_OK if value can be exactly represented by /// the underlying storage type or EINVAL otherwise (e.g., value /// is outside the valid array range). -static inline ArrowErrorCode ArrowArrayAppendUInt(struct ArrowArray* array, - uint64_t value); +static inline ArrowErrorCode ArrowArrayAppendUInt( + struct ArrowArray* array, uint64_t value); /// \brief Append a double value to an array /// @@ -1891,67 +1985,68 @@ static inline ArrowErrorCode ArrowArrayAppendUInt(struct ArrowArray* array, /// the underlying storage type or EINVAL otherwise (e.g., value /// is outside the valid array range or there is an attempt to append /// a non-integer to an array with an integer storage type). -static inline ArrowErrorCode ArrowArrayAppendDouble(struct ArrowArray* array, - double value); +static inline ArrowErrorCode ArrowArrayAppendDouble( + struct ArrowArray* array, double value); /// \brief Append a string of bytes to an array /// /// Returns NANOARROW_OK if value can be exactly represented by /// the underlying storage type, EOVERFLOW if appending value would overflow /// the offset type (e.g., if the data buffer would be larger than 2 GB for a -/// non-large string type), or EINVAL otherwise (e.g., the underlying array is not a -/// binary, string, large binary, large string, or fixed-size binary array, or value is -/// the wrong size for a fixed-size binary array). -static inline ArrowErrorCode ArrowArrayAppendBytes(struct ArrowArray* array, - struct ArrowBufferView value); +/// non-large string type), or EINVAL otherwise (e.g., the underlying array is +/// not a binary, string, large binary, large string, or fixed-size binary +/// array, or value is the wrong size for a fixed-size binary array). +static inline ArrowErrorCode ArrowArrayAppendBytes( + struct ArrowArray* array, struct ArrowBufferView value); /// \brief Append a string value to an array /// /// Returns NANOARROW_OK if value can be exactly represented by /// the underlying storage type, EOVERFLOW if appending value would overflow /// the offset type (e.g., if the data buffer would be larger than 2 GB for a -/// non-large string type), or EINVAL otherwise (e.g., the underlying array is not a -/// string or large string array). -static inline ArrowErrorCode ArrowArrayAppendString(struct ArrowArray* array, - struct ArrowStringView value); +/// non-large string type), or EINVAL otherwise (e.g., the underlying array is +/// not a string or large string array). +static inline ArrowErrorCode ArrowArrayAppendString( + struct ArrowArray* array, struct ArrowStringView value); /// \brief Append a Interval to an array /// /// Returns NANOARROW_OK if value can be exactly represented by /// the underlying storage type or EINVAL otherwise. -static inline ArrowErrorCode ArrowArrayAppendInterval(struct ArrowArray* array, - const struct ArrowInterval* value); +static inline ArrowErrorCode ArrowArrayAppendInterval( + struct ArrowArray* array, const struct ArrowInterval* value); /// \brief Append a decimal value to an array /// /// Returns NANOARROW_OK if array is a decimal array with the appropriate /// bitwidth or EINVAL otherwise. -static inline ArrowErrorCode ArrowArrayAppendDecimal(struct ArrowArray* array, - const struct ArrowDecimal* value); +static inline ArrowErrorCode ArrowArrayAppendDecimal( + struct ArrowArray* array, const struct ArrowDecimal* value); /// \brief Finish a nested array element /// /// Appends a non-null element to the array based on the first child's current /// length. Returns NANOARROW_OK if the item was successfully added, EOVERFLOW /// if the child of a list or map array would exceed INT_MAX elements, or EINVAL -/// if the underlying storage type is not a struct, list, large list, or fixed-size -/// list, or if there was an attempt to add a struct or fixed-size list element where the -/// length of the child array(s) did not match the expected length. +/// if the underlying storage type is not a struct, list, large list, or +/// fixed-size list, or if there was an attempt to add a struct or fixed-size +/// list element where the length of the child array(s) did not match the +/// expected length. static inline ArrowErrorCode ArrowArrayFinishElement(struct ArrowArray* array); /// \brief Finish a union array element /// -/// Appends an element to the union type ids buffer and increments array->length. -/// For sparse unions, up to one element is added to non type-id children. Returns -/// EINVAL if the underlying storage type is not a union, if type_id is not valid, -/// or if child sizes after appending are inconsistent. -static inline ArrowErrorCode ArrowArrayFinishUnionElement(struct ArrowArray* array, - int8_t type_id); +/// Appends an element to the union type ids buffer and increments +/// array->length. For sparse unions, up to one element is added to non type-id +/// children. Returns EINVAL if the underlying storage type is not a union, if +/// type_id is not valid, or if child sizes after appending are inconsistent. +static inline ArrowErrorCode ArrowArrayFinishUnionElement( + struct ArrowArray* array, int8_t type_id); /// \brief Shrink buffer capacity to the size required /// -/// Also applies shrinking to any child arrays. array must have been allocated using -/// ArrowArrayInitFromType +/// Also applies shrinking to any child arrays. array must have been allocated +/// using ArrowArrayInitFromType static inline ArrowErrorCode ArrowArrayShrinkToFit(struct ArrowArray* array); /// \brief Finish building an ArrowArray @@ -1960,19 +2055,20 @@ static inline ArrowErrorCode ArrowArrayShrinkToFit(struct ArrowArray* array); /// into array->buffers and checks the actual size of the buffers /// against the expected size based on the final length. /// array must have been allocated using ArrowArrayInitFromType() -ArrowErrorCode ArrowArrayFinishBuildingDefault(struct ArrowArray* array, - struct ArrowError* error); +ArrowErrorCode ArrowArrayFinishBuildingDefault( + struct ArrowArray* array, struct ArrowError* error); /// \brief Finish building an ArrowArray with explicit validation /// -/// Finish building with an explicit validation level. This could perform less validation -/// (i.e. NANOARROW_VALIDATION_LEVEL_NONE or NANOARROW_VALIDATION_LEVEL_MINIMAL) if CPU -/// buffer data access is not possible or more validation (i.e., -/// NANOARROW_VALIDATION_LEVEL_FULL) if buffer content was obtained from an untrusted or -/// corruptible source. -ArrowErrorCode ArrowArrayFinishBuilding(struct ArrowArray* array, - enum ArrowValidationLevel validation_level, - struct ArrowError* error); +/// Finish building with an explicit validation level. This could perform less +/// validation (i.e. NANOARROW_VALIDATION_LEVEL_NONE or +/// NANOARROW_VALIDATION_LEVEL_MINIMAL) if CPU buffer data access is not +/// possible or more validation (i.e., NANOARROW_VALIDATION_LEVEL_FULL) if +/// buffer content was obtained from an untrusted or corruptible source. +ArrowErrorCode ArrowArrayFinishBuilding( + struct ArrowArray* array, + enum ArrowValidationLevel validation_level, + struct ArrowError* error); /// @} @@ -1983,66 +2079,71 @@ ArrowErrorCode ArrowArrayFinishBuilding(struct ArrowArray* array, /// @{ /// \brief Initialize the contents of an ArrowArrayView -void ArrowArrayViewInitFromType(struct ArrowArrayView* array_view, - enum ArrowType storage_type); +void ArrowArrayViewInitFromType( + struct ArrowArrayView* array_view, enum ArrowType storage_type); /// \brief Move an ArrowArrayView /// /// Transfers the ArrowArrayView data and lifecycle management to another /// address and resets the contents of src. -static inline void ArrowArrayViewMove(struct ArrowArrayView* src, - struct ArrowArrayView* dst); +static inline void ArrowArrayViewMove( + struct ArrowArrayView* src, struct ArrowArrayView* dst); /// \brief Initialize the contents of an ArrowArrayView from an ArrowSchema -ArrowErrorCode ArrowArrayViewInitFromSchema(struct ArrowArrayView* array_view, - const struct ArrowSchema* schema, - struct ArrowError* error); +ArrowErrorCode ArrowArrayViewInitFromSchema( + struct ArrowArrayView* array_view, + const struct ArrowSchema* schema, + struct ArrowError* error); /// \brief Allocate the array_view->children array /// /// Includes the memory for each child struct ArrowArrayView -ArrowErrorCode ArrowArrayViewAllocateChildren(struct ArrowArrayView* array_view, - int64_t n_children); +ArrowErrorCode ArrowArrayViewAllocateChildren( + struct ArrowArrayView* array_view, int64_t n_children); /// \brief Allocate array_view->dictionary -ArrowErrorCode ArrowArrayViewAllocateDictionary(struct ArrowArrayView* array_view); +ArrowErrorCode ArrowArrayViewAllocateDictionary( + struct ArrowArrayView* array_view); /// \brief Set data-independent buffer sizes from length void ArrowArrayViewSetLength(struct ArrowArrayView* array_view, int64_t length); /// \brief Set buffer sizes and data pointers from an ArrowArray -ArrowErrorCode ArrowArrayViewSetArray(struct ArrowArrayView* array_view, - const struct ArrowArray* array, - struct ArrowError* error); +ArrowErrorCode ArrowArrayViewSetArray( + struct ArrowArrayView* array_view, + const struct ArrowArray* array, + struct ArrowError* error); -/// \brief Set buffer sizes and data pointers from an ArrowArray except for those -/// that require dereferencing buffer content. -ArrowErrorCode ArrowArrayViewSetArrayMinimal(struct ArrowArrayView* array_view, - const struct ArrowArray* array, - struct ArrowError* error); +/// \brief Set buffer sizes and data pointers from an ArrowArray except for +/// those that require dereferencing buffer content. +ArrowErrorCode ArrowArrayViewSetArrayMinimal( + struct ArrowArrayView* array_view, + const struct ArrowArray* array, + struct ArrowError* error); /// \brief Performs checks on the content of an ArrowArrayView /// /// If using ArrowArrayViewSetArray() to back array_view with an ArrowArray, /// the buffer sizes and some content (fist and last offset) have already /// been validated at the "default" level. If setting the buffer pointers -/// and sizes otherwise, you may wish to perform checks at a different level. See -/// documentation for ArrowValidationLevel for the details of checks performed -/// at each level. -ArrowErrorCode ArrowArrayViewValidate(struct ArrowArrayView* array_view, - enum ArrowValidationLevel validation_level, - struct ArrowError* error); +/// and sizes otherwise, you may wish to perform checks at a different level. +/// See documentation for ArrowValidationLevel for the details of checks +/// performed at each level. +ArrowErrorCode ArrowArrayViewValidate( + struct ArrowArrayView* array_view, + enum ArrowValidationLevel validation_level, + struct ArrowError* error); /// \brief Reset the contents of an ArrowArrayView and frees resources void ArrowArrayViewReset(struct ArrowArrayView* array_view); /// \brief Check for a null element in an ArrowArrayView -static inline int8_t ArrowArrayViewIsNull(const struct ArrowArrayView* array_view, - int64_t i); +static inline int8_t ArrowArrayViewIsNull( + const struct ArrowArrayView* array_view, int64_t i); /// \brief Get the type id of a union array element -static inline int8_t ArrowArrayViewUnionTypeId(const struct ArrowArrayView* array_view, - int64_t i); +static inline int8_t ArrowArrayViewUnionTypeId( + const struct ArrowArrayView* array_view, int64_t i); /// \brief Get the child index of a union array element static inline int8_t ArrowArrayViewUnionChildIndex( @@ -2054,15 +2155,15 @@ static inline int64_t ArrowArrayViewUnionChildOffset( /// \brief Get an element in an ArrowArrayView as an integer /// -/// This function does not check for null values, that values are actually integers, or -/// that values are within a valid range for an int64. -static inline int64_t ArrowArrayViewGetIntUnsafe(const struct ArrowArrayView* array_view, - int64_t i); +/// This function does not check for null values, that values are actually +/// integers, or that values are within a valid range for an int64. +static inline int64_t ArrowArrayViewGetIntUnsafe( + const struct ArrowArrayView* array_view, int64_t i); /// \brief Get an element in an ArrowArrayView as an unsigned integer /// -/// This function does not check for null values, that values are actually integers, or -/// that values are within a valid range for a uint64. +/// This function does not check for null values, that values are actually +/// integers, or that values are within a valid range for a uint64. static inline uint64_t ArrowArrayViewGetUIntUnsafe( const struct ArrowArrayView* array_view, int64_t i); @@ -2090,8 +2191,10 @@ static inline struct ArrowBufferView ArrowArrayViewGetBytesUnsafe( /// This function does not check for null values. The out parameter must /// be initialized with ArrowDecimalInit() with the proper parameters for this /// type before calling this for the first time. -static inline void ArrowArrayViewGetDecimalUnsafe(const struct ArrowArrayView* array_view, - int64_t i, struct ArrowDecimal* out); +static inline void ArrowArrayViewGetDecimalUnsafe( + const struct ArrowArrayView* array_view, + int64_t i, + struct ArrowDecimal* out); /// @} @@ -2109,8 +2212,10 @@ static inline void ArrowArrayViewGetDecimalUnsafe(const struct ArrowArrayView* a /// This function moves the ownership of schema to the array_stream. If /// this function returns NANOARROW_OK, the caller is responsible for /// releasing the ArrowArrayStream. -ArrowErrorCode ArrowBasicArrayStreamInit(struct ArrowArrayStream* array_stream, - struct ArrowSchema* schema, int64_t n_arrays); +ArrowErrorCode ArrowBasicArrayStreamInit( + struct ArrowArrayStream* array_stream, + struct ArrowSchema* schema, + int64_t n_arrays); /// \brief Set the ith ArrowArray in this ArrowArrayStream. /// @@ -2119,29 +2224,27 @@ ArrowErrorCode ArrowBasicArrayStreamInit(struct ArrowArrayStream* array_stream, /// be greater than zero and less than the value of n_arrays passed in /// ArrowBasicArrayStreamInit(). Callers are not required to fill all /// n_arrays members (i.e., n_arrays is a maximum bound). -void ArrowBasicArrayStreamSetArray(struct ArrowArrayStream* array_stream, int64_t i, - struct ArrowArray* array); +void ArrowBasicArrayStreamSetArray( + struct ArrowArrayStream* array_stream, int64_t i, struct ArrowArray* array); /// \brief Validate the contents of this ArrowArrayStream /// /// array_stream must have been initialized with ArrowBasicArrayStreamInit(). -/// This function uses ArrowArrayStreamInitFromSchema() and ArrowArrayStreamSetArray() -/// to validate the contents of the arrays. -ArrowErrorCode ArrowBasicArrayStreamValidate(const struct ArrowArrayStream* array_stream, - struct ArrowError* error); +/// This function uses ArrowArrayStreamInitFromSchema() and +/// ArrowArrayStreamSetArray() to validate the contents of the arrays. +ArrowErrorCode ArrowBasicArrayStreamValidate( + const struct ArrowArrayStream* array_stream, struct ArrowError* error); /// @} -// Undefine ArrowErrorCode, which may have been defined to annotate functions that return -// it to warn for an unused result. +// Undefine ArrowErrorCode, which may have been defined to annotate functions +// that return it to warn for an unused result. #if defined(ArrowErrorCode) #undef ArrowErrorCode #endif // Inline function definitions - - #ifdef __cplusplus } #endif @@ -2171,13 +2274,12 @@ ArrowErrorCode ArrowBasicArrayStreamValidate(const struct ArrowArrayStream* arra #include #include - - #ifdef __cplusplus extern "C" { #endif -static inline int64_t _ArrowGrowByFactor(int64_t current_capacity, int64_t new_capacity) { +static inline int64_t _ArrowGrowByFactor( + int64_t current_capacity, int64_t new_capacity) { int64_t doubled_capacity = current_capacity * 2; if (doubled_capacity > new_capacity) { return doubled_capacity; @@ -2205,8 +2307,8 @@ static inline ArrowErrorCode ArrowBufferSetAllocator( static inline void ArrowBufferReset(struct ArrowBuffer* buffer) { if (buffer->data != NULL) { - buffer->allocator.free(&buffer->allocator, (uint8_t*)buffer->data, - buffer->capacity_bytes); + buffer->allocator.free( + &buffer->allocator, (uint8_t*)buffer->data, buffer->capacity_bytes); buffer->data = NULL; } @@ -2214,22 +2316,27 @@ static inline void ArrowBufferReset(struct ArrowBuffer* buffer) { buffer->size_bytes = 0; } -static inline void ArrowBufferMove(struct ArrowBuffer* src, struct ArrowBuffer* dst) { +static inline void ArrowBufferMove( + struct ArrowBuffer* src, struct ArrowBuffer* dst) { memcpy(dst, src, sizeof(struct ArrowBuffer)); src->data = NULL; ArrowBufferReset(src); } -static inline ArrowErrorCode ArrowBufferResize(struct ArrowBuffer* buffer, - int64_t new_capacity_bytes, - char shrink_to_fit) { +static inline ArrowErrorCode ArrowBufferResize( + struct ArrowBuffer* buffer, + int64_t new_capacity_bytes, + char shrink_to_fit) { if (new_capacity_bytes < 0) { return EINVAL; } if (new_capacity_bytes > buffer->capacity_bytes || shrink_to_fit) { buffer->data = buffer->allocator.reallocate( - &buffer->allocator, buffer->data, buffer->capacity_bytes, new_capacity_bytes); + &buffer->allocator, + buffer->data, + buffer->capacity_bytes, + new_capacity_bytes); if (buffer->data == NULL && new_capacity_bytes > 0) { buffer->capacity_bytes = 0; buffer->size_bytes = 0; @@ -2247,95 +2354,97 @@ static inline ArrowErrorCode ArrowBufferResize(struct ArrowBuffer* buffer, return NANOARROW_OK; } -static inline ArrowErrorCode ArrowBufferReserve(struct ArrowBuffer* buffer, - int64_t additional_size_bytes) { +static inline ArrowErrorCode ArrowBufferReserve( + struct ArrowBuffer* buffer, int64_t additional_size_bytes) { int64_t min_capacity_bytes = buffer->size_bytes + additional_size_bytes; if (min_capacity_bytes <= buffer->capacity_bytes) { return NANOARROW_OK; } return ArrowBufferResize( - buffer, _ArrowGrowByFactor(buffer->capacity_bytes, min_capacity_bytes), 0); + buffer, + _ArrowGrowByFactor(buffer->capacity_bytes, min_capacity_bytes), + 0); } -static inline void ArrowBufferAppendUnsafe(struct ArrowBuffer* buffer, const void* data, - int64_t size_bytes) { +static inline void ArrowBufferAppendUnsafe( + struct ArrowBuffer* buffer, const void* data, int64_t size_bytes) { if (size_bytes > 0) { memcpy(buffer->data + buffer->size_bytes, data, size_bytes); buffer->size_bytes += size_bytes; } } -static inline ArrowErrorCode ArrowBufferAppend(struct ArrowBuffer* buffer, - const void* data, int64_t size_bytes) { +static inline ArrowErrorCode ArrowBufferAppend( + struct ArrowBuffer* buffer, const void* data, int64_t size_bytes) { NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, size_bytes)); ArrowBufferAppendUnsafe(buffer, data, size_bytes); return NANOARROW_OK; } -static inline ArrowErrorCode ArrowBufferAppendInt8(struct ArrowBuffer* buffer, - int8_t value) { +static inline ArrowErrorCode ArrowBufferAppendInt8( + struct ArrowBuffer* buffer, int8_t value) { return ArrowBufferAppend(buffer, &value, sizeof(int8_t)); } -static inline ArrowErrorCode ArrowBufferAppendUInt8(struct ArrowBuffer* buffer, - uint8_t value) { +static inline ArrowErrorCode ArrowBufferAppendUInt8( + struct ArrowBuffer* buffer, uint8_t value) { return ArrowBufferAppend(buffer, &value, sizeof(uint8_t)); } -static inline ArrowErrorCode ArrowBufferAppendInt16(struct ArrowBuffer* buffer, - int16_t value) { +static inline ArrowErrorCode ArrowBufferAppendInt16( + struct ArrowBuffer* buffer, int16_t value) { return ArrowBufferAppend(buffer, &value, sizeof(int16_t)); } -static inline ArrowErrorCode ArrowBufferAppendUInt16(struct ArrowBuffer* buffer, - uint16_t value) { +static inline ArrowErrorCode ArrowBufferAppendUInt16( + struct ArrowBuffer* buffer, uint16_t value) { return ArrowBufferAppend(buffer, &value, sizeof(uint16_t)); } -static inline ArrowErrorCode ArrowBufferAppendInt32(struct ArrowBuffer* buffer, - int32_t value) { +static inline ArrowErrorCode ArrowBufferAppendInt32( + struct ArrowBuffer* buffer, int32_t value) { return ArrowBufferAppend(buffer, &value, sizeof(int32_t)); } -static inline ArrowErrorCode ArrowBufferAppendUInt32(struct ArrowBuffer* buffer, - uint32_t value) { +static inline ArrowErrorCode ArrowBufferAppendUInt32( + struct ArrowBuffer* buffer, uint32_t value) { return ArrowBufferAppend(buffer, &value, sizeof(uint32_t)); } -static inline ArrowErrorCode ArrowBufferAppendInt64(struct ArrowBuffer* buffer, - int64_t value) { +static inline ArrowErrorCode ArrowBufferAppendInt64( + struct ArrowBuffer* buffer, int64_t value) { return ArrowBufferAppend(buffer, &value, sizeof(int64_t)); } -static inline ArrowErrorCode ArrowBufferAppendUInt64(struct ArrowBuffer* buffer, - uint64_t value) { +static inline ArrowErrorCode ArrowBufferAppendUInt64( + struct ArrowBuffer* buffer, uint64_t value) { return ArrowBufferAppend(buffer, &value, sizeof(uint64_t)); } -static inline ArrowErrorCode ArrowBufferAppendDouble(struct ArrowBuffer* buffer, - double value) { +static inline ArrowErrorCode ArrowBufferAppendDouble( + struct ArrowBuffer* buffer, double value) { return ArrowBufferAppend(buffer, &value, sizeof(double)); } -static inline ArrowErrorCode ArrowBufferAppendFloat(struct ArrowBuffer* buffer, - float value) { +static inline ArrowErrorCode ArrowBufferAppendFloat( + struct ArrowBuffer* buffer, float value) { return ArrowBufferAppend(buffer, &value, sizeof(float)); } -static inline ArrowErrorCode ArrowBufferAppendStringView(struct ArrowBuffer* buffer, - struct ArrowStringView value) { +static inline ArrowErrorCode ArrowBufferAppendStringView( + struct ArrowBuffer* buffer, struct ArrowStringView value) { return ArrowBufferAppend(buffer, value.data, value.size_bytes); } -static inline ArrowErrorCode ArrowBufferAppendBufferView(struct ArrowBuffer* buffer, - struct ArrowBufferView value) { +static inline ArrowErrorCode ArrowBufferAppendBufferView( + struct ArrowBuffer* buffer, struct ArrowBufferView value) { return ArrowBufferAppend(buffer, value.data.data, value.size_bytes); } -static inline ArrowErrorCode ArrowBufferAppendFill(struct ArrowBuffer* buffer, - uint8_t value, int64_t size_bytes) { +static inline ArrowErrorCode ArrowBufferAppendFill( + struct ArrowBuffer* buffer, uint8_t value, int64_t size_bytes) { NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, size_bytes)); memset(buffer->data + buffer->size_bytes, value, size_bytes); @@ -2344,20 +2453,24 @@ static inline ArrowErrorCode ArrowBufferAppendFill(struct ArrowBuffer* buffer, } static const uint8_t _ArrowkBitmask[] = {1, 2, 4, 8, 16, 32, 64, 128}; -static const uint8_t _ArrowkFlippedBitmask[] = {254, 253, 251, 247, 239, 223, 191, 127}; +static const uint8_t _ArrowkFlippedBitmask[] = { + 254, 253, 251, 247, 239, 223, 191, 127}; static const uint8_t _ArrowkPrecedingBitmask[] = {0, 1, 3, 7, 15, 31, 63, 127}; -static const uint8_t _ArrowkTrailingBitmask[] = {255, 254, 252, 248, 240, 224, 192, 128}; +static const uint8_t _ArrowkTrailingBitmask[] = { + 255, 254, 252, 248, 240, 224, 192, 128}; static const uint8_t _ArrowkBytePopcount[] = { - 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, - 4, 4, 5, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, - 4, 5, 4, 5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, - 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, - 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, - 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, - 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, - 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 3, 4, 4, 5, 4, 5, 5, 6, - 4, 5, 5, 6, 5, 6, 6, 7, 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8}; + 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, + 2, 3, 3, 4, 3, 4, 4, 5, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, + 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, + 4, 5, 5, 6, 5, 6, 6, 7, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, + 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, + 4, 5, 5, 6, 5, 6, 6, 7, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8}; static inline int64_t _ArrowRoundUpToMultipleOf8(int64_t value) { return (value + 7) & ~((int64_t)7); @@ -2394,25 +2507,25 @@ static inline void _ArrowBitsUnpackInt32(const uint8_t word, int32_t* out) { } static inline void _ArrowBitmapPackInt8(const int8_t* values, uint8_t* out) { - *out = (uint8_t)(values[0] | ((values[1] + 0x1) & 0x2) | ((values[2] + 0x3) & 0x4) | - ((values[3] + 0x7) & 0x8) | ((values[4] + 0xf) & 0x10) | - ((values[5] + 0x1f) & 0x20) | ((values[6] + 0x3f) & 0x40) | - ((values[7] + 0x7f) & 0x80)); + *out = (uint8_t)(values[0] | ((values[1] + 0x1) & 0x2) | + ((values[2] + 0x3) & 0x4) | ((values[3] + 0x7) & 0x8) | + ((values[4] + 0xf) & 0x10) | ((values[5] + 0x1f) & 0x20) | + ((values[6] + 0x3f) & 0x40) | ((values[7] + 0x7f) & 0x80)); } static inline void _ArrowBitmapPackInt32(const int32_t* values, uint8_t* out) { - *out = (uint8_t)(values[0] | ((values[1] + 0x1) & 0x2) | ((values[2] + 0x3) & 0x4) | - ((values[3] + 0x7) & 0x8) | ((values[4] + 0xf) & 0x10) | - ((values[5] + 0x1f) & 0x20) | ((values[6] + 0x3f) & 0x40) | - ((values[7] + 0x7f) & 0x80)); + *out = (uint8_t)(values[0] | ((values[1] + 0x1) & 0x2) | + ((values[2] + 0x3) & 0x4) | ((values[3] + 0x7) & 0x8) | + ((values[4] + 0xf) & 0x10) | ((values[5] + 0x1f) & 0x20) | + ((values[6] + 0x3f) & 0x40) | ((values[7] + 0x7f) & 0x80)); } static inline int8_t ArrowBitGet(const uint8_t* bits, int64_t i) { return (bits[i >> 3] >> (i & 0x07)) & 1; } -static inline void ArrowBitsUnpackInt8(const uint8_t* bits, int64_t start_offset, - int64_t length, int8_t* out) { +static inline void ArrowBitsUnpackInt8( + const uint8_t* bits, int64_t start_offset, int64_t length, int8_t* out) { if (length == 0) { return; } @@ -2450,8 +2563,8 @@ static inline void ArrowBitsUnpackInt8(const uint8_t* bits, int64_t start_offset } } -static inline void ArrowBitsUnpackInt32(const uint8_t* bits, int64_t start_offset, - int64_t length, int32_t* out) { +static inline void ArrowBitsUnpackInt32( + const uint8_t* bits, int64_t start_offset, int64_t length, int32_t* out) { if (length == 0) { return; } @@ -2498,12 +2611,12 @@ static inline void ArrowBitClear(uint8_t* bits, int64_t i) { } static inline void ArrowBitSetTo(uint8_t* bits, int64_t i, uint8_t bit_is_set) { - bits[i / 8] ^= - ((uint8_t)(-((uint8_t)(bit_is_set != 0)) ^ bits[i / 8])) & _ArrowkBitmask[i % 8]; + bits[i / 8] ^= ((uint8_t)(-((uint8_t)(bit_is_set != 0)) ^ bits[i / 8])) & + _ArrowkBitmask[i % 8]; } -static inline void ArrowBitsSetTo(uint8_t* bits, int64_t start_offset, int64_t length, - uint8_t bits_are_set) { +static inline void ArrowBitsSetTo( + uint8_t* bits, int64_t start_offset, int64_t length, uint8_t bits_are_set) { const int64_t i_begin = start_offset; const int64_t i_end = start_offset + length; const uint8_t fill_byte = (uint8_t)(-bits_are_set); @@ -2517,7 +2630,8 @@ static inline void ArrowBitsSetTo(uint8_t* bits, int64_t start_offset, int64_t l if (bytes_end == bytes_begin + 1) { // set bits within a single byte const uint8_t only_byte_mask = - i_end % 8 == 0 ? first_byte_mask : (uint8_t)(first_byte_mask | last_byte_mask); + i_end % 8 == 0 ? first_byte_mask : + (uint8_t)(first_byte_mask | last_byte_mask); bits[bytes_begin] &= only_byte_mask; bits[bytes_begin] |= (uint8_t)(fill_byte & ~only_byte_mask); return; @@ -2529,7 +2643,10 @@ static inline void ArrowBitsSetTo(uint8_t* bits, int64_t start_offset, int64_t l if (bytes_end - bytes_begin > 2) { // set/clear whole bytes - memset(bits + bytes_begin + 1, fill_byte, (size_t)(bytes_end - bytes_begin - 2)); + memset( + bits + bytes_begin + 1, + fill_byte, + (size_t)(bytes_end - bytes_begin - 2)); } if (i_end % 8 == 0) { @@ -2541,8 +2658,8 @@ static inline void ArrowBitsSetTo(uint8_t* bits, int64_t start_offset, int64_t l bits[bytes_end - 1] |= (uint8_t)(fill_byte & ~last_byte_mask); } -static inline int64_t ArrowBitCountSet(const uint8_t* bits, int64_t start_offset, - int64_t length) { +static inline int64_t ArrowBitCountSet( + const uint8_t* bits, int64_t start_offset, int64_t length) { if (length == 0) { return 0; } @@ -2560,14 +2677,16 @@ static inline int64_t ArrowBitCountSet(const uint8_t* bits, int64_t start_offset const uint8_t last_byte_mask = _ArrowkTrailingBitmask[i_begin % 8]; const uint8_t only_byte_mask = - i_end % 8 == 0 ? last_byte_mask : (uint8_t)(first_byte_mask & last_byte_mask); + i_end % 8 == 0 ? last_byte_mask : + (uint8_t)(first_byte_mask & last_byte_mask); const uint8_t byte_masked = bits[bytes_begin] & only_byte_mask; return _ArrowkBytePopcount[byte_masked]; } const uint8_t first_byte_mask = _ArrowkPrecedingBitmask[i_begin % 8]; - const uint8_t last_byte_mask = i_end % 8 == 0 ? 0 : _ArrowkTrailingBitmask[i_end % 8]; + const uint8_t last_byte_mask = + i_end % 8 == 0 ? 0 : _ArrowkTrailingBitmask[i_end % 8]; int64_t count = 0; // first byte @@ -2589,29 +2708,29 @@ static inline void ArrowBitmapInit(struct ArrowBitmap* bitmap) { bitmap->size_bits = 0; } -static inline void ArrowBitmapMove(struct ArrowBitmap* src, struct ArrowBitmap* dst) { +static inline void ArrowBitmapMove( + struct ArrowBitmap* src, struct ArrowBitmap* dst) { ArrowBufferMove(&src->buffer, &dst->buffer); dst->size_bits = src->size_bits; src->size_bits = 0; } -static inline ArrowErrorCode ArrowBitmapReserve(struct ArrowBitmap* bitmap, - int64_t additional_size_bits) { +static inline ArrowErrorCode ArrowBitmapReserve( + struct ArrowBitmap* bitmap, int64_t additional_size_bits) { int64_t min_capacity_bits = bitmap->size_bits + additional_size_bits; if (min_capacity_bits <= (bitmap->buffer.capacity_bytes * 8)) { return NANOARROW_OK; } - NANOARROW_RETURN_NOT_OK( - ArrowBufferReserve(&bitmap->buffer, _ArrowBytesForBits(additional_size_bits))); + NANOARROW_RETURN_NOT_OK(ArrowBufferReserve( + &bitmap->buffer, _ArrowBytesForBits(additional_size_bits))); bitmap->buffer.data[bitmap->buffer.capacity_bytes - 1] = 0; return NANOARROW_OK; } -static inline ArrowErrorCode ArrowBitmapResize(struct ArrowBitmap* bitmap, - int64_t new_capacity_bits, - char shrink_to_fit) { +static inline ArrowErrorCode ArrowBitmapResize( + struct ArrowBitmap* bitmap, int64_t new_capacity_bits, char shrink_to_fit) { if (new_capacity_bits < 0) { return EINVAL; } @@ -2627,23 +2746,23 @@ static inline ArrowErrorCode ArrowBitmapResize(struct ArrowBitmap* bitmap, return NANOARROW_OK; } -static inline ArrowErrorCode ArrowBitmapAppend(struct ArrowBitmap* bitmap, - uint8_t bits_are_set, int64_t length) { +static inline ArrowErrorCode ArrowBitmapAppend( + struct ArrowBitmap* bitmap, uint8_t bits_are_set, int64_t length) { NANOARROW_RETURN_NOT_OK(ArrowBitmapReserve(bitmap, length)); ArrowBitmapAppendUnsafe(bitmap, bits_are_set, length); return NANOARROW_OK; } -static inline void ArrowBitmapAppendUnsafe(struct ArrowBitmap* bitmap, - uint8_t bits_are_set, int64_t length) { +static inline void ArrowBitmapAppendUnsafe( + struct ArrowBitmap* bitmap, uint8_t bits_are_set, int64_t length) { ArrowBitsSetTo(bitmap->buffer.data, bitmap->size_bits, length, bits_are_set); bitmap->size_bits += length; bitmap->buffer.size_bytes = _ArrowBytesForBits(bitmap->size_bits); } -static inline void ArrowBitmapAppendInt8Unsafe(struct ArrowBitmap* bitmap, - const int8_t* values, int64_t n_values) { +static inline void ArrowBitmapAppendInt8Unsafe( + struct ArrowBitmap* bitmap, const int8_t* values, int64_t n_values) { if (n_values == 0) { return; } @@ -2655,7 +2774,8 @@ static inline void ArrowBitmapAppendInt8Unsafe(struct ArrowBitmap* bitmap, // First byte if ((out_i_cursor % 8) != 0) { - int64_t n_partial_bits = _ArrowRoundUpToMultipleOf8(out_i_cursor) - out_i_cursor; + int64_t n_partial_bits = + _ArrowRoundUpToMultipleOf8(out_i_cursor) - out_i_cursor; for (int i = 0; i < n_partial_bits; i++) { ArrowBitSetTo(bitmap->buffer.data, out_i_cursor++, values[i]); } @@ -2689,8 +2809,8 @@ static inline void ArrowBitmapAppendInt8Unsafe(struct ArrowBitmap* bitmap, bitmap->buffer.size_bytes = out_cursor - bitmap->buffer.data; } -static inline void ArrowBitmapAppendInt32Unsafe(struct ArrowBitmap* bitmap, - const int32_t* values, int64_t n_values) { +static inline void ArrowBitmapAppendInt32Unsafe( + struct ArrowBitmap* bitmap, const int32_t* values, int64_t n_values) { if (n_values == 0) { return; } @@ -2702,7 +2822,8 @@ static inline void ArrowBitmapAppendInt32Unsafe(struct ArrowBitmap* bitmap, // First byte if ((out_i_cursor % 8) != 0) { - int64_t n_partial_bits = _ArrowRoundUpToMultipleOf8(out_i_cursor) - out_i_cursor; + int64_t n_partial_bits = + _ArrowRoundUpToMultipleOf8(out_i_cursor) - out_i_cursor; for (int i = 0; i < n_partial_bits; i++) { ArrowBitSetTo(bitmap->buffer.data, out_i_cursor++, (uint8_t)values[i]); } @@ -2727,7 +2848,8 @@ static inline void ArrowBitmapAppendInt32Unsafe(struct ArrowBitmap* bitmap, // Zero out the last byte *out_cursor = 0x00; for (int i = 0; i < n_remaining; i++) { - ArrowBitSetTo(bitmap->buffer.data, out_i_cursor++, (uint8_t)values_cursor[i]); + ArrowBitSetTo( + bitmap->buffer.data, out_i_cursor++, (uint8_t)values_cursor[i]); } out_cursor++; } @@ -2772,20 +2894,19 @@ static inline void ArrowBitmapReset(struct ArrowBitmap* bitmap) { #include #include - - - #ifdef __cplusplus extern "C" { #endif -static inline struct ArrowBitmap* ArrowArrayValidityBitmap(struct ArrowArray* array) { +static inline struct ArrowBitmap* ArrowArrayValidityBitmap( + struct ArrowArray* array) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; return &private_data->bitmap; } -static inline struct ArrowBuffer* ArrowArrayBuffer(struct ArrowArray* array, int64_t i) { +static inline struct ArrowBuffer* ArrowArrayBuffer( + struct ArrowArray* array, int64_t i) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; switch (i) { @@ -2799,19 +2920,20 @@ static inline struct ArrowBuffer* ArrowArrayBuffer(struct ArrowArray* array, int // We don't currently support the case of unions where type_id != child_index; // however, these functions are used to keep track of where that assumption // is made. -static inline int8_t _ArrowArrayUnionChildIndex(struct ArrowArray* array, - int8_t type_id) { +static inline int8_t _ArrowArrayUnionChildIndex( + struct ArrowArray* array, int8_t type_id) { NANOARROW_UNUSED(array); return type_id; } -static inline int8_t _ArrowArrayUnionTypeId(struct ArrowArray* array, - int8_t child_index) { +static inline int8_t _ArrowArrayUnionTypeId( + struct ArrowArray* array, int8_t child_index) { NANOARROW_UNUSED(array); return child_index; } -static inline int32_t _ArrowParseUnionTypeIds(const char* type_ids, int8_t* out) { +static inline int32_t _ArrowParseUnionTypeIds( + const char* type_ids, int8_t* out) { if (*type_ids == '\0') { return 0; } @@ -2844,9 +2966,8 @@ static inline int32_t _ArrowParseUnionTypeIds(const char* type_ids, int8_t* out) return -1; } -static inline int8_t _ArrowParsedUnionTypeIdsWillEqualChildIndices(const int8_t* type_ids, - int64_t n_type_ids, - int64_t n_children) { +static inline int8_t _ArrowParsedUnionTypeIdsWillEqualChildIndices( + const int8_t* type_ids, int64_t n_type_ids, int64_t n_children) { if (n_type_ids != n_children) { return 0; } @@ -2860,14 +2981,16 @@ static inline int8_t _ArrowParsedUnionTypeIdsWillEqualChildIndices(const int8_t* return 1; } -static inline int8_t _ArrowUnionTypeIdsWillEqualChildIndices(const char* type_id_str, - int64_t n_children) { +static inline int8_t _ArrowUnionTypeIdsWillEqualChildIndices( + const char* type_id_str, int64_t n_children) { int8_t type_ids[128]; int32_t n_type_ids = _ArrowParseUnionTypeIds(type_id_str, type_ids); - return _ArrowParsedUnionTypeIdsWillEqualChildIndices(type_ids, n_type_ids, n_children); + return _ArrowParsedUnionTypeIdsWillEqualChildIndices( + type_ids, n_type_ids, n_children); } -static inline ArrowErrorCode ArrowArrayStartAppending(struct ArrowArray* array) { +static inline ArrowErrorCode ArrowArrayStartAppending( + struct ArrowArray* array) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -2891,12 +3014,17 @@ static inline ArrowErrorCode ArrowArrayStartAppending(struct ArrowArray* array) // Initialize any data offset buffer with a single zero for (int i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) { - if (private_data->layout.buffer_type[i] == NANOARROW_BUFFER_TYPE_DATA_OFFSET && + if (private_data->layout.buffer_type[i] == + NANOARROW_BUFFER_TYPE_DATA_OFFSET && private_data->layout.element_size_bits[i] == 64) { - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt64(ArrowArrayBuffer(array, i), 0)); - } else if (private_data->layout.buffer_type[i] == NANOARROW_BUFFER_TYPE_DATA_OFFSET && - private_data->layout.element_size_bits[i] == 32) { - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(ArrowArrayBuffer(array, i), 0)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendInt64(ArrowArrayBuffer(array, i), 0)); + } else if ( + private_data->layout.buffer_type[i] == + NANOARROW_BUFFER_TYPE_DATA_OFFSET && + private_data->layout.element_size_bits[i] == 32) { + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendInt32(ArrowArrayBuffer(array, i), 0)); } } @@ -2929,15 +3057,15 @@ static inline ArrowErrorCode ArrowArrayShrinkToFit(struct ArrowArray* array) { return NANOARROW_OK; } -static inline ArrowErrorCode _ArrowArrayAppendBits(struct ArrowArray* array, - int64_t buffer_i, uint8_t value, - int64_t n) { +static inline ArrowErrorCode _ArrowArrayAppendBits( + struct ArrowArray* array, int64_t buffer_i, uint8_t value, int64_t n) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; struct ArrowBuffer* buffer = ArrowArrayBuffer(array, buffer_i); int64_t bytes_required = - _ArrowRoundUpToMultipleOf8(private_data->layout.element_size_bits[buffer_i] * - (array->length + 1)) / + _ArrowRoundUpToMultipleOf8( + private_data->layout.element_size_bits[buffer_i] * + (array->length + 1)) / 8; if (bytes_required > buffer->size_bytes) { NANOARROW_RETURN_NOT_OK( @@ -2948,8 +3076,8 @@ static inline ArrowErrorCode _ArrowArrayAppendBits(struct ArrowArray* array, return NANOARROW_OK; } -static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal(struct ArrowArray* array, - int64_t n, uint8_t is_valid) { +static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal( + struct ArrowArray* array, int64_t n, uint8_t is_valid) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -2974,10 +3102,11 @@ static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal(struct ArrowArray* a ArrowBufferAppendFill(ArrowArrayBuffer(array, 0), type_id, n)); for (int64_t i = 0; i < n; i++) { NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32( - ArrowArrayBuffer(array, 1), (int32_t)array->children[0]->length - 1)); + ArrowArrayBuffer(array, 1), + (int32_t)array->children[0]->length - 1)); } - // For the purposes of array->null_count, union elements are never considered "null" - // even if some children contain nulls. + // For the purposes of array->null_count, union elements are never + // considered "null" even if some children contain nulls. array->length += n; return NANOARROW_OK; } @@ -2993,8 +3122,8 @@ static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal(struct ArrowArray* a NANOARROW_RETURN_NOT_OK( ArrowBufferAppendFill(ArrowArrayBuffer(array, 0), type_id, n)); - // For the purposes of array->null_count, union elements are never considered "null" - // even if some children contain nulls. + // For the purposes of array->null_count, union elements are never + // considered "null" even if some children contain nulls. array->length += n; return NANOARROW_OK; } @@ -3013,10 +3142,11 @@ static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal(struct ArrowArray* a break; } - // Append n is_valid bits to the validity bitmap. If we haven't allocated a bitmap yet - // and we need to append nulls, do it now. + // Append n is_valid bits to the validity bitmap. If we haven't allocated a + // bitmap yet and we need to append nulls, do it now. if (!is_valid && private_data->bitmap.buffer.data == NULL) { - NANOARROW_RETURN_NOT_OK(ArrowBitmapReserve(&private_data->bitmap, array->length + n)); + NANOARROW_RETURN_NOT_OK( + ArrowBitmapReserve(&private_data->bitmap, array->length + n)); ArrowBitmapAppendUnsafe(&private_data->bitmap, 1, array->length); ArrowBitmapAppendUnsafe(&private_data->bitmap, is_valid, n); } else if (private_data->bitmap.buffer.data != NULL) { @@ -3037,12 +3167,15 @@ static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal(struct ArrowArray* a case NANOARROW_BUFFER_TYPE_VALIDITY: continue; case NANOARROW_BUFFER_TYPE_DATA_OFFSET: - // Append the current value at the end of the offset buffer for each element + // Append the current value at the end of the offset buffer for each + // element NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, size_bytes * n)); for (int64_t j = 0; j < n; j++) { - ArrowBufferAppendUnsafe(buffer, buffer->data + size_bytes * (array->length + j), - size_bytes); + ArrowBufferAppendUnsafe( + buffer, + buffer->data + size_bytes * (array->length + j), + size_bytes); } // Skip the data buffer @@ -3051,7 +3184,8 @@ static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal(struct ArrowArray* a case NANOARROW_BUFFER_TYPE_DATA: // Zero out the next bit of memory if (private_data->layout.element_size_bits[i] % 8 == 0) { - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendFill(buffer, 0, size_bytes * n)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendFill(buffer, 0, size_bytes * n)); } else { NANOARROW_RETURN_NOT_OK(_ArrowArrayAppendBits(array, i, 0, n)); } @@ -3069,16 +3203,18 @@ static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal(struct ArrowArray* a return NANOARROW_OK; } -static inline ArrowErrorCode ArrowArrayAppendNull(struct ArrowArray* array, int64_t n) { +static inline ArrowErrorCode ArrowArrayAppendNull( + struct ArrowArray* array, int64_t n) { return _ArrowArrayAppendEmptyInternal(array, n, 0); } -static inline ArrowErrorCode ArrowArrayAppendEmpty(struct ArrowArray* array, int64_t n) { +static inline ArrowErrorCode ArrowArrayAppendEmpty( + struct ArrowArray* array, int64_t n) { return _ArrowArrayAppendEmptyInternal(array, n, 1); } -static inline ArrowErrorCode ArrowArrayAppendInt(struct ArrowArray* array, - int64_t value) { +static inline ArrowErrorCode ArrowArrayAppendInt( + struct ArrowArray* array, int64_t value) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -3086,19 +3222,23 @@ static inline ArrowErrorCode ArrowArrayAppendInt(struct ArrowArray* array, switch (private_data->storage_type) { case NANOARROW_TYPE_INT64: - NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(data_buffer, &value, sizeof(int64_t))); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppend(data_buffer, &value, sizeof(int64_t))); break; case NANOARROW_TYPE_INT32: _NANOARROW_CHECK_RANGE(value, INT32_MIN, INT32_MAX); - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(data_buffer, (int32_t)value)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendInt32(data_buffer, (int32_t)value)); break; case NANOARROW_TYPE_INT16: _NANOARROW_CHECK_RANGE(value, INT16_MIN, INT16_MAX); - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt16(data_buffer, (int16_t)value)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendInt16(data_buffer, (int16_t)value)); break; case NANOARROW_TYPE_INT8: _NANOARROW_CHECK_RANGE(value, INT8_MIN, INT8_MAX); - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt8(data_buffer, (int8_t)value)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendInt8(data_buffer, (int8_t)value)); break; case NANOARROW_TYPE_UINT64: case NANOARROW_TYPE_UINT32: @@ -3107,10 +3247,12 @@ static inline ArrowErrorCode ArrowArrayAppendInt(struct ArrowArray* array, _NANOARROW_CHECK_RANGE(value, 0, INT64_MAX); return ArrowArrayAppendUInt(array, value); case NANOARROW_TYPE_DOUBLE: - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendDouble(data_buffer, (double)value)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendDouble(data_buffer, (double)value)); break; case NANOARROW_TYPE_FLOAT: - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendFloat(data_buffer, (float)value)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendFloat(data_buffer, (float)value)); break; case NANOARROW_TYPE_BOOL: NANOARROW_RETURN_NOT_OK(_ArrowArrayAppendBits(array, 1, value != 0, 1)); @@ -3120,15 +3262,16 @@ static inline ArrowErrorCode ArrowArrayAppendInt(struct ArrowArray* array, } if (private_data->bitmap.buffer.data != NULL) { - NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); + NANOARROW_RETURN_NOT_OK( + ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); } array->length++; return NANOARROW_OK; } -static inline ArrowErrorCode ArrowArrayAppendUInt(struct ArrowArray* array, - uint64_t value) { +static inline ArrowErrorCode ArrowArrayAppendUInt( + struct ArrowArray* array, uint64_t value) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -3136,19 +3279,23 @@ static inline ArrowErrorCode ArrowArrayAppendUInt(struct ArrowArray* array, switch (private_data->storage_type) { case NANOARROW_TYPE_UINT64: - NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(data_buffer, &value, sizeof(uint64_t))); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppend(data_buffer, &value, sizeof(uint64_t))); break; case NANOARROW_TYPE_UINT32: _NANOARROW_CHECK_UPPER_LIMIT(value, UINT32_MAX); - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendUInt32(data_buffer, (uint32_t)value)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendUInt32(data_buffer, (uint32_t)value)); break; case NANOARROW_TYPE_UINT16: _NANOARROW_CHECK_UPPER_LIMIT(value, UINT16_MAX); - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendUInt16(data_buffer, (uint16_t)value)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendUInt16(data_buffer, (uint16_t)value)); break; case NANOARROW_TYPE_UINT8: _NANOARROW_CHECK_UPPER_LIMIT(value, UINT8_MAX); - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendUInt8(data_buffer, (uint8_t)value)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendUInt8(data_buffer, (uint8_t)value)); break; case NANOARROW_TYPE_INT64: case NANOARROW_TYPE_INT32: @@ -3157,10 +3304,12 @@ static inline ArrowErrorCode ArrowArrayAppendUInt(struct ArrowArray* array, _NANOARROW_CHECK_UPPER_LIMIT(value, INT64_MAX); return ArrowArrayAppendInt(array, value); case NANOARROW_TYPE_DOUBLE: - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendDouble(data_buffer, (double)value)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendDouble(data_buffer, (double)value)); break; case NANOARROW_TYPE_FLOAT: - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendFloat(data_buffer, (float)value)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendFloat(data_buffer, (float)value)); break; case NANOARROW_TYPE_BOOL: NANOARROW_RETURN_NOT_OK(_ArrowArrayAppendBits(array, 1, value != 0, 1)); @@ -3170,15 +3319,16 @@ static inline ArrowErrorCode ArrowArrayAppendUInt(struct ArrowArray* array, } if (private_data->bitmap.buffer.data != NULL) { - NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); + NANOARROW_RETURN_NOT_OK( + ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); } array->length++; return NANOARROW_OK; } -static inline ArrowErrorCode ArrowArrayAppendDouble(struct ArrowArray* array, - double value) { +static inline ArrowErrorCode ArrowArrayAppendDouble( + struct ArrowArray* array, double value) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -3186,31 +3336,35 @@ static inline ArrowErrorCode ArrowArrayAppendDouble(struct ArrowArray* array, switch (private_data->storage_type) { case NANOARROW_TYPE_DOUBLE: - NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(data_buffer, &value, sizeof(double))); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppend(data_buffer, &value, sizeof(double))); break; case NANOARROW_TYPE_FLOAT: - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendFloat(data_buffer, (float)value)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendFloat(data_buffer, (float)value)); break; default: return EINVAL; } if (private_data->bitmap.buffer.data != NULL) { - NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); + NANOARROW_RETURN_NOT_OK( + ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); } array->length++; return NANOARROW_OK; } -static inline ArrowErrorCode ArrowArrayAppendBytes(struct ArrowArray* array, - struct ArrowBufferView value) { +static inline ArrowErrorCode ArrowArrayAppendBytes( + struct ArrowArray* array, struct ArrowBufferView value) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; struct ArrowBuffer* offset_buffer = ArrowArrayBuffer(array, 1); struct ArrowBuffer* data_buffer = ArrowArrayBuffer( - array, 1 + (private_data->storage_type != NANOARROW_TYPE_FIXED_SIZE_BINARY)); + array, + 1 + (private_data->storage_type != NANOARROW_TYPE_FIXED_SIZE_BINARY)); int32_t offset; int64_t large_offset; int64_t fixed_size_bytes = private_data->layout.element_size_bits[1] / 8; @@ -3224,7 +3378,8 @@ static inline ArrowErrorCode ArrowArrayAppendBytes(struct ArrowArray* array, } offset += (int32_t)value.size_bytes; - NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(offset_buffer, &offset, sizeof(int32_t))); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppend(offset_buffer, &offset, sizeof(int32_t))); NANOARROW_RETURN_NOT_OK( ArrowBufferAppend(data_buffer, value.data.data, value.size_bytes)); break; @@ -3252,15 +3407,16 @@ static inline ArrowErrorCode ArrowArrayAppendBytes(struct ArrowArray* array, } if (private_data->bitmap.buffer.data != NULL) { - NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); + NANOARROW_RETURN_NOT_OK( + ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); } array->length++; return NANOARROW_OK; } -static inline ArrowErrorCode ArrowArrayAppendString(struct ArrowArray* array, - struct ArrowStringView value) { +static inline ArrowErrorCode ArrowArrayAppendString( + struct ArrowArray* array, struct ArrowStringView value) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -3279,8 +3435,8 @@ static inline ArrowErrorCode ArrowArrayAppendString(struct ArrowArray* array, } } -static inline ArrowErrorCode ArrowArrayAppendInterval(struct ArrowArray* array, - const struct ArrowInterval* value) { +static inline ArrowErrorCode ArrowArrayAppendInterval( + struct ArrowArray* array, const struct ArrowInterval* value) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -3292,7 +3448,8 @@ static inline ArrowErrorCode ArrowArrayAppendInterval(struct ArrowArray* array, return EINVAL; } - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(data_buffer, value->months)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendInt32(data_buffer, value->months)); break; } case NANOARROW_TYPE_INTERVAL_DAY_TIME: { @@ -3309,7 +3466,8 @@ static inline ArrowErrorCode ArrowArrayAppendInterval(struct ArrowArray* array, return EINVAL; } - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(data_buffer, value->months)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendInt32(data_buffer, value->months)); NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(data_buffer, value->days)); NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt64(data_buffer, value->ns)); break; @@ -3319,15 +3477,16 @@ static inline ArrowErrorCode ArrowArrayAppendInterval(struct ArrowArray* array, } if (private_data->bitmap.buffer.data != NULL) { - NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); + NANOARROW_RETURN_NOT_OK( + ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); } array->length++; return NANOARROW_OK; } -static inline ArrowErrorCode ArrowArrayAppendDecimal(struct ArrowArray* array, - const struct ArrowDecimal* value) { +static inline ArrowErrorCode ArrowArrayAppendDecimal( + struct ArrowArray* array, const struct ArrowDecimal* value) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; struct ArrowBuffer* data_buffer = ArrowArrayBuffer(array, 1); @@ -3354,7 +3513,8 @@ static inline ArrowErrorCode ArrowArrayAppendDecimal(struct ArrowArray* array, } if (private_data->bitmap.buffer.data != NULL) { - NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); + NANOARROW_RETURN_NOT_OK( + ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); } array->length++; @@ -3374,8 +3534,8 @@ static inline ArrowErrorCode ArrowArrayFinishElement(struct ArrowArray* array) { if (child_length > INT32_MAX) { return EOVERFLOW; } - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendInt32(ArrowArrayBuffer(array, 1), (int32_t)child_length)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32( + ArrowArrayBuffer(array, 1), (int32_t)child_length)); break; case NANOARROW_TYPE_LARGE_LIST: child_length = array->children[0]->length; @@ -3402,15 +3562,16 @@ static inline ArrowErrorCode ArrowArrayFinishElement(struct ArrowArray* array) { } if (private_data->bitmap.buffer.data != NULL) { - NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); + NANOARROW_RETURN_NOT_OK( + ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); } array->length++; return NANOARROW_OK; } -static inline ArrowErrorCode ArrowArrayFinishUnionElement(struct ArrowArray* array, - int8_t type_id) { +static inline ArrowErrorCode ArrowArrayFinishUnionElement( + struct ArrowArray* array, int8_t type_id) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -3422,15 +3583,19 @@ static inline ArrowErrorCode ArrowArrayFinishUnionElement(struct ArrowArray* arr switch (private_data->storage_type) { case NANOARROW_TYPE_DENSE_UNION: // Append the target child length to the union offsets buffer - _NANOARROW_CHECK_RANGE(array->children[child_index]->length, 0, INT32_MAX); + _NANOARROW_CHECK_RANGE( + array->children[child_index]->length, 0, INT32_MAX); NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32( - ArrowArrayBuffer(array, 1), (int32_t)array->children[child_index]->length - 1)); + ArrowArrayBuffer(array, 1), + (int32_t)array->children[child_index]->length - 1)); break; case NANOARROW_TYPE_SPARSE_UNION: - // Append one empty to any non-target column that isn't already the right length - // or abort if appending a null will result in a column with invalid length + // Append one empty to any non-target column that isn't already the right + // length or abort if appending a null will result in a column with + // invalid length for (int64_t i = 0; i < array->n_children; i++) { - if (i == child_index || array->children[i]->length == (array->length + 1)) { + if (i == child_index || + array->children[i]->length == (array->length + 1)) { continue; } @@ -3453,14 +3618,14 @@ static inline ArrowErrorCode ArrowArrayFinishUnionElement(struct ArrowArray* arr return NANOARROW_OK; } -static inline void ArrowArrayViewMove(struct ArrowArrayView* src, - struct ArrowArrayView* dst) { +static inline void ArrowArrayViewMove( + struct ArrowArrayView* src, struct ArrowArrayView* dst) { memcpy(dst, src, sizeof(struct ArrowArrayView)); ArrowArrayViewInitFromType(src, NANOARROW_TYPE_UNINITIALIZED); } -static inline int8_t ArrowArrayViewIsNull(const struct ArrowArrayView* array_view, - int64_t i) { +static inline int8_t ArrowArrayViewIsNull( + const struct ArrowArrayView* array_view, int64_t i) { const uint8_t* validity_buffer = array_view->buffer_views[0].data.as_uint8; i += array_view->offset; switch (array_view->storage_type) { @@ -3475,8 +3640,8 @@ static inline int8_t ArrowArrayViewIsNull(const struct ArrowArrayView* array_vie } } -static inline int8_t ArrowArrayViewUnionTypeId(const struct ArrowArrayView* array_view, - int64_t i) { +static inline int8_t ArrowArrayViewUnionTypeId( + const struct ArrowArrayView* array_view, int64_t i) { switch (array_view->storage_type) { case NANOARROW_TYPE_DENSE_UNION: case NANOARROW_TYPE_SPARSE_UNION: @@ -3520,8 +3685,8 @@ static inline int64_t ArrowArrayViewListChildOffset( } } -static inline int64_t ArrowArrayViewGetIntUnsafe(const struct ArrowArrayView* array_view, - int64_t i) { +static inline int64_t ArrowArrayViewGetIntUnsafe( + const struct ArrowArrayView* array_view, int64_t i) { const struct ArrowBufferView* data_view = &array_view->buffer_views[1]; i += array_view->offset; switch (array_view->storage_type) { @@ -3640,7 +3805,8 @@ static inline struct ArrowStringView ArrowArrayViewGetStringUnsafe( break; case NANOARROW_TYPE_FIXED_SIZE_BINARY: view.size_bytes = array_view->layout.element_size_bits[1] / 8; - view.data = array_view->buffer_views[1].data.as_char + (i * view.size_bytes); + view.data = + array_view->buffer_views[1].data.as_char + (i * view.size_bytes); break; default: view.data = NULL; @@ -3686,7 +3852,9 @@ static inline struct ArrowBufferView ArrowArrayViewGetBytesUnsafe( } static inline void ArrowArrayViewGetIntervalUnsafe( - const struct ArrowArrayView* array_view, int64_t i, struct ArrowInterval* out) { + const struct ArrowArrayView* array_view, + int64_t i, + struct ArrowInterval* out) { const uint8_t* data_view = array_view->buffer_views[1].data.as_uint8; switch (array_view->storage_type) { case NANOARROW_TYPE_INTERVAL_MONTHS: { @@ -3712,8 +3880,10 @@ static inline void ArrowArrayViewGetIntervalUnsafe( } } -static inline void ArrowArrayViewGetDecimalUnsafe(const struct ArrowArrayView* array_view, - int64_t i, struct ArrowDecimal* out) { +static inline void ArrowArrayViewGetDecimalUnsafe( + const struct ArrowArrayView* array_view, + int64_t i, + struct ArrowDecimal* out) { i += array_view->offset; const uint8_t* data_view = array_view->buffer_views[1].data.as_uint8; switch (array_view->storage_type) { diff --git a/libtiledbsoma/src/utils/arrow_adapter.cc b/libtiledbsoma/src/utils/arrow_adapter.cc index 13f66069e1..146cf99679 100644 --- a/libtiledbsoma/src/utils/arrow_adapter.cc +++ b/libtiledbsoma/src/utils/arrow_adapter.cc @@ -221,6 +221,137 @@ std::unique_ptr ArrowAdapter::arrow_schema_from_tiledb_array( return arrow_schema; } +ArraySchema ArrowAdapter::tiledb_schema_from_arrow_schema( + std::shared_ptr ctx, + std::shared_ptr arrow_schema, + ColumnIndexInfo index_column_info, + std::optional platform_config) { + auto [index_column_names, domains, extents] = index_column_info; + + ArraySchema schema(*ctx, TILEDB_SPARSE); + Domain domain(*ctx); + + if (platform_config) { + std::map convert_filter = { + {"GzipFilter", TILEDB_FILTER_GZIP}, + {"ZstdFilter", TILEDB_FILTER_ZSTD}, + {"LZ4Filter", TILEDB_FILTER_LZ4}, + {"Bzip2Filter", TILEDB_FILTER_BZIP2}, + {"RleFilter", TILEDB_FILTER_RLE}, + {"DeltaFilter", TILEDB_FILTER_DELTA}, + {"DoubleDeltaFilter", TILEDB_FILTER_DOUBLE_DELTA}, + {"BitWidthReductionFilter", TILEDB_FILTER_BIT_WIDTH_REDUCTION}, + {"BitShuffleFilter", TILEDB_FILTER_BITSHUFFLE}, + {"ByteShuffleFilter", TILEDB_FILTER_BYTESHUFFLE}, + {"PositiveDeltaFilter", TILEDB_FILTER_POSITIVE_DELTA}, + {"ChecksumMD5Filter", TILEDB_FILTER_CHECKSUM_MD5}, + {"ChecksumSHA256Filter", TILEDB_FILTER_CHECKSUM_SHA256}, + {"DictionaryFilter", TILEDB_FILTER_DICTIONARY}, + {"FloatScaleFilter", TILEDB_FILTER_SCALE_FLOAT}, + {"XORFilter", TILEDB_FILTER_XOR}, + {"WebpFilter", TILEDB_FILTER_WEBP}, + {"NoOpFilter", TILEDB_FILTER_NONE}, + }; + + schema.set_capacity(platform_config->capacity); + + if (platform_config->offsets_filters.size() != 0) { + FilterList offset_filter_list(*ctx); + for (auto offset : platform_config->offsets_filters) { + offset_filter_list.add_filter( + Filter(*ctx, convert_filter[offset])); + } + schema.set_offsets_filter_list(offset_filter_list); + } + + if (platform_config->validity_filters.size() != 0) { + FilterList validity_filter_list(*ctx); + for (auto validity : platform_config->validity_filters) { + validity_filter_list.add_filter( + Filter(*ctx, convert_filter[validity])); + } + schema.set_validity_filter_list(validity_filter_list); + } + + schema.set_allows_dups(platform_config->allows_duplicates); + + if (platform_config->tile_order) + schema.set_tile_order( + platform_config->tile_order == "row" ? TILEDB_ROW_MAJOR : + TILEDB_COL_MAJOR); + + if (platform_config->cell_order) + schema.set_cell_order( + platform_config->cell_order == "row" ? TILEDB_ROW_MAJOR : + TILEDB_COL_MAJOR); + } + + std::map dims; + + for (int64_t sch_idx = 0; sch_idx < arrow_schema->n_children; ++sch_idx) { + auto child = arrow_schema->children[sch_idx]; + auto type = ArrowAdapter::to_tiledb_format(child->format); + + auto idx_col_begin = index_column_names.begin(); + auto idx_col_end = index_column_names.end(); + auto idx_col_it = std::find(idx_col_begin, idx_col_end, child->name); + + if (idx_col_it != idx_col_end) { + auto idx_col_idx = std::distance(idx_col_begin, idx_col_it); + if (ArrowAdapter::_isvar(child->format)) { + type = TILEDB_STRING_ASCII; + } + + auto dim = Dimension::create( + *ctx, + child->name, + type, + type == TILEDB_STRING_ASCII ? + nullptr : + domains->children[idx_col_idx]->buffers[1], + type == TILEDB_STRING_ASCII ? + nullptr : + extents->children[idx_col_idx]->buffers[1]); + + dims.insert({dim.name(), dim}); + } else { + Attribute attr(*ctx, child->name, type); + + if (child->flags & ARROW_FLAG_NULLABLE) { + attr.set_nullable(true); + } + + if (ArrowAdapter::_isvar(child->format)) { + attr.set_cell_val_num(TILEDB_VAR_NUM); + } + + if (child->dictionary != nullptr) { + auto enmr_format = child->dictionary->format; + auto enmr_type = ArrowAdapter::to_tiledb_format(enmr_format); + auto enmr = Enumeration::create_empty( + *ctx, + child->name, + enmr_type, + ArrowAdapter::_isvar(enmr_format) ? TILEDB_VAR_NUM : 1, + child->flags & ARROW_FLAG_DICTIONARY_ORDERED); + ArraySchemaExperimental::add_enumeration(*ctx, schema, enmr); + AttributeExperimental::set_enumeration_name( + *ctx, attr, child->name); + } + + schema.add_attribute(attr); + } + } + + for (auto column_name : index_column_names) + domain.add_dimension(dims.at(column_name)); + schema.set_domain(domain); + + schema.check(); + + return schema; +} + std::pair ArrowAdapter::_get_data_and_length( Enumeration& enmr, const void* dst) { switch (enmr.type()) { @@ -473,63 +604,60 @@ ArrowAdapter::to_arrow(std::shared_ptr column) { return std::pair(std::move(array), std::move(schema)); } +bool ArrowAdapter::_isvar(const char* format) { + if ((strcmp(format, "U") == 0) || (strcmp(format, "Z") == 0) || + (strcmp(format, "u") == 0) || (strcmp(format, "z") == 0)) { + return true; + } + return false; +} + std::string_view ArrowAdapter::to_arrow_format( - tiledb_datatype_t datatype, bool use_large) { - switch (datatype) { - case TILEDB_STRING_ASCII: - case TILEDB_STRING_UTF8: - return use_large ? "U" : "u"; // large because TileDB - // uses 64bit offsets - case TILEDB_CHAR: - case TILEDB_BLOB: - return use_large ? "Z" : "z"; // large because TileDB - // uses 64bit offsets - case TILEDB_BOOL: - return "b"; - case TILEDB_INT32: - return "i"; - case TILEDB_INT64: - return "l"; - case TILEDB_FLOAT32: - return "f"; - case TILEDB_FLOAT64: - return "g"; - case TILEDB_INT8: - return "c"; - case TILEDB_UINT8: - return "C"; - case TILEDB_INT16: - return "s"; - case TILEDB_UINT16: - return "S"; - case TILEDB_UINT32: - return "I"; - case TILEDB_UINT64: - return "L"; - case TILEDB_TIME_SEC: - return "tts"; - case TILEDB_TIME_MS: - return "ttm"; - case TILEDB_TIME_US: - return "ttu"; - case TILEDB_TIME_NS: - return "ttn"; - case TILEDB_DATETIME_DAY: - return "tdD"; - case TILEDB_DATETIME_SEC: - return "tss:"; - case TILEDB_DATETIME_MS: - return "tsm:"; - case TILEDB_DATETIME_US: - return "tsu:"; - case TILEDB_DATETIME_NS: - return "tsn:"; - default: - break; + tiledb_datatype_t tiledb_dtype, bool use_large) { + auto u = use_large ? "U" : "u"; + auto z = use_large ? "Z" : "z"; + std::map _to_arrow_format_map = { + {TILEDB_STRING_ASCII, u}, {TILEDB_CHAR, z}, + {TILEDB_STRING_UTF8, u}, {TILEDB_BLOB, z}, + {TILEDB_INT8, "c"}, {TILEDB_UINT8, "C"}, + {TILEDB_INT16, "s"}, {TILEDB_UINT16, "S"}, + {TILEDB_INT32, "i"}, {TILEDB_UINT32, "I"}, + {TILEDB_INT64, "l"}, {TILEDB_UINT64, "L"}, + {TILEDB_FLOAT32, "f"}, {TILEDB_FLOAT64, "g"}, + {TILEDB_BOOL, "b"}, {TILEDB_DATETIME_SEC, "tss:"}, + {TILEDB_DATETIME_MS, "tsm:"}, {TILEDB_DATETIME_US, "tsu:"}, + {TILEDB_DATETIME_NS, "tsn:"}, + }; + + try { + return _to_arrow_format_map.at(tiledb_dtype); + } catch (const std::out_of_range& e) { + throw std::out_of_range(fmt::format( + "ArrowAdapter: Unsupported TileDB type: {} ", + tiledb::impl::type_to_str(tiledb_dtype))); + } +} + +tiledb_datatype_t ArrowAdapter::to_tiledb_format(std::string_view arrow_dtype) { + std::map _to_tiledb_format_map = { + {"u", TILEDB_STRING_UTF8}, {"U", TILEDB_STRING_UTF8}, + {"z", TILEDB_CHAR}, {"Z", TILEDB_CHAR}, + {"c", TILEDB_INT8}, {"C", TILEDB_UINT8}, + {"s", TILEDB_INT16}, {"S", TILEDB_UINT16}, + {"i", TILEDB_INT32}, {"I", TILEDB_UINT32}, + {"l", TILEDB_INT64}, {"L", TILEDB_UINT64}, + {"f", TILEDB_FLOAT32}, {"g", TILEDB_FLOAT64}, + {"b", TILEDB_BOOL}, {"tss:", TILEDB_DATETIME_SEC}, + {"tsm:", TILEDB_DATETIME_MS}, {"tsu:", TILEDB_DATETIME_US}, + {"tsn:", TILEDB_DATETIME_NS}, + }; + + try { + return _to_tiledb_format_map.at(arrow_dtype); + } catch (const std::out_of_range& e) { + throw std::out_of_range(fmt::format( + "ArrowAdapter: Unsupported Arrow type: {} ", arrow_dtype)); } - throw TileDBSOMAError(fmt::format( - "ArrowAdapter: Unsupported TileDB datatype: {} ", - tiledb::impl::type_to_str(datatype))); } // FIXME: Add more types, maybe make it a map diff --git a/libtiledbsoma/src/utils/arrow_adapter.h b/libtiledbsoma/src/utils/arrow_adapter.h index 818f5cc370..5ac12bca38 100644 --- a/libtiledbsoma/src/utils/arrow_adapter.h +++ b/libtiledbsoma/src/utils/arrow_adapter.h @@ -31,6 +31,32 @@ struct ArrowBuffer { std::shared_ptr buffer_; }; +using ArrowTable = + std::pair, std::shared_ptr>; + +using ColumnIndexInfo = std::tuple< + std::vector, // name of column + std::shared_ptr, // domain + std::shared_ptr // tile extent + >; + +class PlatformConfig { + public: + uint64_t dataframe_dim_zstd_level = 3; + uint64_t sparse_nd_array_dim_zstd_level = 3; + bool write_X_chunked = true; + uint64_t goal_chunk_nnz = 100000000; + uint64_t remote_cap_nbytes = 2400000000; + uint64_t capacity = 100000; + std::vector offsets_filters = { + "DoubleDeltaFilter", "BitWidthReductionFilter", "ZstdFilter"}; + std::vector validity_filters; + bool allows_duplicates = false; + std::optional tile_order = std::nullopt; + std::optional cell_order = std::nullopt; + bool consolidate_and_vacuum = false; +}; + class ArrowAdapter { public: static void release_schema(struct ArrowSchema* schema); @@ -47,17 +73,41 @@ class ArrowAdapter { static std::pair, std::unique_ptr> to_arrow(std::shared_ptr column); + /** + * @brief Create a an ArrowSchema from TileDB Schema + * + * @return ArrowSchema + */ static std::unique_ptr arrow_schema_from_tiledb_array( std::shared_ptr ctx, std::shared_ptr tiledb_array); + /** + * @brief Create a TileDB ArraySchema from ArrowSchema + * + * @return tiledb::ArraySchema + */ + static ArraySchema tiledb_schema_from_arrow_schema( + std::shared_ptr ctx, + std::shared_ptr arrow_schema, + ColumnIndexInfo index_column_info, + std::optional platform_config); + /** * @brief Get Arrow format string from TileDB datatype. * - * @param datatype TileDB datatype. + * @param tiledb_dtype TileDB datatype. * @return std::string_view Arrow format string. */ static std::string_view to_arrow_format( - tiledb_datatype_t datatype, bool use_large = true); + tiledb_datatype_t tiledb_dtype, bool use_large = true); + + /** + * @brief Get TileDB datatype from Arrow format string. + * + * @param datatype TileDB datatype. + * @return std::string_view Arrow format string. + */ + static tiledb_datatype_t to_tiledb_format(std::string_view arrow_dtype); static enum ArrowType to_nanoarrow_type(std::string_view sv); @@ -72,6 +122,11 @@ class ArrowAdapter { std::memcpy((void*)dst, src.data(), sz); return dst; } + + static std::optional> _get_dim_info( + std::string_view dim_name, ArrowTable index_columns); + + static bool _isvar(const char* format); }; }; // namespace tiledbsoma From 5fa143e43c88cafa4dfa7e656c7d1b32335e1673 Mon Sep 17 00:00:00 2001 From: Vivian Nguyen Date: Tue, 9 Apr 2024 12:09:06 -0500 Subject: [PATCH 02/10] Restore nanoarrow.h --- .../external/include/nanoarrow/nanoarrow.h | 1274 +++++++---------- 1 file changed, 552 insertions(+), 722 deletions(-) diff --git a/libtiledbsoma/src/external/include/nanoarrow/nanoarrow.h b/libtiledbsoma/src/external/include/nanoarrow/nanoarrow.h index fa99f12f25..e338560f1a 100644 --- a/libtiledbsoma/src/external/include/nanoarrow/nanoarrow.h +++ b/libtiledbsoma/src/external/include/nanoarrow/nanoarrow.h @@ -53,6 +53,8 @@ #include #include + + #if defined(NANOARROW_DEBUG) && !defined(NANOARROW_PRINT_AND_DIE) #include #include @@ -68,11 +70,11 @@ extern "C" { /// \defgroup nanoarrow-arrow-cdata Arrow C Data interface /// /// The Arrow C Data (https://arrow.apache.org/docs/format/CDataInterface.html) -/// and Arrow C Stream -/// (https://arrow.apache.org/docs/format/CStreamInterface.html) interfaces are -/// part of the Arrow Columnar Format specification -/// (https://arrow.apache.org/docs/format/Columnar.html). See the Arrow -/// documentation for documentation of these structures. +/// and Arrow C Stream (https://arrow.apache.org/docs/format/CStreamInterface.html) +/// interfaces are part of the +/// Arrow Columnar Format specification +/// (https://arrow.apache.org/docs/format/Columnar.html). See the Arrow documentation for +/// documentation of these structures. /// /// @{ @@ -127,8 +129,7 @@ struct ArrowArrayStream { // // Return value: 0 if successful, an `errno`-compatible error code otherwise. // - // If successful, the ArrowSchema must be released independently from the - // stream. + // If successful, the ArrowSchema must be released independently from the stream. int (*get_schema)(struct ArrowArrayStream*, struct ArrowSchema* out); // Callback to get the next array @@ -136,8 +137,7 @@ struct ArrowArrayStream { // // Return value: 0 if successful, an `errno`-compatible error code otherwise. // - // If successful, the ArrowArray must be released independently from the - // stream. + // If successful, the ArrowArray must be released independently from the stream. int (*get_next)(struct ArrowArrayStream*, struct ArrowArray* out); // Callback to get optional detailed error information. @@ -171,8 +171,7 @@ struct ArrowArrayStream { #define _NANOARROW_RETURN_NOT_OK_IMPL(NAME, EXPR) \ do { \ const int NAME = (EXPR); \ - if (NAME) \ - return NAME; \ + if (NAME) return NAME; \ } while (0) #define _NANOARROW_CHECK_RANGE(x_, min_, max_) \ @@ -182,38 +181,29 @@ struct ArrowArrayStream { NANOARROW_RETURN_NOT_OK((x_ <= max_) ? NANOARROW_OK : EINVAL) #if defined(NANOARROW_DEBUG) -#define _NANOARROW_RETURN_NOT_OK_WITH_ERROR_IMPL( \ - NAME, EXPR, ERROR_PTR_EXPR, EXPR_STR) \ - do { \ - const int NAME = (EXPR); \ - if (NAME) { \ - ArrowErrorSet( \ - (ERROR_PTR_EXPR), \ - "%s failed with errno %d\n* %s:%d", \ - EXPR_STR, \ - NAME, \ - __FILE__, \ - __LINE__); \ - return NAME; \ - } \ +#define _NANOARROW_RETURN_NOT_OK_WITH_ERROR_IMPL(NAME, EXPR, ERROR_PTR_EXPR, EXPR_STR) \ + do { \ + const int NAME = (EXPR); \ + if (NAME) { \ + ArrowErrorSet((ERROR_PTR_EXPR), "%s failed with errno %d\n* %s:%d", EXPR_STR, \ + NAME, __FILE__, __LINE__); \ + return NAME; \ + } \ } while (0) #else -#define _NANOARROW_RETURN_NOT_OK_WITH_ERROR_IMPL( \ - NAME, EXPR, ERROR_PTR_EXPR, EXPR_STR) \ - do { \ - const int NAME = (EXPR); \ - if (NAME) { \ - ArrowErrorSet( \ - (ERROR_PTR_EXPR), "%s failed with errno %d", EXPR_STR, NAME); \ - return NAME; \ - } \ +#define _NANOARROW_RETURN_NOT_OK_WITH_ERROR_IMPL(NAME, EXPR, ERROR_PTR_EXPR, EXPR_STR) \ + do { \ + const int NAME = (EXPR); \ + if (NAME) { \ + ArrowErrorSet((ERROR_PTR_EXPR), "%s failed with errno %d", EXPR_STR, NAME); \ + return NAME; \ + } \ } while (0) #endif #if defined(NANOARROW_DEBUG) // For checking ArrowErrorSet() calls for valid printf format strings/arguments -// If using mingw's c99-compliant printf, we need a different format-checking -// attribute +// If using mingw's c99-compliant printf, we need a different format-checking attribute #if defined(__USE_MINGW_ANSI_STDIO) && defined(__MINGW_PRINTF_FORMAT) #define NANOARROW_CHECK_PRINTF_ATTRIBUTE \ __attribute__((format(__MINGW_PRINTF_FORMAT, 2, 3))) @@ -253,9 +243,8 @@ typedef int ArrowErrorCode; /// \brief Flags supported by ArrowSchemaViewInit() /// \ingroup nanoarrow-schema-view -#define NANOARROW_FLAG_ALL_SUPPORTED \ - (ARROW_FLAG_DICTIONARY_ORDERED | ARROW_FLAG_NULLABLE | \ - ARROW_FLAG_MAP_KEYS_SORTED) +#define NANOARROW_FLAG_ALL_SUPPORTED \ + (ARROW_FLAG_DICTIONARY_ORDERED | ARROW_FLAG_NULLABLE | ARROW_FLAG_MAP_KEYS_SORTED) /// \brief Error type containing a UTF-8 encoded message. /// \ingroup nanoarrow-errors @@ -264,8 +253,8 @@ struct ArrowError { char message[1024]; }; -/// \brief Ensure an ArrowError is null-terminated by zeroing the first -/// character. \ingroup nanoarrow-errors +/// \brief Ensure an ArrowError is null-terminated by zeroing the first character. +/// \ingroup nanoarrow-errors /// /// If error is NULL, this function does nothing. static inline void ArrowErrorInit(struct ArrowError* error) { @@ -291,8 +280,7 @@ static inline const char* ArrowErrorMessage(struct ArrowError* error) { /// \ingroup nanoarrow-errors /// /// If error is NULL, this function does nothing. -static inline void ArrowErrorSetString( - struct ArrowError* error, const char* src) { +static inline void ArrowErrorSetString(struct ArrowError* error, const char* src) { if (error == NULL) { return; } @@ -310,34 +298,25 @@ static inline void ArrowErrorSetString( /// \brief Check the result of an expression and return it if not NANOARROW_OK /// \ingroup nanoarrow-errors #define NANOARROW_RETURN_NOT_OK(EXPR) \ - _NANOARROW_RETURN_NOT_OK_IMPL( \ - _NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), EXPR) + _NANOARROW_RETURN_NOT_OK_IMPL(_NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), EXPR) /// \brief Check the result of an expression and return it if not NANOARROW_OK, /// adding an auto-generated message to an ArrowError. /// \ingroup nanoarrow-errors /// /// This macro is used to ensure that functions that accept an ArrowError -/// as input always set its message when returning an error code (e.g., when -/// calling a nanoarrow function that does *not* accept ArrowError). +/// as input always set its message when returning an error code (e.g., when calling +/// a nanoarrow function that does *not* accept ArrowError). #define NANOARROW_RETURN_NOT_OK_WITH_ERROR(EXPR, ERROR_EXPR) \ _NANOARROW_RETURN_NOT_OK_WITH_ERROR_IMPL( \ - _NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), \ - EXPR, \ - ERROR_EXPR, \ - #EXPR) + _NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), EXPR, ERROR_EXPR, #EXPR) #if defined(NANOARROW_DEBUG) && !defined(NANOARROW_PRINT_AND_DIE) -#define NANOARROW_PRINT_AND_DIE(VALUE, EXPR_STR) \ - do { \ - fprintf( \ - stderr, \ - "%s failed with code %d\n* %s:%d\n", \ - EXPR_STR, \ - (int)(VALUE), \ - __FILE__, \ - (int)__LINE__); \ - abort(); \ +#define NANOARROW_PRINT_AND_DIE(VALUE, EXPR_STR) \ + do { \ + fprintf(stderr, "%s failed with code %d\n* %s:%d\n", EXPR_STR, (int)(VALUE), \ + __FILE__, (int)__LINE__); \ + abort(); \ } while (0) #endif @@ -345,27 +324,23 @@ static inline void ArrowErrorSetString( #define _NANOARROW_ASSERT_OK_IMPL(NAME, EXPR, EXPR_STR) \ do { \ const int NAME = (EXPR); \ - if (NAME) \ - NANOARROW_PRINT_AND_DIE(NAME, EXPR_STR); \ + if (NAME) NANOARROW_PRINT_AND_DIE(NAME, EXPR_STR); \ } while (0) /// \brief Assert that an expression's value is NANOARROW_OK /// \ingroup nanoarrow-errors /// -/// If nanoarrow was built in debug mode (i.e., defined(NANOARROW_DEBUG) is -/// true), print a message to stderr and abort. If nanoarrow was built in -/// release mode, this statement has no effect. You can customize fatal error -/// behaviour be defining the NANOARROW_PRINT_AND_DIE macro before including -/// nanoarrow.h This macro is provided as a convenience for users and is not -/// used internally. +/// If nanoarrow was built in debug mode (i.e., defined(NANOARROW_DEBUG) is true), +/// print a message to stderr and abort. If nanoarrow was built in release mode, +/// this statement has no effect. You can customize fatal error behaviour +/// be defining the NANOARROW_PRINT_AND_DIE macro before including nanoarrow.h +/// This macro is provided as a convenience for users and is not used internally. #define NANOARROW_ASSERT_OK(EXPR) \ - _NANOARROW_ASSERT_OK_IMPL( \ - _NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), EXPR, #EXPR) + _NANOARROW_ASSERT_OK_IMPL(_NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), EXPR, #EXPR) -#define _NANOARROW_DCHECK_IMPL(EXPR, EXPR_STR) \ - do { \ - if (!(EXPR)) \ - NANOARROW_PRINT_AND_DIE(-1, EXPR_STR); \ +#define _NANOARROW_DCHECK_IMPL(EXPR, EXPR_STR) \ + do { \ + if (!(EXPR)) NANOARROW_PRINT_AND_DIE(-1, EXPR_STR); \ } while (0) #define NANOARROW_DCHECK(EXPR) _NANOARROW_DCHECK_IMPL(EXPR, #EXPR) @@ -374,8 +349,7 @@ static inline void ArrowErrorSetString( #define NANOARROW_DCHECK(EXPR) #endif -static inline void ArrowSchemaMove( - struct ArrowSchema* src, struct ArrowSchema* dst) { +static inline void ArrowSchemaMove(struct ArrowSchema* src, struct ArrowSchema* dst) { NANOARROW_DCHECK(src != NULL); NANOARROW_DCHECK(dst != NULL); @@ -389,8 +363,7 @@ static inline void ArrowSchemaRelease(struct ArrowSchema* schema) { NANOARROW_DCHECK(schema->release == NULL); } -static inline void ArrowArrayMove( - struct ArrowArray* src, struct ArrowArray* dst) { +static inline void ArrowArrayMove(struct ArrowArray* src, struct ArrowArray* dst) { NANOARROW_DCHECK(src != NULL); NANOARROW_DCHECK(dst != NULL); @@ -404,8 +377,8 @@ static inline void ArrowArrayRelease(struct ArrowArray* array) { NANOARROW_DCHECK(array->release == NULL); } -static inline void ArrowArrayStreamMove( - struct ArrowArrayStream* src, struct ArrowArrayStream* dst) { +static inline void ArrowArrayStreamMove(struct ArrowArrayStream* src, + struct ArrowArrayStream* dst) { NANOARROW_DCHECK(src != NULL); NANOARROW_DCHECK(dst != NULL); @@ -426,8 +399,7 @@ static inline const char* ArrowArrayStreamGetLastError( } static inline ArrowErrorCode ArrowArrayStreamGetSchema( - struct ArrowArrayStream* array_stream, - struct ArrowSchema* out, + struct ArrowArrayStream* array_stream, struct ArrowSchema* out, struct ArrowError* error) { NANOARROW_DCHECK(array_stream != NULL); @@ -440,8 +412,7 @@ static inline ArrowErrorCode ArrowArrayStreamGetSchema( } static inline ArrowErrorCode ArrowArrayStreamGetNext( - struct ArrowArrayStream* array_stream, - struct ArrowArray* out, + struct ArrowArrayStream* array_stream, struct ArrowArray* out, struct ArrowError* error) { NANOARROW_DCHECK(array_stream != NULL); @@ -453,8 +424,7 @@ static inline ArrowErrorCode ArrowArrayStreamGetNext( return result; } -static inline void ArrowArrayStreamRelease( - struct ArrowArrayStream* array_stream) { +static inline void ArrowArrayStreamRelease(struct ArrowArrayStream* array_stream) { NANOARROW_DCHECK(array_stream != NULL); array_stream->release(array_stream); NANOARROW_DCHECK(array_stream->release == NULL); @@ -622,21 +592,16 @@ enum ArrowValidationLevel { /// \brief Do not validate buffer sizes or content. NANOARROW_VALIDATION_LEVEL_NONE = 0, - /// \brief Validate buffer sizes that depend on array length but do not - /// validate buffer + /// \brief Validate buffer sizes that depend on array length but do not validate buffer /// sizes that depend on buffer data access. NANOARROW_VALIDATION_LEVEL_MINIMAL = 1, - /// \brief Validate all buffer sizes, including those that require buffer data - /// access, - /// but do not perform any checks that are O(1) along the length of the - /// buffers. + /// \brief Validate all buffer sizes, including those that require buffer data access, + /// but do not perform any checks that are O(1) along the length of the buffers. NANOARROW_VALIDATION_LEVEL_DEFAULT = 2, - /// \brief Validate all buffer sizes and all buffer content. This is useful in - /// the - /// context of untrusted input or input that may have been corrupted in - /// transit. + /// \brief Validate all buffer sizes and all buffer content. This is useful in the + /// context of untrusted input or input that may have been corrupted in transit. NANOARROW_VALIDATION_LEVEL_FULL = 3 }; @@ -661,8 +626,8 @@ static inline const char* ArrowTimeUnitString(enum ArrowTimeUnit time_unit) { } } -/// \brief Functional types of buffers as described in the Arrow Columnar -/// Specification \ingroup nanoarrow-array-view +/// \brief Functional types of buffers as described in the Arrow Columnar Specification +/// \ingroup nanoarrow-array-view enum ArrowBufferType { NANOARROW_BUFFER_TYPE_NONE, NANOARROW_BUFFER_TYPE_VALIDITY, @@ -677,8 +642,7 @@ enum ArrowBufferType { /// /// All currently supported types have 3 buffers or fewer; however, future types /// may involve a variable number of buffers (e.g., string view). These buffers -/// will be represented by separate members of the ArrowArrayView or -/// ArrowLayout. +/// will be represented by separate members of the ArrowArrayView or ArrowLayout. #define NANOARROW_MAX_FIXED_BUFFERS 3 /// \brief An non-owning view of a string @@ -747,15 +711,11 @@ struct ArrowBufferView { /// an ArrowArray. struct ArrowBufferAllocator { /// \brief Reallocate a buffer or return NULL if it cannot be reallocated - uint8_t* (*reallocate)( - struct ArrowBufferAllocator* allocator, - uint8_t* ptr, - int64_t old_size, - int64_t new_size); + uint8_t* (*reallocate)(struct ArrowBufferAllocator* allocator, uint8_t* ptr, + int64_t old_size, int64_t new_size); /// \brief Deallocate a buffer allocated by this allocator - void (*free)( - struct ArrowBufferAllocator* allocator, uint8_t* ptr, int64_t size); + void (*free)(struct ArrowBufferAllocator* allocator, uint8_t* ptr, int64_t size); /// \brief Opaque data specific to the allocator void* private_data; @@ -775,8 +735,7 @@ struct ArrowBuffer { /// \brief The capacity of the buffer in bytes int64_t capacity_bytes; - /// \brief The allocator that will be used to reallocate and/or free the - /// buffer + /// \brief The allocator that will be used to reallocate and/or free the buffer struct ArrowBufferAllocator allocator; }; @@ -803,12 +762,11 @@ struct ArrowLayout { /// \brief The data type of each buffer enum ArrowType buffer_data_type[NANOARROW_MAX_FIXED_BUFFERS]; - /// \brief The size of an element each buffer or 0 if this size is variable or - /// unknown + /// \brief The size of an element each buffer or 0 if this size is variable or unknown int64_t element_size_bits[NANOARROW_MAX_FIXED_BUFFERS]; - /// \brief The number of elements in the child array per element in this array - /// for a fixed-size list + /// \brief The number of elements in the child array per element in this array for a + /// fixed-size list int64_t child_size_elements; }; @@ -911,8 +869,8 @@ struct ArrowInterval { /// \brief Zero initialize an Interval with a given unit /// \ingroup nanoarrow-utils -static inline void ArrowIntervalInit( - struct ArrowInterval* interval, enum ArrowType type) { +static inline void ArrowIntervalInit(struct ArrowInterval* interval, + enum ArrowType type) { memset(interval, 0, sizeof(struct ArrowInterval)); interval->type = type; } @@ -924,8 +882,7 @@ static inline void ArrowIntervalInit( /// values set using ArrowDecimalSetInt(), ArrowDecimalSetBytes128(), /// or ArrowDecimalSetBytes256(). struct ArrowDecimal { - /// \brief An array of 64-bit integers of n_words length defined in - /// native-endian order + /// \brief An array of 64-bit integers of n_words length defined in native-endian order uint64_t words[4]; /// \brief The number of significant digits this decimal number can represent @@ -946,11 +903,8 @@ struct ArrowDecimal { /// \brief Initialize a decimal with a given set of type parameters /// \ingroup nanoarrow-utils -static inline void ArrowDecimalInit( - struct ArrowDecimal* decimal, - int32_t bitwidth, - int32_t precision, - int32_t scale) { +static inline void ArrowDecimalInit(struct ArrowDecimal* decimal, int32_t bitwidth, + int32_t precision, int32_t scale) { memset(decimal->words, 0, sizeof(decimal->words)); decimal->precision = precision; decimal->scale = scale; @@ -970,15 +924,14 @@ static inline void ArrowDecimalInit( /// This does not check if the decimal's precision sufficiently small to fit /// within the signed 64-bit integer range (A precision less than or equal /// to 18 is sufficiently small). -static inline int64_t ArrowDecimalGetIntUnsafe( - const struct ArrowDecimal* decimal) { +static inline int64_t ArrowDecimalGetIntUnsafe(const struct ArrowDecimal* decimal) { return (int64_t)decimal->words[decimal->low_word_index]; } /// \brief Copy the bytes of this decimal into a sufficiently large buffer /// \ingroup nanoarrow-utils -static inline void ArrowDecimalGetBytes( - const struct ArrowDecimal* decimal, uint8_t* out) { +static inline void ArrowDecimalGetBytes(const struct ArrowDecimal* decimal, + uint8_t* out) { memcpy(out, decimal->words, decimal->n_words * sizeof(uint64_t)); } @@ -990,8 +943,7 @@ static inline int64_t ArrowDecimalSign(const struct ArrowDecimal* decimal) { /// \brief Sets the integer value of this decimal /// \ingroup nanoarrow-utils -static inline void ArrowDecimalSetInt( - struct ArrowDecimal* decimal, int64_t value) { +static inline void ArrowDecimalSetInt(struct ArrowDecimal* decimal, int64_t value) { if (value < 0) { memset(decimal->words, 0xff, decimal->n_words * sizeof(uint64_t)); } else { @@ -1025,8 +977,8 @@ static inline void ArrowDecimalNegate(struct ArrowDecimal* decimal) { /// \brief Copy bytes from a buffer into this decimal /// \ingroup nanoarrow-utils -static inline void ArrowDecimalSetBytes( - struct ArrowDecimal* decimal, const uint8_t* value) { +static inline void ArrowDecimalSetBytes(struct ArrowDecimal* decimal, + const uint8_t* value) { memcpy(decimal->words, value, decimal->n_words * sizeof(uint64_t)); } @@ -1059,9 +1011,11 @@ static inline void ArrowDecimalSetBytes( #include #include -// If using CMake, optionally pass -DNANOARROW_NAMESPACE=MyNamespace which will -// set this define in nanoarrow_config.h. If not, you can optionally #define -// NANOARROW_NAMESPACE MyNamespace here. + + +// If using CMake, optionally pass -DNANOARROW_NAMESPACE=MyNamespace which will set this +// define in nanoarrow_config.h. If not, you can optionally #define NANOARROW_NAMESPACE +// MyNamespace here. // This section remaps the non-prefixed symbols to the prefixed symbols so that // code written against this build can be used independent of the value of @@ -1070,8 +1024,7 @@ static inline void ArrowDecimalSetBytes( #define NANOARROW_CAT(A, B) A##B #define NANOARROW_SYMBOL(A, B) NANOARROW_CAT(A, B) -#define ArrowNanoarrowVersion \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowNanoarrowVersion) +#define ArrowNanoarrowVersion NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowNanoarrowVersion) #define ArrowNanoarrowVersionInt \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowNanoarrowVersionInt) #define ArrowMalloc NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMalloc) @@ -1083,15 +1036,13 @@ static inline void ArrowDecimalSetBytes( NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowBufferDeallocator) #define ArrowErrorSet NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowErrorSet) #define ArrowLayoutInit NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowLayoutInit) -#define ArrowDecimalSetDigits \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowDecimalSetDigits) +#define ArrowDecimalSetDigits NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowDecimalSetDigits) #define ArrowDecimalAppendDigitsToBuffer \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowDecimalAppendDigitsToBuffer) #define ArrowSchemaInit NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaInit) #define ArrowSchemaInitFromType \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaInitFromType) -#define ArrowSchemaSetType \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetType) +#define ArrowSchemaSetType NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetType) #define ArrowSchemaSetTypeStruct \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetTypeStruct) #define ArrowSchemaSetTypeFixedSize \ @@ -1102,12 +1053,9 @@ static inline void ArrowDecimalSetBytes( NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetTypeDateTime) #define ArrowSchemaSetTypeUnion \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetTypeUnion) -#define ArrowSchemaDeepCopy \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaDeepCopy) -#define ArrowSchemaSetFormat \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetFormat) -#define ArrowSchemaSetName \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetName) +#define ArrowSchemaDeepCopy NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaDeepCopy) +#define ArrowSchemaSetFormat NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetFormat) +#define ArrowSchemaSetName NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetName) #define ArrowSchemaSetMetadata \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetMetadata) #define ArrowSchemaAllocateChildren \ @@ -1118,12 +1066,9 @@ static inline void ArrowDecimalSetBytes( NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataReaderInit) #define ArrowMetadataReaderRead \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataReaderRead) -#define ArrowMetadataSizeOf \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataSizeOf) -#define ArrowMetadataHasKey \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataHasKey) -#define ArrowMetadataGetValue \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataGetValue) +#define ArrowMetadataSizeOf NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataSizeOf) +#define ArrowMetadataHasKey NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataHasKey) +#define ArrowMetadataGetValue NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataGetValue) #define ArrowMetadataBuilderInit \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataBuilderInit) #define ArrowMetadataBuilderAppend \ @@ -1132,10 +1077,8 @@ static inline void ArrowDecimalSetBytes( NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataBuilderSet) #define ArrowMetadataBuilderRemove \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataBuilderRemove) -#define ArrowSchemaViewInit \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaViewInit) -#define ArrowSchemaToString \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaToString) +#define ArrowSchemaViewInit NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaViewInit) +#define ArrowSchemaToString NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaToString) #define ArrowArrayInitFromType \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayInitFromType) #define ArrowArrayInitFromSchema \ @@ -1150,10 +1093,8 @@ static inline void ArrowDecimalSetBytes( NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayAllocateDictionary) #define ArrowArraySetValidityBitmap \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArraySetValidityBitmap) -#define ArrowArraySetBuffer \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArraySetBuffer) -#define ArrowArrayReserve \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayReserve) +#define ArrowArraySetBuffer NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArraySetBuffer) +#define ArrowArrayReserve NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayReserve) #define ArrowArrayFinishBuilding \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayFinishBuilding) #define ArrowArrayFinishBuildingDefault \ @@ -1174,8 +1115,7 @@ static inline void ArrowDecimalSetBytes( NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewSetArrayMinimal) #define ArrowArrayViewValidate \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewValidate) -#define ArrowArrayViewReset \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewReset) +#define ArrowArrayViewReset NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewReset) #define ArrowBasicArrayStreamInit \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowBasicArrayStreamInit) #define ArrowBasicArrayStreamSetArray \ @@ -1229,33 +1169,31 @@ struct ArrowBufferAllocator ArrowBufferAllocatorDefault(void); /// avoid copying an existing buffer that was not allocated using the /// infrastructure provided here (e.g., by an R or Python object). struct ArrowBufferAllocator ArrowBufferDeallocator( - void (*custom_free)( - struct ArrowBufferAllocator* allocator, uint8_t* ptr, int64_t size), + void (*custom_free)(struct ArrowBufferAllocator* allocator, uint8_t* ptr, + int64_t size), void* private_data); /// @} -/// \brief Move the contents of an src ArrowSchema into dst and set src->release -/// to NULL \ingroup nanoarrow-arrow-cdata -static inline void ArrowSchemaMove( - struct ArrowSchema* src, struct ArrowSchema* dst); +/// \brief Move the contents of an src ArrowSchema into dst and set src->release to NULL +/// \ingroup nanoarrow-arrow-cdata +static inline void ArrowSchemaMove(struct ArrowSchema* src, struct ArrowSchema* dst); /// \brief Call the release callback of an ArrowSchema /// \ingroup nanoarrow-arrow-cdata static inline void ArrowSchemaRelease(struct ArrowSchema* schema); -/// \brief Move the contents of an src ArrowArray into dst and set src->release -/// to NULL \ingroup nanoarrow-arrow-cdata -static inline void ArrowArrayMove( - struct ArrowArray* src, struct ArrowArray* dst); +/// \brief Move the contents of an src ArrowArray into dst and set src->release to NULL +/// \ingroup nanoarrow-arrow-cdata +static inline void ArrowArrayMove(struct ArrowArray* src, struct ArrowArray* dst); /// \brief Call the release callback of an ArrowArray static inline void ArrowArrayRelease(struct ArrowArray* array); -/// \brief Move the contents of an src ArrowArrayStream into dst and set -/// src->release to NULL \ingroup nanoarrow-arrow-cdata -static inline void ArrowArrayStreamMove( - struct ArrowArrayStream* src, struct ArrowArrayStream* dst); +/// \brief Move the contents of an src ArrowArrayStream into dst and set src->release to +/// NULL \ingroup nanoarrow-arrow-cdata +static inline void ArrowArrayStreamMove(struct ArrowArrayStream* src, + struct ArrowArrayStream* dst); /// \brief Call the get_schema callback of an ArrowArrayStream /// \ingroup nanoarrow-arrow-cdata @@ -1265,8 +1203,7 @@ static inline void ArrowArrayStreamMove( /// makes it significantly less verbose to iterate over array streams /// using NANOARROW_RETURN_NOT_OK()-style error handling. static inline ArrowErrorCode ArrowArrayStreamGetSchema( - struct ArrowArrayStream* array_stream, - struct ArrowSchema* out, + struct ArrowArrayStream* array_stream, struct ArrowSchema* out, struct ArrowError* error); /// \brief Call the get_schema callback of an ArrowArrayStream @@ -1277,22 +1214,20 @@ static inline ArrowErrorCode ArrowArrayStreamGetSchema( /// makes it significantly less verbose to iterate over array streams /// using NANOARROW_RETURN_NOT_OK()-style error handling. static inline ArrowErrorCode ArrowArrayStreamGetNext( - struct ArrowArrayStream* array_stream, - struct ArrowArray* out, + struct ArrowArrayStream* array_stream, struct ArrowArray* out, struct ArrowError* error); /// \brief Call the get_next callback of an ArrowArrayStream /// \ingroup nanoarrow-arrow-cdata /// /// Unlike the get_next callback, this function never returns NULL (i.e., its -/// result is safe to use in printf-style error formatters). Null values from -/// the original callback are reported as "". +/// result is safe to use in printf-style error formatters). Null values from the +/// original callback are reported as "". static inline const char* ArrowArrayStreamGetLastError( struct ArrowArrayStream* array_stream); /// \brief Call the release callback of an ArrowArrayStream -static inline void ArrowArrayStreamRelease( - struct ArrowArrayStream* array_stream); +static inline void ArrowArrayStreamRelease(struct ArrowArrayStream* array_stream); /// \defgroup nanoarrow-errors Error handling /// @@ -1300,24 +1235,24 @@ static inline void ArrowArrayStreamRelease( /// need to communicate more verbose error information accept a pointer /// to an ArrowError. This can be stack or statically allocated. The /// content of the message is undefined unless an error code has been -/// returned. If a nanoarrow function is passed a non-null ArrowError pointer, -/// the ArrowError pointed to by the argument will be propagated with a +/// returned. If a nanoarrow function is passed a non-null ArrowError pointer, the +/// ArrowError pointed to by the argument will be propagated with a /// null-terminated error message. It is safe to pass a NULL ArrowError anywhere /// in the nanoarrow API. /// /// Except where documented, it is generally not safe to continue after a -/// function has returned a non-zero ArrowErrorCode. The NANOARROW_RETURN_NOT_OK -/// and NANOARROW_ASSERT_OK macros are provided to help propagate errors. C++ -/// clients can use the helpers provided in the nanoarrow.hpp header to -/// facilitate using C++ idioms for memory management and error propgagtion. +/// function has returned a non-zero ArrowErrorCode. The NANOARROW_RETURN_NOT_OK and +/// NANOARROW_ASSERT_OK macros are provided to help propagate errors. C++ clients can use +/// the helpers provided in the nanoarrow.hpp header to facilitate using C++ idioms +/// for memory management and error propgagtion. /// /// @{ /// \brief Set the contents of an error using printf syntax. /// /// If error is NULL, this function does nothing and returns NANOARROW_OK. -NANOARROW_CHECK_PRINTF_ATTRIBUTE int ArrowErrorSet( - struct ArrowError* error, const char* fmt, ...); +NANOARROW_CHECK_PRINTF_ATTRIBUTE int ArrowErrorSet(struct ArrowError* error, + const char* fmt, ...); /// @} @@ -1338,12 +1273,12 @@ void ArrowLayoutInit(struct ArrowLayout* layout, enum ArrowType storage_type); static inline struct ArrowStringView ArrowCharView(const char* value); /// \brief Sets the integer value of an ArrowDecimal from a string -ArrowErrorCode ArrowDecimalSetDigits( - struct ArrowDecimal* decimal, struct ArrowStringView value); +ArrowErrorCode ArrowDecimalSetDigits(struct ArrowDecimal* decimal, + struct ArrowStringView value); /// \brief Get the integer value of an ArrowDecimal as string -ArrowErrorCode ArrowDecimalAppendDigitsToBuffer( - const struct ArrowDecimal* decimal, struct ArrowBuffer* buffer); +ArrowErrorCode ArrowDecimalAppendDigitsToBuffer(const struct ArrowDecimal* decimal, + struct ArrowBuffer* buffer); /// @} @@ -1364,10 +1299,9 @@ void ArrowSchemaInit(struct ArrowSchema* schema); /// /// A convenience constructor for that calls ArrowSchemaInit() and /// ArrowSchemaSetType() for the common case of constructing an -/// unparameterized type. The caller is responsible for calling the -/// schema->release callback if NANOARROW_OK is returned. -ArrowErrorCode ArrowSchemaInitFromType( - struct ArrowSchema* schema, enum ArrowType type); +/// unparameterized type. The caller is responsible for calling the schema->release +/// callback if NANOARROW_OK is returned. +ArrowErrorCode ArrowSchemaInitFromType(struct ArrowSchema* schema, enum ArrowType type); /// \brief Get a human-readable summary of a Schema /// @@ -1375,8 +1309,8 @@ ArrowErrorCode ArrowSchemaInitFromType( /// and returns the number of characters required for the output if /// n were sufficiently large. If recursive is non-zero, the result will /// also include children. -int64_t ArrowSchemaToString( - const struct ArrowSchema* schema, char* out, int64_t n, char recursive); +int64_t ArrowSchemaToString(const struct ArrowSchema* schema, char* out, int64_t n, + char recursive); /// \brief Set the format field of a schema from an ArrowType /// @@ -1384,19 +1318,16 @@ int64_t ArrowSchemaToString( /// NANOARROW_TYPE_LIST, NANOARROW_TYPE_LARGE_LIST, and /// NANOARROW_TYPE_MAP, the appropriate number of children are /// allocated, initialized, and named; however, the caller must -/// ArrowSchemaSetType() on the preinitialized children. Schema must have been -/// initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy(). -ArrowErrorCode ArrowSchemaSetType( - struct ArrowSchema* schema, enum ArrowType type); +/// ArrowSchemaSetType() on the preinitialized children. Schema must have been initialized +/// using ArrowSchemaInit() or ArrowSchemaDeepCopy(). +ArrowErrorCode ArrowSchemaSetType(struct ArrowSchema* schema, enum ArrowType type); /// \brief Set the format field and initialize children of a struct schema /// -/// The specified number of children are initialized; however, the caller is -/// responsible for calling ArrowSchemaSetType() and ArrowSchemaSetName() on -/// each child. Schema must have been initialized using ArrowSchemaInit() or -/// ArrowSchemaDeepCopy(). -ArrowErrorCode ArrowSchemaSetTypeStruct( - struct ArrowSchema* schema, int64_t n_children); +/// The specified number of children are initialized; however, the caller is responsible +/// for calling ArrowSchemaSetType() and ArrowSchemaSetName() on each child. +/// Schema must have been initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy(). +ArrowErrorCode ArrowSchemaSetTypeStruct(struct ArrowSchema* schema, int64_t n_children); /// \brief Set the format field of a fixed-size schema /// @@ -1404,55 +1335,50 @@ ArrowErrorCode ArrowSchemaSetTypeStruct( /// NANOARROW_TYPE_FIXED_SIZE_BINARY or NANOARROW_TYPE_FIXED_SIZE_LIST. /// For NANOARROW_TYPE_FIXED_SIZE_LIST, the appropriate number of children are /// allocated, initialized, and named; however, the caller must -/// ArrowSchemaSetType() the first child. Schema must have been initialized -/// using ArrowSchemaInit() or ArrowSchemaDeepCopy(). -ArrowErrorCode ArrowSchemaSetTypeFixedSize( - struct ArrowSchema* schema, enum ArrowType type, int32_t fixed_size); +/// ArrowSchemaSetType() the first child. Schema must have been initialized using +/// ArrowSchemaInit() or ArrowSchemaDeepCopy(). +ArrowErrorCode ArrowSchemaSetTypeFixedSize(struct ArrowSchema* schema, + enum ArrowType type, int32_t fixed_size); /// \brief Set the format field of a decimal schema /// /// Returns EINVAL for scale <= 0 or for type that is not -/// NANOARROW_TYPE_DECIMAL128 or NANOARROW_TYPE_DECIMAL256. Schema must have -/// been initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy(). -ArrowErrorCode ArrowSchemaSetTypeDecimal( - struct ArrowSchema* schema, - enum ArrowType type, - int32_t decimal_precision, - int32_t decimal_scale); +/// NANOARROW_TYPE_DECIMAL128 or NANOARROW_TYPE_DECIMAL256. Schema must have been +/// initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy(). +ArrowErrorCode ArrowSchemaSetTypeDecimal(struct ArrowSchema* schema, enum ArrowType type, + int32_t decimal_precision, + int32_t decimal_scale); /// \brief Set the format field of a time, timestamp, or duration schema /// /// Returns EINVAL for type that is not /// NANOARROW_TYPE_TIME32, NANOARROW_TYPE_TIME64, /// NANOARROW_TYPE_TIMESTAMP, or NANOARROW_TYPE_DURATION. The -/// timezone parameter must be NULL for a non-timestamp type. Schema must have -/// been initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy(). -ArrowErrorCode ArrowSchemaSetTypeDateTime( - struct ArrowSchema* schema, - enum ArrowType type, - enum ArrowTimeUnit time_unit, - const char* timezone); +/// timezone parameter must be NULL for a non-timestamp type. Schema must have been +/// initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy(). +ArrowErrorCode ArrowSchemaSetTypeDateTime(struct ArrowSchema* schema, enum ArrowType type, + enum ArrowTimeUnit time_unit, + const char* timezone); /// \brief Seet the format field of a union schema /// /// Returns EINVAL for a type that is not NANOARROW_TYPE_DENSE_UNION /// or NANOARROW_TYPE_SPARSE_UNION. The specified number of children are /// allocated, and initialized. -ArrowErrorCode ArrowSchemaSetTypeUnion( - struct ArrowSchema* schema, enum ArrowType type, int64_t n_children); +ArrowErrorCode ArrowSchemaSetTypeUnion(struct ArrowSchema* schema, enum ArrowType type, + int64_t n_children); /// \brief Make a (recursive) copy of a schema /// /// Allocates and copies fields of schema into schema_out. -ArrowErrorCode ArrowSchemaDeepCopy( - const struct ArrowSchema* schema, struct ArrowSchema* schema_out); +ArrowErrorCode ArrowSchemaDeepCopy(const struct ArrowSchema* schema, + struct ArrowSchema* schema_out); /// \brief Copy format into schema->format /// /// schema must have been allocated using ArrowSchemaInitFromType() or /// ArrowSchemaDeepCopy(). -ArrowErrorCode ArrowSchemaSetFormat( - struct ArrowSchema* schema, const char* format); +ArrowErrorCode ArrowSchemaSetFormat(struct ArrowSchema* schema, const char* format); /// \brief Copy name into schema->name /// @@ -1464,16 +1390,15 @@ ArrowErrorCode ArrowSchemaSetName(struct ArrowSchema* schema, const char* name); /// /// schema must have been allocated using ArrowSchemaInitFromType() or /// ArrowSchemaDeepCopy. -ArrowErrorCode ArrowSchemaSetMetadata( - struct ArrowSchema* schema, const char* metadata); +ArrowErrorCode ArrowSchemaSetMetadata(struct ArrowSchema* schema, const char* metadata); /// \brief Allocate the schema->children array /// /// Includes the memory for each child struct ArrowSchema. /// schema must have been allocated using ArrowSchemaInitFromType() or /// ArrowSchemaDeepCopy(). -ArrowErrorCode ArrowSchemaAllocateChildren( - struct ArrowSchema* schema, int64_t n_children); +ArrowErrorCode ArrowSchemaAllocateChildren(struct ArrowSchema* schema, + int64_t n_children); /// \brief Allocate the schema->dictionary member /// @@ -1503,14 +1428,13 @@ struct ArrowMetadataReader { }; /// \brief Initialize an ArrowMetadataReader -ArrowErrorCode ArrowMetadataReaderInit( - struct ArrowMetadataReader* reader, const char* metadata); +ArrowErrorCode ArrowMetadataReaderInit(struct ArrowMetadataReader* reader, + const char* metadata); /// \brief Read the next key/value pair from an ArrowMetadataReader -ArrowErrorCode ArrowMetadataReaderRead( - struct ArrowMetadataReader* reader, - struct ArrowStringView* key_out, - struct ArrowStringView* value_out); +ArrowErrorCode ArrowMetadataReaderRead(struct ArrowMetadataReader* reader, + struct ArrowStringView* key_out, + struct ArrowStringView* value_out); /// \brief The number of bytes in in a key/value metadata string int64_t ArrowMetadataSizeOf(const char* metadata); @@ -1521,37 +1445,32 @@ char ArrowMetadataHasKey(const char* metadata, struct ArrowStringView key); /// \brief Extract a value from schema metadata /// /// If key does not exist in metadata, value_out is unmodified -ArrowErrorCode ArrowMetadataGetValue( - const char* metadata, - struct ArrowStringView key, - struct ArrowStringView* value_out); +ArrowErrorCode ArrowMetadataGetValue(const char* metadata, struct ArrowStringView key, + struct ArrowStringView* value_out); /// \brief Initialize a builder for schema metadata from key/value pairs /// /// metadata can be an existing metadata string or NULL to initialize /// an empty metadata string. -ArrowErrorCode ArrowMetadataBuilderInit( - struct ArrowBuffer* buffer, const char* metadata); +ArrowErrorCode ArrowMetadataBuilderInit(struct ArrowBuffer* buffer, const char* metadata); /// \brief Append a key/value pair to a buffer containing serialized metadata -ArrowErrorCode ArrowMetadataBuilderAppend( - struct ArrowBuffer* buffer, - struct ArrowStringView key, - struct ArrowStringView value); +ArrowErrorCode ArrowMetadataBuilderAppend(struct ArrowBuffer* buffer, + struct ArrowStringView key, + struct ArrowStringView value); /// \brief Set a key/value pair to a buffer containing serialized metadata /// /// Ensures that the only entry for key in the metadata is set to value. /// This function maintains the existing position of (the first instance of) /// key if present in the data. -ArrowErrorCode ArrowMetadataBuilderSet( - struct ArrowBuffer* buffer, - struct ArrowStringView key, - struct ArrowStringView value); +ArrowErrorCode ArrowMetadataBuilderSet(struct ArrowBuffer* buffer, + struct ArrowStringView key, + struct ArrowStringView value); /// \brief Remove a key from a buffer containing serialized metadata -ArrowErrorCode ArrowMetadataBuilderRemove( - struct ArrowBuffer* buffer, struct ArrowStringView key); +ArrowErrorCode ArrowMetadataBuilderRemove(struct ArrowBuffer* buffer, + struct ArrowStringView key); /// @} @@ -1580,9 +1499,9 @@ struct ArrowSchemaView { /// \brief The storage data type represented by the schema /// - /// This value will never be NANOARROW_TYPE_DICTIONARY, - /// NANOARROW_TYPE_EXTENSION or any datetime type. This value represents only - /// the type required to interpret the buffers in the array. + /// This value will never be NANOARROW_TYPE_DICTIONARY, NANOARROW_TYPE_EXTENSION + /// or any datetime type. This value represents only the type required to + /// interpret the buffers in the array. enum ArrowType storage_type; /// \brief The storage layout represented by the schema @@ -1649,10 +1568,9 @@ struct ArrowSchemaView { }; /// \brief Initialize an ArrowSchemaView -ArrowErrorCode ArrowSchemaViewInit( - struct ArrowSchemaView* schema_view, - const struct ArrowSchema* schema, - struct ArrowError* error); +ArrowErrorCode ArrowSchemaViewInit(struct ArrowSchemaView* schema_view, + const struct ArrowSchema* schema, + struct ArrowError* error); /// @} @@ -1683,8 +1601,7 @@ static inline void ArrowBufferReset(struct ArrowBuffer* buffer); /// /// Transfers the buffer data and lifecycle management to another /// address and resets buffer. -static inline void ArrowBufferMove( - struct ArrowBuffer* src, struct ArrowBuffer* dst); +static inline void ArrowBufferMove(struct ArrowBuffer* src, struct ArrowBuffer* dst); /// \brief Grow or shrink a buffer to a given capacity /// @@ -1692,84 +1609,85 @@ static inline void ArrowBufferMove( /// if shrink_to_fit is non-zero. Calling ArrowBufferResize() does not /// adjust the buffer's size member except to ensure that the invariant /// capacity >= size remains true. -static inline ArrowErrorCode ArrowBufferResize( - struct ArrowBuffer* buffer, int64_t new_capacity_bytes, char shrink_to_fit); +static inline ArrowErrorCode ArrowBufferResize(struct ArrowBuffer* buffer, + int64_t new_capacity_bytes, + char shrink_to_fit); /// \brief Ensure a buffer has at least a given additional capacity /// /// Ensures that the buffer has space to append at least /// additional_size_bytes, overallocating when required. -static inline ArrowErrorCode ArrowBufferReserve( - struct ArrowBuffer* buffer, int64_t additional_size_bytes); +static inline ArrowErrorCode ArrowBufferReserve(struct ArrowBuffer* buffer, + int64_t additional_size_bytes); /// \brief Write data to buffer and increment the buffer size /// /// This function does not check that buffer has the required capacity -static inline void ArrowBufferAppendUnsafe( - struct ArrowBuffer* buffer, const void* data, int64_t size_bytes); +static inline void ArrowBufferAppendUnsafe(struct ArrowBuffer* buffer, const void* data, + int64_t size_bytes); /// \brief Write data to buffer and increment the buffer size /// /// This function writes and ensures that the buffer has the required capacity, /// possibly by reallocating the buffer. Like ArrowBufferReserve, this will /// overallocate when reallocation is required. -static inline ArrowErrorCode ArrowBufferAppend( - struct ArrowBuffer* buffer, const void* data, int64_t size_bytes); +static inline ArrowErrorCode ArrowBufferAppend(struct ArrowBuffer* buffer, + const void* data, int64_t size_bytes); /// \brief Write fill to buffer and increment the buffer size /// /// This function writes the specified number of fill bytes and /// ensures that the buffer has the required capacity, -static inline ArrowErrorCode ArrowBufferAppendFill( - struct ArrowBuffer* buffer, uint8_t value, int64_t size_bytes); +static inline ArrowErrorCode ArrowBufferAppendFill(struct ArrowBuffer* buffer, + uint8_t value, int64_t size_bytes); /// \brief Write an 8-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendInt8( - struct ArrowBuffer* buffer, int8_t value); +static inline ArrowErrorCode ArrowBufferAppendInt8(struct ArrowBuffer* buffer, + int8_t value); /// \brief Write an unsigned 8-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendUInt8( - struct ArrowBuffer* buffer, uint8_t value); +static inline ArrowErrorCode ArrowBufferAppendUInt8(struct ArrowBuffer* buffer, + uint8_t value); /// \brief Write a 16-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendInt16( - struct ArrowBuffer* buffer, int16_t value); +static inline ArrowErrorCode ArrowBufferAppendInt16(struct ArrowBuffer* buffer, + int16_t value); /// \brief Write an unsigned 16-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendUInt16( - struct ArrowBuffer* buffer, uint16_t value); +static inline ArrowErrorCode ArrowBufferAppendUInt16(struct ArrowBuffer* buffer, + uint16_t value); /// \brief Write a 32-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendInt32( - struct ArrowBuffer* buffer, int32_t value); +static inline ArrowErrorCode ArrowBufferAppendInt32(struct ArrowBuffer* buffer, + int32_t value); /// \brief Write an unsigned 32-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendUInt32( - struct ArrowBuffer* buffer, uint32_t value); +static inline ArrowErrorCode ArrowBufferAppendUInt32(struct ArrowBuffer* buffer, + uint32_t value); /// \brief Write a 64-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendInt64( - struct ArrowBuffer* buffer, int64_t value); +static inline ArrowErrorCode ArrowBufferAppendInt64(struct ArrowBuffer* buffer, + int64_t value); /// \brief Write an unsigned 64-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendUInt64( - struct ArrowBuffer* buffer, uint64_t value); +static inline ArrowErrorCode ArrowBufferAppendUInt64(struct ArrowBuffer* buffer, + uint64_t value); /// \brief Write a double to a buffer -static inline ArrowErrorCode ArrowBufferAppendDouble( - struct ArrowBuffer* buffer, double value); +static inline ArrowErrorCode ArrowBufferAppendDouble(struct ArrowBuffer* buffer, + double value); /// \brief Write a float to a buffer -static inline ArrowErrorCode ArrowBufferAppendFloat( - struct ArrowBuffer* buffer, float value); +static inline ArrowErrorCode ArrowBufferAppendFloat(struct ArrowBuffer* buffer, + float value); /// \brief Write an ArrowStringView to a buffer -static inline ArrowErrorCode ArrowBufferAppendStringView( - struct ArrowBuffer* buffer, struct ArrowStringView value); +static inline ArrowErrorCode ArrowBufferAppendStringView(struct ArrowBuffer* buffer, + struct ArrowStringView value); /// \brief Write an ArrowBufferView to a buffer -static inline ArrowErrorCode ArrowBufferAppendBufferView( - struct ArrowBuffer* buffer, struct ArrowBufferView value); +static inline ArrowErrorCode ArrowBufferAppendBufferView(struct ArrowBuffer* buffer, + struct ArrowBufferView value); /// @} @@ -1790,20 +1708,19 @@ static inline void ArrowBitClear(uint8_t* bits, int64_t i); static inline void ArrowBitSetTo(uint8_t* bits, int64_t i, uint8_t value); /// \brief Set a boolean value to a range in a bitmap -static inline void ArrowBitsSetTo( - uint8_t* bits, int64_t start_offset, int64_t length, uint8_t bits_are_set); +static inline void ArrowBitsSetTo(uint8_t* bits, int64_t start_offset, int64_t length, + uint8_t bits_are_set); /// \brief Count true values in a bitmap -static inline int64_t ArrowBitCountSet( - const uint8_t* bits, int64_t i_from, int64_t i_to); +static inline int64_t ArrowBitCountSet(const uint8_t* bits, int64_t i_from, int64_t i_to); /// \brief Extract int8 boolean values from a range in a bitmap -static inline void ArrowBitsUnpackInt8( - const uint8_t* bits, int64_t start_offset, int64_t length, int8_t* out); +static inline void ArrowBitsUnpackInt8(const uint8_t* bits, int64_t start_offset, + int64_t length, int8_t* out); /// \brief Extract int32 boolean values from a range in a bitmap -static inline void ArrowBitsUnpackInt32( - const uint8_t* bits, int64_t start_offset, int64_t length, int32_t* out); +static inline void ArrowBitsUnpackInt32(const uint8_t* bits, int64_t start_offset, + int64_t length, int32_t* out); /// \brief Initialize an ArrowBitmap /// @@ -1814,15 +1731,14 @@ static inline void ArrowBitmapInit(struct ArrowBitmap* bitmap); /// /// Transfers the underlying buffer data and lifecycle management to another /// address and resets the bitmap. -static inline void ArrowBitmapMove( - struct ArrowBitmap* src, struct ArrowBitmap* dst); +static inline void ArrowBitmapMove(struct ArrowBitmap* src, struct ArrowBitmap* dst); /// \brief Ensure a bitmap builder has at least a given additional capacity /// /// Ensures that the buffer has space to append at least /// additional_size_bits, overallocating when required. -static inline ArrowErrorCode ArrowBitmapReserve( - struct ArrowBitmap* bitmap, int64_t additional_size_bits); +static inline ArrowErrorCode ArrowBitmapReserve(struct ArrowBitmap* bitmap, + int64_t additional_size_bits); /// \brief Grow or shrink a bitmap to a given capacity /// @@ -1830,34 +1746,33 @@ static inline ArrowErrorCode ArrowBitmapReserve( /// if shrink_to_fit is non-zero. Calling ArrowBitmapResize() does not /// adjust the buffer's size member except when shrinking new_capacity_bits /// to a value less than the current number of bits in the bitmap. -static inline ArrowErrorCode ArrowBitmapResize( - struct ArrowBitmap* bitmap, int64_t new_capacity_bits, char shrink_to_fit); +static inline ArrowErrorCode ArrowBitmapResize(struct ArrowBitmap* bitmap, + int64_t new_capacity_bits, + char shrink_to_fit); -/// \brief Reserve space for and append zero or more of the same boolean value -/// to a bitmap -static inline ArrowErrorCode ArrowBitmapAppend( - struct ArrowBitmap* bitmap, uint8_t bits_are_set, int64_t length); +/// \brief Reserve space for and append zero or more of the same boolean value to a bitmap +static inline ArrowErrorCode ArrowBitmapAppend(struct ArrowBitmap* bitmap, + uint8_t bits_are_set, int64_t length); /// \brief Append zero or more of the same boolean value to a bitmap -static inline void ArrowBitmapAppendUnsafe( - struct ArrowBitmap* bitmap, uint8_t bits_are_set, int64_t length); +static inline void ArrowBitmapAppendUnsafe(struct ArrowBitmap* bitmap, + uint8_t bits_are_set, int64_t length); /// \brief Append boolean values encoded as int8_t to a bitmap /// /// The values must all be 0 or 1. -static inline void ArrowBitmapAppendInt8Unsafe( - struct ArrowBitmap* bitmap, const int8_t* values, int64_t n_values); +static inline void ArrowBitmapAppendInt8Unsafe(struct ArrowBitmap* bitmap, + const int8_t* values, int64_t n_values); /// \brief Append boolean values encoded as int32_t to a bitmap /// /// The values must all be 0 or 1. -static inline void ArrowBitmapAppendInt32Unsafe( - struct ArrowBitmap* bitmap, const int32_t* values, int64_t n_values); +static inline void ArrowBitmapAppendInt32Unsafe(struct ArrowBitmap* bitmap, + const int32_t* values, int64_t n_values); /// \brief Reset a bitmap builder /// -/// Releases any memory held by buffer, empties the cache, and resets the size -/// to zero +/// Releases any memory held by buffer, empties the cache, and resets the size to zero static inline void ArrowBitmapReset(struct ArrowBitmap* bitmap); /// @} @@ -1876,26 +1791,24 @@ static inline void ArrowBitmapReset(struct ArrowBitmap* bitmap); /// Initializes the fields and release callback of array. Caller /// is responsible for calling the array->release callback if /// NANOARROW_OK is returned. -ArrowErrorCode ArrowArrayInitFromType( - struct ArrowArray* array, enum ArrowType storage_type); +ArrowErrorCode ArrowArrayInitFromType(struct ArrowArray* array, + enum ArrowType storage_type); /// \brief Initialize the contents of an ArrowArray from an ArrowSchema /// /// Caller is responsible for calling the array->release callback if /// NANOARROW_OK is returned. -ArrowErrorCode ArrowArrayInitFromSchema( - struct ArrowArray* array, - const struct ArrowSchema* schema, - struct ArrowError* error); +ArrowErrorCode ArrowArrayInitFromSchema(struct ArrowArray* array, + const struct ArrowSchema* schema, + struct ArrowError* error); /// \brief Initialize the contents of an ArrowArray from an ArrowArrayView /// /// Caller is responsible for calling the array->release callback if /// NANOARROW_OK is returned. -ArrowErrorCode ArrowArrayInitFromArrayView( - struct ArrowArray* array, - const struct ArrowArrayView* array_view, - struct ArrowError* error); +ArrowErrorCode ArrowArrayInitFromArrayView(struct ArrowArray* array, + const struct ArrowArrayView* array_view, + struct ArrowError* error); /// \brief Allocate the array->children array /// @@ -1903,8 +1816,7 @@ ArrowErrorCode ArrowArrayInitFromArrayView( /// whose members are marked as released and may be subsequently initialized /// with ArrowArrayInitFromType() or moved from an existing ArrowArray. /// schema must have been allocated using ArrowArrayInitFromType(). -ArrowErrorCode ArrowArrayAllocateChildren( - struct ArrowArray* array, int64_t n_children); +ArrowErrorCode ArrowArrayAllocateChildren(struct ArrowArray* array, int64_t n_children); /// \brief Allocate the array->dictionary member /// @@ -1917,33 +1829,30 @@ ArrowErrorCode ArrowArrayAllocateDictionary(struct ArrowArray* array); /// \brief Set the validity bitmap of an ArrowArray /// /// array must have been allocated using ArrowArrayInitFromType() -void ArrowArraySetValidityBitmap( - struct ArrowArray* array, struct ArrowBitmap* bitmap); +void ArrowArraySetValidityBitmap(struct ArrowArray* array, struct ArrowBitmap* bitmap); /// \brief Set a buffer of an ArrowArray /// /// array must have been allocated using ArrowArrayInitFromType() -ArrowErrorCode ArrowArraySetBuffer( - struct ArrowArray* array, int64_t i, struct ArrowBuffer* buffer); +ArrowErrorCode ArrowArraySetBuffer(struct ArrowArray* array, int64_t i, + struct ArrowBuffer* buffer); /// \brief Get the validity bitmap of an ArrowArray /// /// array must have been allocated using ArrowArrayInitFromType() -static inline struct ArrowBitmap* ArrowArrayValidityBitmap( - struct ArrowArray* array); +static inline struct ArrowBitmap* ArrowArrayValidityBitmap(struct ArrowArray* array); /// \brief Get a buffer of an ArrowArray /// /// array must have been allocated using ArrowArrayInitFromType() -static inline struct ArrowBuffer* ArrowArrayBuffer( - struct ArrowArray* array, int64_t i); +static inline struct ArrowBuffer* ArrowArrayBuffer(struct ArrowArray* array, int64_t i); /// \brief Start element-wise appending to an ArrowArray /// /// Initializes any values needed to use ArrowArrayAppend*() functions. -/// All element-wise appenders append by value and return EINVAL if the exact -/// value cannot be represented by the underlying storage type. array must have -/// been allocated using ArrowArrayInitFromType() +/// All element-wise appenders append by value and return EINVAL if the exact value +/// cannot be represented by the underlying storage type. +/// array must have been allocated using ArrowArrayInitFromType() static inline ArrowErrorCode ArrowArrayStartAppending(struct ArrowArray* array); /// \brief Reserve space for future appends @@ -1952,32 +1861,29 @@ static inline ArrowErrorCode ArrowArrayStartAppending(struct ArrowArray* array); /// child array sizes for non-fixed-size arrays), recursively reserve space for /// additional elements. This is useful for reducing the number of reallocations /// that occur using the item-wise appenders. -ArrowErrorCode ArrowArrayReserve( - struct ArrowArray* array, int64_t additional_size_elements); +ArrowErrorCode ArrowArrayReserve(struct ArrowArray* array, + int64_t additional_size_elements); /// \brief Append a null value to an array -static inline ArrowErrorCode ArrowArrayAppendNull( - struct ArrowArray* array, int64_t n); +static inline ArrowErrorCode ArrowArrayAppendNull(struct ArrowArray* array, int64_t n); /// \brief Append an empty, non-null value to an array -static inline ArrowErrorCode ArrowArrayAppendEmpty( - struct ArrowArray* array, int64_t n); +static inline ArrowErrorCode ArrowArrayAppendEmpty(struct ArrowArray* array, int64_t n); /// \brief Append a signed integer value to an array /// /// Returns NANOARROW_OK if value can be exactly represented by /// the underlying storage type or EINVAL otherwise (e.g., value /// is outside the valid array range). -static inline ArrowErrorCode ArrowArrayAppendInt( - struct ArrowArray* array, int64_t value); +static inline ArrowErrorCode ArrowArrayAppendInt(struct ArrowArray* array, int64_t value); /// \brief Append an unsigned integer value to an array /// /// Returns NANOARROW_OK if value can be exactly represented by /// the underlying storage type or EINVAL otherwise (e.g., value /// is outside the valid array range). -static inline ArrowErrorCode ArrowArrayAppendUInt( - struct ArrowArray* array, uint64_t value); +static inline ArrowErrorCode ArrowArrayAppendUInt(struct ArrowArray* array, + uint64_t value); /// \brief Append a double value to an array /// @@ -1985,68 +1891,67 @@ static inline ArrowErrorCode ArrowArrayAppendUInt( /// the underlying storage type or EINVAL otherwise (e.g., value /// is outside the valid array range or there is an attempt to append /// a non-integer to an array with an integer storage type). -static inline ArrowErrorCode ArrowArrayAppendDouble( - struct ArrowArray* array, double value); +static inline ArrowErrorCode ArrowArrayAppendDouble(struct ArrowArray* array, + double value); /// \brief Append a string of bytes to an array /// /// Returns NANOARROW_OK if value can be exactly represented by /// the underlying storage type, EOVERFLOW if appending value would overflow /// the offset type (e.g., if the data buffer would be larger than 2 GB for a -/// non-large string type), or EINVAL otherwise (e.g., the underlying array is -/// not a binary, string, large binary, large string, or fixed-size binary -/// array, or value is the wrong size for a fixed-size binary array). -static inline ArrowErrorCode ArrowArrayAppendBytes( - struct ArrowArray* array, struct ArrowBufferView value); +/// non-large string type), or EINVAL otherwise (e.g., the underlying array is not a +/// binary, string, large binary, large string, or fixed-size binary array, or value is +/// the wrong size for a fixed-size binary array). +static inline ArrowErrorCode ArrowArrayAppendBytes(struct ArrowArray* array, + struct ArrowBufferView value); /// \brief Append a string value to an array /// /// Returns NANOARROW_OK if value can be exactly represented by /// the underlying storage type, EOVERFLOW if appending value would overflow /// the offset type (e.g., if the data buffer would be larger than 2 GB for a -/// non-large string type), or EINVAL otherwise (e.g., the underlying array is -/// not a string or large string array). -static inline ArrowErrorCode ArrowArrayAppendString( - struct ArrowArray* array, struct ArrowStringView value); +/// non-large string type), or EINVAL otherwise (e.g., the underlying array is not a +/// string or large string array). +static inline ArrowErrorCode ArrowArrayAppendString(struct ArrowArray* array, + struct ArrowStringView value); /// \brief Append a Interval to an array /// /// Returns NANOARROW_OK if value can be exactly represented by /// the underlying storage type or EINVAL otherwise. -static inline ArrowErrorCode ArrowArrayAppendInterval( - struct ArrowArray* array, const struct ArrowInterval* value); +static inline ArrowErrorCode ArrowArrayAppendInterval(struct ArrowArray* array, + const struct ArrowInterval* value); /// \brief Append a decimal value to an array /// /// Returns NANOARROW_OK if array is a decimal array with the appropriate /// bitwidth or EINVAL otherwise. -static inline ArrowErrorCode ArrowArrayAppendDecimal( - struct ArrowArray* array, const struct ArrowDecimal* value); +static inline ArrowErrorCode ArrowArrayAppendDecimal(struct ArrowArray* array, + const struct ArrowDecimal* value); /// \brief Finish a nested array element /// /// Appends a non-null element to the array based on the first child's current /// length. Returns NANOARROW_OK if the item was successfully added, EOVERFLOW /// if the child of a list or map array would exceed INT_MAX elements, or EINVAL -/// if the underlying storage type is not a struct, list, large list, or -/// fixed-size list, or if there was an attempt to add a struct or fixed-size -/// list element where the length of the child array(s) did not match the -/// expected length. +/// if the underlying storage type is not a struct, list, large list, or fixed-size +/// list, or if there was an attempt to add a struct or fixed-size list element where the +/// length of the child array(s) did not match the expected length. static inline ArrowErrorCode ArrowArrayFinishElement(struct ArrowArray* array); /// \brief Finish a union array element /// -/// Appends an element to the union type ids buffer and increments -/// array->length. For sparse unions, up to one element is added to non type-id -/// children. Returns EINVAL if the underlying storage type is not a union, if -/// type_id is not valid, or if child sizes after appending are inconsistent. -static inline ArrowErrorCode ArrowArrayFinishUnionElement( - struct ArrowArray* array, int8_t type_id); +/// Appends an element to the union type ids buffer and increments array->length. +/// For sparse unions, up to one element is added to non type-id children. Returns +/// EINVAL if the underlying storage type is not a union, if type_id is not valid, +/// or if child sizes after appending are inconsistent. +static inline ArrowErrorCode ArrowArrayFinishUnionElement(struct ArrowArray* array, + int8_t type_id); /// \brief Shrink buffer capacity to the size required /// -/// Also applies shrinking to any child arrays. array must have been allocated -/// using ArrowArrayInitFromType +/// Also applies shrinking to any child arrays. array must have been allocated using +/// ArrowArrayInitFromType static inline ArrowErrorCode ArrowArrayShrinkToFit(struct ArrowArray* array); /// \brief Finish building an ArrowArray @@ -2055,20 +1960,19 @@ static inline ArrowErrorCode ArrowArrayShrinkToFit(struct ArrowArray* array); /// into array->buffers and checks the actual size of the buffers /// against the expected size based on the final length. /// array must have been allocated using ArrowArrayInitFromType() -ArrowErrorCode ArrowArrayFinishBuildingDefault( - struct ArrowArray* array, struct ArrowError* error); +ArrowErrorCode ArrowArrayFinishBuildingDefault(struct ArrowArray* array, + struct ArrowError* error); /// \brief Finish building an ArrowArray with explicit validation /// -/// Finish building with an explicit validation level. This could perform less -/// validation (i.e. NANOARROW_VALIDATION_LEVEL_NONE or -/// NANOARROW_VALIDATION_LEVEL_MINIMAL) if CPU buffer data access is not -/// possible or more validation (i.e., NANOARROW_VALIDATION_LEVEL_FULL) if -/// buffer content was obtained from an untrusted or corruptible source. -ArrowErrorCode ArrowArrayFinishBuilding( - struct ArrowArray* array, - enum ArrowValidationLevel validation_level, - struct ArrowError* error); +/// Finish building with an explicit validation level. This could perform less validation +/// (i.e. NANOARROW_VALIDATION_LEVEL_NONE or NANOARROW_VALIDATION_LEVEL_MINIMAL) if CPU +/// buffer data access is not possible or more validation (i.e., +/// NANOARROW_VALIDATION_LEVEL_FULL) if buffer content was obtained from an untrusted or +/// corruptible source. +ArrowErrorCode ArrowArrayFinishBuilding(struct ArrowArray* array, + enum ArrowValidationLevel validation_level, + struct ArrowError* error); /// @} @@ -2079,71 +1983,66 @@ ArrowErrorCode ArrowArrayFinishBuilding( /// @{ /// \brief Initialize the contents of an ArrowArrayView -void ArrowArrayViewInitFromType( - struct ArrowArrayView* array_view, enum ArrowType storage_type); +void ArrowArrayViewInitFromType(struct ArrowArrayView* array_view, + enum ArrowType storage_type); /// \brief Move an ArrowArrayView /// /// Transfers the ArrowArrayView data and lifecycle management to another /// address and resets the contents of src. -static inline void ArrowArrayViewMove( - struct ArrowArrayView* src, struct ArrowArrayView* dst); +static inline void ArrowArrayViewMove(struct ArrowArrayView* src, + struct ArrowArrayView* dst); /// \brief Initialize the contents of an ArrowArrayView from an ArrowSchema -ArrowErrorCode ArrowArrayViewInitFromSchema( - struct ArrowArrayView* array_view, - const struct ArrowSchema* schema, - struct ArrowError* error); +ArrowErrorCode ArrowArrayViewInitFromSchema(struct ArrowArrayView* array_view, + const struct ArrowSchema* schema, + struct ArrowError* error); /// \brief Allocate the array_view->children array /// /// Includes the memory for each child struct ArrowArrayView -ArrowErrorCode ArrowArrayViewAllocateChildren( - struct ArrowArrayView* array_view, int64_t n_children); +ArrowErrorCode ArrowArrayViewAllocateChildren(struct ArrowArrayView* array_view, + int64_t n_children); /// \brief Allocate array_view->dictionary -ArrowErrorCode ArrowArrayViewAllocateDictionary( - struct ArrowArrayView* array_view); +ArrowErrorCode ArrowArrayViewAllocateDictionary(struct ArrowArrayView* array_view); /// \brief Set data-independent buffer sizes from length void ArrowArrayViewSetLength(struct ArrowArrayView* array_view, int64_t length); /// \brief Set buffer sizes and data pointers from an ArrowArray -ArrowErrorCode ArrowArrayViewSetArray( - struct ArrowArrayView* array_view, - const struct ArrowArray* array, - struct ArrowError* error); +ArrowErrorCode ArrowArrayViewSetArray(struct ArrowArrayView* array_view, + const struct ArrowArray* array, + struct ArrowError* error); -/// \brief Set buffer sizes and data pointers from an ArrowArray except for -/// those that require dereferencing buffer content. -ArrowErrorCode ArrowArrayViewSetArrayMinimal( - struct ArrowArrayView* array_view, - const struct ArrowArray* array, - struct ArrowError* error); +/// \brief Set buffer sizes and data pointers from an ArrowArray except for those +/// that require dereferencing buffer content. +ArrowErrorCode ArrowArrayViewSetArrayMinimal(struct ArrowArrayView* array_view, + const struct ArrowArray* array, + struct ArrowError* error); /// \brief Performs checks on the content of an ArrowArrayView /// /// If using ArrowArrayViewSetArray() to back array_view with an ArrowArray, /// the buffer sizes and some content (fist and last offset) have already /// been validated at the "default" level. If setting the buffer pointers -/// and sizes otherwise, you may wish to perform checks at a different level. -/// See documentation for ArrowValidationLevel for the details of checks -/// performed at each level. -ArrowErrorCode ArrowArrayViewValidate( - struct ArrowArrayView* array_view, - enum ArrowValidationLevel validation_level, - struct ArrowError* error); +/// and sizes otherwise, you may wish to perform checks at a different level. See +/// documentation for ArrowValidationLevel for the details of checks performed +/// at each level. +ArrowErrorCode ArrowArrayViewValidate(struct ArrowArrayView* array_view, + enum ArrowValidationLevel validation_level, + struct ArrowError* error); /// \brief Reset the contents of an ArrowArrayView and frees resources void ArrowArrayViewReset(struct ArrowArrayView* array_view); /// \brief Check for a null element in an ArrowArrayView -static inline int8_t ArrowArrayViewIsNull( - const struct ArrowArrayView* array_view, int64_t i); +static inline int8_t ArrowArrayViewIsNull(const struct ArrowArrayView* array_view, + int64_t i); /// \brief Get the type id of a union array element -static inline int8_t ArrowArrayViewUnionTypeId( - const struct ArrowArrayView* array_view, int64_t i); +static inline int8_t ArrowArrayViewUnionTypeId(const struct ArrowArrayView* array_view, + int64_t i); /// \brief Get the child index of a union array element static inline int8_t ArrowArrayViewUnionChildIndex( @@ -2155,15 +2054,15 @@ static inline int64_t ArrowArrayViewUnionChildOffset( /// \brief Get an element in an ArrowArrayView as an integer /// -/// This function does not check for null values, that values are actually -/// integers, or that values are within a valid range for an int64. -static inline int64_t ArrowArrayViewGetIntUnsafe( - const struct ArrowArrayView* array_view, int64_t i); +/// This function does not check for null values, that values are actually integers, or +/// that values are within a valid range for an int64. +static inline int64_t ArrowArrayViewGetIntUnsafe(const struct ArrowArrayView* array_view, + int64_t i); /// \brief Get an element in an ArrowArrayView as an unsigned integer /// -/// This function does not check for null values, that values are actually -/// integers, or that values are within a valid range for a uint64. +/// This function does not check for null values, that values are actually integers, or +/// that values are within a valid range for a uint64. static inline uint64_t ArrowArrayViewGetUIntUnsafe( const struct ArrowArrayView* array_view, int64_t i); @@ -2191,10 +2090,8 @@ static inline struct ArrowBufferView ArrowArrayViewGetBytesUnsafe( /// This function does not check for null values. The out parameter must /// be initialized with ArrowDecimalInit() with the proper parameters for this /// type before calling this for the first time. -static inline void ArrowArrayViewGetDecimalUnsafe( - const struct ArrowArrayView* array_view, - int64_t i, - struct ArrowDecimal* out); +static inline void ArrowArrayViewGetDecimalUnsafe(const struct ArrowArrayView* array_view, + int64_t i, struct ArrowDecimal* out); /// @} @@ -2212,10 +2109,8 @@ static inline void ArrowArrayViewGetDecimalUnsafe( /// This function moves the ownership of schema to the array_stream. If /// this function returns NANOARROW_OK, the caller is responsible for /// releasing the ArrowArrayStream. -ArrowErrorCode ArrowBasicArrayStreamInit( - struct ArrowArrayStream* array_stream, - struct ArrowSchema* schema, - int64_t n_arrays); +ArrowErrorCode ArrowBasicArrayStreamInit(struct ArrowArrayStream* array_stream, + struct ArrowSchema* schema, int64_t n_arrays); /// \brief Set the ith ArrowArray in this ArrowArrayStream. /// @@ -2224,27 +2119,29 @@ ArrowErrorCode ArrowBasicArrayStreamInit( /// be greater than zero and less than the value of n_arrays passed in /// ArrowBasicArrayStreamInit(). Callers are not required to fill all /// n_arrays members (i.e., n_arrays is a maximum bound). -void ArrowBasicArrayStreamSetArray( - struct ArrowArrayStream* array_stream, int64_t i, struct ArrowArray* array); +void ArrowBasicArrayStreamSetArray(struct ArrowArrayStream* array_stream, int64_t i, + struct ArrowArray* array); /// \brief Validate the contents of this ArrowArrayStream /// /// array_stream must have been initialized with ArrowBasicArrayStreamInit(). -/// This function uses ArrowArrayStreamInitFromSchema() and -/// ArrowArrayStreamSetArray() to validate the contents of the arrays. -ArrowErrorCode ArrowBasicArrayStreamValidate( - const struct ArrowArrayStream* array_stream, struct ArrowError* error); +/// This function uses ArrowArrayStreamInitFromSchema() and ArrowArrayStreamSetArray() +/// to validate the contents of the arrays. +ArrowErrorCode ArrowBasicArrayStreamValidate(const struct ArrowArrayStream* array_stream, + struct ArrowError* error); /// @} -// Undefine ArrowErrorCode, which may have been defined to annotate functions -// that return it to warn for an unused result. +// Undefine ArrowErrorCode, which may have been defined to annotate functions that return +// it to warn for an unused result. #if defined(ArrowErrorCode) #undef ArrowErrorCode #endif // Inline function definitions + + #ifdef __cplusplus } #endif @@ -2274,12 +2171,13 @@ ArrowErrorCode ArrowBasicArrayStreamValidate( #include #include + + #ifdef __cplusplus extern "C" { #endif -static inline int64_t _ArrowGrowByFactor( - int64_t current_capacity, int64_t new_capacity) { +static inline int64_t _ArrowGrowByFactor(int64_t current_capacity, int64_t new_capacity) { int64_t doubled_capacity = current_capacity * 2; if (doubled_capacity > new_capacity) { return doubled_capacity; @@ -2307,8 +2205,8 @@ static inline ArrowErrorCode ArrowBufferSetAllocator( static inline void ArrowBufferReset(struct ArrowBuffer* buffer) { if (buffer->data != NULL) { - buffer->allocator.free( - &buffer->allocator, (uint8_t*)buffer->data, buffer->capacity_bytes); + buffer->allocator.free(&buffer->allocator, (uint8_t*)buffer->data, + buffer->capacity_bytes); buffer->data = NULL; } @@ -2316,27 +2214,22 @@ static inline void ArrowBufferReset(struct ArrowBuffer* buffer) { buffer->size_bytes = 0; } -static inline void ArrowBufferMove( - struct ArrowBuffer* src, struct ArrowBuffer* dst) { +static inline void ArrowBufferMove(struct ArrowBuffer* src, struct ArrowBuffer* dst) { memcpy(dst, src, sizeof(struct ArrowBuffer)); src->data = NULL; ArrowBufferReset(src); } -static inline ArrowErrorCode ArrowBufferResize( - struct ArrowBuffer* buffer, - int64_t new_capacity_bytes, - char shrink_to_fit) { +static inline ArrowErrorCode ArrowBufferResize(struct ArrowBuffer* buffer, + int64_t new_capacity_bytes, + char shrink_to_fit) { if (new_capacity_bytes < 0) { return EINVAL; } if (new_capacity_bytes > buffer->capacity_bytes || shrink_to_fit) { buffer->data = buffer->allocator.reallocate( - &buffer->allocator, - buffer->data, - buffer->capacity_bytes, - new_capacity_bytes); + &buffer->allocator, buffer->data, buffer->capacity_bytes, new_capacity_bytes); if (buffer->data == NULL && new_capacity_bytes > 0) { buffer->capacity_bytes = 0; buffer->size_bytes = 0; @@ -2354,97 +2247,95 @@ static inline ArrowErrorCode ArrowBufferResize( return NANOARROW_OK; } -static inline ArrowErrorCode ArrowBufferReserve( - struct ArrowBuffer* buffer, int64_t additional_size_bytes) { +static inline ArrowErrorCode ArrowBufferReserve(struct ArrowBuffer* buffer, + int64_t additional_size_bytes) { int64_t min_capacity_bytes = buffer->size_bytes + additional_size_bytes; if (min_capacity_bytes <= buffer->capacity_bytes) { return NANOARROW_OK; } return ArrowBufferResize( - buffer, - _ArrowGrowByFactor(buffer->capacity_bytes, min_capacity_bytes), - 0); + buffer, _ArrowGrowByFactor(buffer->capacity_bytes, min_capacity_bytes), 0); } -static inline void ArrowBufferAppendUnsafe( - struct ArrowBuffer* buffer, const void* data, int64_t size_bytes) { +static inline void ArrowBufferAppendUnsafe(struct ArrowBuffer* buffer, const void* data, + int64_t size_bytes) { if (size_bytes > 0) { memcpy(buffer->data + buffer->size_bytes, data, size_bytes); buffer->size_bytes += size_bytes; } } -static inline ArrowErrorCode ArrowBufferAppend( - struct ArrowBuffer* buffer, const void* data, int64_t size_bytes) { +static inline ArrowErrorCode ArrowBufferAppend(struct ArrowBuffer* buffer, + const void* data, int64_t size_bytes) { NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, size_bytes)); ArrowBufferAppendUnsafe(buffer, data, size_bytes); return NANOARROW_OK; } -static inline ArrowErrorCode ArrowBufferAppendInt8( - struct ArrowBuffer* buffer, int8_t value) { +static inline ArrowErrorCode ArrowBufferAppendInt8(struct ArrowBuffer* buffer, + int8_t value) { return ArrowBufferAppend(buffer, &value, sizeof(int8_t)); } -static inline ArrowErrorCode ArrowBufferAppendUInt8( - struct ArrowBuffer* buffer, uint8_t value) { +static inline ArrowErrorCode ArrowBufferAppendUInt8(struct ArrowBuffer* buffer, + uint8_t value) { return ArrowBufferAppend(buffer, &value, sizeof(uint8_t)); } -static inline ArrowErrorCode ArrowBufferAppendInt16( - struct ArrowBuffer* buffer, int16_t value) { +static inline ArrowErrorCode ArrowBufferAppendInt16(struct ArrowBuffer* buffer, + int16_t value) { return ArrowBufferAppend(buffer, &value, sizeof(int16_t)); } -static inline ArrowErrorCode ArrowBufferAppendUInt16( - struct ArrowBuffer* buffer, uint16_t value) { +static inline ArrowErrorCode ArrowBufferAppendUInt16(struct ArrowBuffer* buffer, + uint16_t value) { return ArrowBufferAppend(buffer, &value, sizeof(uint16_t)); } -static inline ArrowErrorCode ArrowBufferAppendInt32( - struct ArrowBuffer* buffer, int32_t value) { +static inline ArrowErrorCode ArrowBufferAppendInt32(struct ArrowBuffer* buffer, + int32_t value) { return ArrowBufferAppend(buffer, &value, sizeof(int32_t)); } -static inline ArrowErrorCode ArrowBufferAppendUInt32( - struct ArrowBuffer* buffer, uint32_t value) { +static inline ArrowErrorCode ArrowBufferAppendUInt32(struct ArrowBuffer* buffer, + uint32_t value) { return ArrowBufferAppend(buffer, &value, sizeof(uint32_t)); } -static inline ArrowErrorCode ArrowBufferAppendInt64( - struct ArrowBuffer* buffer, int64_t value) { +static inline ArrowErrorCode ArrowBufferAppendInt64(struct ArrowBuffer* buffer, + int64_t value) { return ArrowBufferAppend(buffer, &value, sizeof(int64_t)); } -static inline ArrowErrorCode ArrowBufferAppendUInt64( - struct ArrowBuffer* buffer, uint64_t value) { +static inline ArrowErrorCode ArrowBufferAppendUInt64(struct ArrowBuffer* buffer, + uint64_t value) { return ArrowBufferAppend(buffer, &value, sizeof(uint64_t)); } -static inline ArrowErrorCode ArrowBufferAppendDouble( - struct ArrowBuffer* buffer, double value) { +static inline ArrowErrorCode ArrowBufferAppendDouble(struct ArrowBuffer* buffer, + double value) { return ArrowBufferAppend(buffer, &value, sizeof(double)); } -static inline ArrowErrorCode ArrowBufferAppendFloat( - struct ArrowBuffer* buffer, float value) { +static inline ArrowErrorCode ArrowBufferAppendFloat(struct ArrowBuffer* buffer, + float value) { return ArrowBufferAppend(buffer, &value, sizeof(float)); } -static inline ArrowErrorCode ArrowBufferAppendStringView( - struct ArrowBuffer* buffer, struct ArrowStringView value) { +static inline ArrowErrorCode ArrowBufferAppendStringView(struct ArrowBuffer* buffer, + struct ArrowStringView value) { return ArrowBufferAppend(buffer, value.data, value.size_bytes); } -static inline ArrowErrorCode ArrowBufferAppendBufferView( - struct ArrowBuffer* buffer, struct ArrowBufferView value) { +static inline ArrowErrorCode ArrowBufferAppendBufferView(struct ArrowBuffer* buffer, + struct ArrowBufferView value) { return ArrowBufferAppend(buffer, value.data.data, value.size_bytes); } -static inline ArrowErrorCode ArrowBufferAppendFill( - struct ArrowBuffer* buffer, uint8_t value, int64_t size_bytes) { +static inline ArrowErrorCode ArrowBufferAppendFill(struct ArrowBuffer* buffer, + uint8_t value, int64_t size_bytes) { NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, size_bytes)); memset(buffer->data + buffer->size_bytes, value, size_bytes); @@ -2453,24 +2344,20 @@ static inline ArrowErrorCode ArrowBufferAppendFill( } static const uint8_t _ArrowkBitmask[] = {1, 2, 4, 8, 16, 32, 64, 128}; -static const uint8_t _ArrowkFlippedBitmask[] = { - 254, 253, 251, 247, 239, 223, 191, 127}; +static const uint8_t _ArrowkFlippedBitmask[] = {254, 253, 251, 247, 239, 223, 191, 127}; static const uint8_t _ArrowkPrecedingBitmask[] = {0, 1, 3, 7, 15, 31, 63, 127}; -static const uint8_t _ArrowkTrailingBitmask[] = { - 255, 254, 252, 248, 240, 224, 192, 128}; +static const uint8_t _ArrowkTrailingBitmask[] = {255, 254, 252, 248, 240, 224, 192, 128}; static const uint8_t _ArrowkBytePopcount[] = { - 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, - 2, 3, 3, 4, 3, 4, 4, 5, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, - 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, - 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, - 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, - 4, 5, 5, 6, 5, 6, 6, 7, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, - 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, - 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, - 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, - 4, 5, 5, 6, 5, 6, 6, 7, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, - 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8}; + 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, + 4, 4, 5, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, + 4, 5, 4, 5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, + 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, + 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, + 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, + 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, + 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 3, 4, 4, 5, 4, 5, 5, 6, + 4, 5, 5, 6, 5, 6, 6, 7, 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8}; static inline int64_t _ArrowRoundUpToMultipleOf8(int64_t value) { return (value + 7) & ~((int64_t)7); @@ -2507,25 +2394,25 @@ static inline void _ArrowBitsUnpackInt32(const uint8_t word, int32_t* out) { } static inline void _ArrowBitmapPackInt8(const int8_t* values, uint8_t* out) { - *out = (uint8_t)(values[0] | ((values[1] + 0x1) & 0x2) | - ((values[2] + 0x3) & 0x4) | ((values[3] + 0x7) & 0x8) | - ((values[4] + 0xf) & 0x10) | ((values[5] + 0x1f) & 0x20) | - ((values[6] + 0x3f) & 0x40) | ((values[7] + 0x7f) & 0x80)); + *out = (uint8_t)(values[0] | ((values[1] + 0x1) & 0x2) | ((values[2] + 0x3) & 0x4) | + ((values[3] + 0x7) & 0x8) | ((values[4] + 0xf) & 0x10) | + ((values[5] + 0x1f) & 0x20) | ((values[6] + 0x3f) & 0x40) | + ((values[7] + 0x7f) & 0x80)); } static inline void _ArrowBitmapPackInt32(const int32_t* values, uint8_t* out) { - *out = (uint8_t)(values[0] | ((values[1] + 0x1) & 0x2) | - ((values[2] + 0x3) & 0x4) | ((values[3] + 0x7) & 0x8) | - ((values[4] + 0xf) & 0x10) | ((values[5] + 0x1f) & 0x20) | - ((values[6] + 0x3f) & 0x40) | ((values[7] + 0x7f) & 0x80)); + *out = (uint8_t)(values[0] | ((values[1] + 0x1) & 0x2) | ((values[2] + 0x3) & 0x4) | + ((values[3] + 0x7) & 0x8) | ((values[4] + 0xf) & 0x10) | + ((values[5] + 0x1f) & 0x20) | ((values[6] + 0x3f) & 0x40) | + ((values[7] + 0x7f) & 0x80)); } static inline int8_t ArrowBitGet(const uint8_t* bits, int64_t i) { return (bits[i >> 3] >> (i & 0x07)) & 1; } -static inline void ArrowBitsUnpackInt8( - const uint8_t* bits, int64_t start_offset, int64_t length, int8_t* out) { +static inline void ArrowBitsUnpackInt8(const uint8_t* bits, int64_t start_offset, + int64_t length, int8_t* out) { if (length == 0) { return; } @@ -2563,8 +2450,8 @@ static inline void ArrowBitsUnpackInt8( } } -static inline void ArrowBitsUnpackInt32( - const uint8_t* bits, int64_t start_offset, int64_t length, int32_t* out) { +static inline void ArrowBitsUnpackInt32(const uint8_t* bits, int64_t start_offset, + int64_t length, int32_t* out) { if (length == 0) { return; } @@ -2611,12 +2498,12 @@ static inline void ArrowBitClear(uint8_t* bits, int64_t i) { } static inline void ArrowBitSetTo(uint8_t* bits, int64_t i, uint8_t bit_is_set) { - bits[i / 8] ^= ((uint8_t)(-((uint8_t)(bit_is_set != 0)) ^ bits[i / 8])) & - _ArrowkBitmask[i % 8]; + bits[i / 8] ^= + ((uint8_t)(-((uint8_t)(bit_is_set != 0)) ^ bits[i / 8])) & _ArrowkBitmask[i % 8]; } -static inline void ArrowBitsSetTo( - uint8_t* bits, int64_t start_offset, int64_t length, uint8_t bits_are_set) { +static inline void ArrowBitsSetTo(uint8_t* bits, int64_t start_offset, int64_t length, + uint8_t bits_are_set) { const int64_t i_begin = start_offset; const int64_t i_end = start_offset + length; const uint8_t fill_byte = (uint8_t)(-bits_are_set); @@ -2630,8 +2517,7 @@ static inline void ArrowBitsSetTo( if (bytes_end == bytes_begin + 1) { // set bits within a single byte const uint8_t only_byte_mask = - i_end % 8 == 0 ? first_byte_mask : - (uint8_t)(first_byte_mask | last_byte_mask); + i_end % 8 == 0 ? first_byte_mask : (uint8_t)(first_byte_mask | last_byte_mask); bits[bytes_begin] &= only_byte_mask; bits[bytes_begin] |= (uint8_t)(fill_byte & ~only_byte_mask); return; @@ -2643,10 +2529,7 @@ static inline void ArrowBitsSetTo( if (bytes_end - bytes_begin > 2) { // set/clear whole bytes - memset( - bits + bytes_begin + 1, - fill_byte, - (size_t)(bytes_end - bytes_begin - 2)); + memset(bits + bytes_begin + 1, fill_byte, (size_t)(bytes_end - bytes_begin - 2)); } if (i_end % 8 == 0) { @@ -2658,8 +2541,8 @@ static inline void ArrowBitsSetTo( bits[bytes_end - 1] |= (uint8_t)(fill_byte & ~last_byte_mask); } -static inline int64_t ArrowBitCountSet( - const uint8_t* bits, int64_t start_offset, int64_t length) { +static inline int64_t ArrowBitCountSet(const uint8_t* bits, int64_t start_offset, + int64_t length) { if (length == 0) { return 0; } @@ -2677,16 +2560,14 @@ static inline int64_t ArrowBitCountSet( const uint8_t last_byte_mask = _ArrowkTrailingBitmask[i_begin % 8]; const uint8_t only_byte_mask = - i_end % 8 == 0 ? last_byte_mask : - (uint8_t)(first_byte_mask & last_byte_mask); + i_end % 8 == 0 ? last_byte_mask : (uint8_t)(first_byte_mask & last_byte_mask); const uint8_t byte_masked = bits[bytes_begin] & only_byte_mask; return _ArrowkBytePopcount[byte_masked]; } const uint8_t first_byte_mask = _ArrowkPrecedingBitmask[i_begin % 8]; - const uint8_t last_byte_mask = - i_end % 8 == 0 ? 0 : _ArrowkTrailingBitmask[i_end % 8]; + const uint8_t last_byte_mask = i_end % 8 == 0 ? 0 : _ArrowkTrailingBitmask[i_end % 8]; int64_t count = 0; // first byte @@ -2708,29 +2589,29 @@ static inline void ArrowBitmapInit(struct ArrowBitmap* bitmap) { bitmap->size_bits = 0; } -static inline void ArrowBitmapMove( - struct ArrowBitmap* src, struct ArrowBitmap* dst) { +static inline void ArrowBitmapMove(struct ArrowBitmap* src, struct ArrowBitmap* dst) { ArrowBufferMove(&src->buffer, &dst->buffer); dst->size_bits = src->size_bits; src->size_bits = 0; } -static inline ArrowErrorCode ArrowBitmapReserve( - struct ArrowBitmap* bitmap, int64_t additional_size_bits) { +static inline ArrowErrorCode ArrowBitmapReserve(struct ArrowBitmap* bitmap, + int64_t additional_size_bits) { int64_t min_capacity_bits = bitmap->size_bits + additional_size_bits; if (min_capacity_bits <= (bitmap->buffer.capacity_bytes * 8)) { return NANOARROW_OK; } - NANOARROW_RETURN_NOT_OK(ArrowBufferReserve( - &bitmap->buffer, _ArrowBytesForBits(additional_size_bits))); + NANOARROW_RETURN_NOT_OK( + ArrowBufferReserve(&bitmap->buffer, _ArrowBytesForBits(additional_size_bits))); bitmap->buffer.data[bitmap->buffer.capacity_bytes - 1] = 0; return NANOARROW_OK; } -static inline ArrowErrorCode ArrowBitmapResize( - struct ArrowBitmap* bitmap, int64_t new_capacity_bits, char shrink_to_fit) { +static inline ArrowErrorCode ArrowBitmapResize(struct ArrowBitmap* bitmap, + int64_t new_capacity_bits, + char shrink_to_fit) { if (new_capacity_bits < 0) { return EINVAL; } @@ -2746,23 +2627,23 @@ static inline ArrowErrorCode ArrowBitmapResize( return NANOARROW_OK; } -static inline ArrowErrorCode ArrowBitmapAppend( - struct ArrowBitmap* bitmap, uint8_t bits_are_set, int64_t length) { +static inline ArrowErrorCode ArrowBitmapAppend(struct ArrowBitmap* bitmap, + uint8_t bits_are_set, int64_t length) { NANOARROW_RETURN_NOT_OK(ArrowBitmapReserve(bitmap, length)); ArrowBitmapAppendUnsafe(bitmap, bits_are_set, length); return NANOARROW_OK; } -static inline void ArrowBitmapAppendUnsafe( - struct ArrowBitmap* bitmap, uint8_t bits_are_set, int64_t length) { +static inline void ArrowBitmapAppendUnsafe(struct ArrowBitmap* bitmap, + uint8_t bits_are_set, int64_t length) { ArrowBitsSetTo(bitmap->buffer.data, bitmap->size_bits, length, bits_are_set); bitmap->size_bits += length; bitmap->buffer.size_bytes = _ArrowBytesForBits(bitmap->size_bits); } -static inline void ArrowBitmapAppendInt8Unsafe( - struct ArrowBitmap* bitmap, const int8_t* values, int64_t n_values) { +static inline void ArrowBitmapAppendInt8Unsafe(struct ArrowBitmap* bitmap, + const int8_t* values, int64_t n_values) { if (n_values == 0) { return; } @@ -2774,8 +2655,7 @@ static inline void ArrowBitmapAppendInt8Unsafe( // First byte if ((out_i_cursor % 8) != 0) { - int64_t n_partial_bits = - _ArrowRoundUpToMultipleOf8(out_i_cursor) - out_i_cursor; + int64_t n_partial_bits = _ArrowRoundUpToMultipleOf8(out_i_cursor) - out_i_cursor; for (int i = 0; i < n_partial_bits; i++) { ArrowBitSetTo(bitmap->buffer.data, out_i_cursor++, values[i]); } @@ -2809,8 +2689,8 @@ static inline void ArrowBitmapAppendInt8Unsafe( bitmap->buffer.size_bytes = out_cursor - bitmap->buffer.data; } -static inline void ArrowBitmapAppendInt32Unsafe( - struct ArrowBitmap* bitmap, const int32_t* values, int64_t n_values) { +static inline void ArrowBitmapAppendInt32Unsafe(struct ArrowBitmap* bitmap, + const int32_t* values, int64_t n_values) { if (n_values == 0) { return; } @@ -2822,8 +2702,7 @@ static inline void ArrowBitmapAppendInt32Unsafe( // First byte if ((out_i_cursor % 8) != 0) { - int64_t n_partial_bits = - _ArrowRoundUpToMultipleOf8(out_i_cursor) - out_i_cursor; + int64_t n_partial_bits = _ArrowRoundUpToMultipleOf8(out_i_cursor) - out_i_cursor; for (int i = 0; i < n_partial_bits; i++) { ArrowBitSetTo(bitmap->buffer.data, out_i_cursor++, (uint8_t)values[i]); } @@ -2848,8 +2727,7 @@ static inline void ArrowBitmapAppendInt32Unsafe( // Zero out the last byte *out_cursor = 0x00; for (int i = 0; i < n_remaining; i++) { - ArrowBitSetTo( - bitmap->buffer.data, out_i_cursor++, (uint8_t)values_cursor[i]); + ArrowBitSetTo(bitmap->buffer.data, out_i_cursor++, (uint8_t)values_cursor[i]); } out_cursor++; } @@ -2894,19 +2772,20 @@ static inline void ArrowBitmapReset(struct ArrowBitmap* bitmap) { #include #include + + + #ifdef __cplusplus extern "C" { #endif -static inline struct ArrowBitmap* ArrowArrayValidityBitmap( - struct ArrowArray* array) { +static inline struct ArrowBitmap* ArrowArrayValidityBitmap(struct ArrowArray* array) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; return &private_data->bitmap; } -static inline struct ArrowBuffer* ArrowArrayBuffer( - struct ArrowArray* array, int64_t i) { +static inline struct ArrowBuffer* ArrowArrayBuffer(struct ArrowArray* array, int64_t i) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; switch (i) { @@ -2920,20 +2799,19 @@ static inline struct ArrowBuffer* ArrowArrayBuffer( // We don't currently support the case of unions where type_id != child_index; // however, these functions are used to keep track of where that assumption // is made. -static inline int8_t _ArrowArrayUnionChildIndex( - struct ArrowArray* array, int8_t type_id) { +static inline int8_t _ArrowArrayUnionChildIndex(struct ArrowArray* array, + int8_t type_id) { NANOARROW_UNUSED(array); return type_id; } -static inline int8_t _ArrowArrayUnionTypeId( - struct ArrowArray* array, int8_t child_index) { +static inline int8_t _ArrowArrayUnionTypeId(struct ArrowArray* array, + int8_t child_index) { NANOARROW_UNUSED(array); return child_index; } -static inline int32_t _ArrowParseUnionTypeIds( - const char* type_ids, int8_t* out) { +static inline int32_t _ArrowParseUnionTypeIds(const char* type_ids, int8_t* out) { if (*type_ids == '\0') { return 0; } @@ -2966,8 +2844,9 @@ static inline int32_t _ArrowParseUnionTypeIds( return -1; } -static inline int8_t _ArrowParsedUnionTypeIdsWillEqualChildIndices( - const int8_t* type_ids, int64_t n_type_ids, int64_t n_children) { +static inline int8_t _ArrowParsedUnionTypeIdsWillEqualChildIndices(const int8_t* type_ids, + int64_t n_type_ids, + int64_t n_children) { if (n_type_ids != n_children) { return 0; } @@ -2981,16 +2860,14 @@ static inline int8_t _ArrowParsedUnionTypeIdsWillEqualChildIndices( return 1; } -static inline int8_t _ArrowUnionTypeIdsWillEqualChildIndices( - const char* type_id_str, int64_t n_children) { +static inline int8_t _ArrowUnionTypeIdsWillEqualChildIndices(const char* type_id_str, + int64_t n_children) { int8_t type_ids[128]; int32_t n_type_ids = _ArrowParseUnionTypeIds(type_id_str, type_ids); - return _ArrowParsedUnionTypeIdsWillEqualChildIndices( - type_ids, n_type_ids, n_children); + return _ArrowParsedUnionTypeIdsWillEqualChildIndices(type_ids, n_type_ids, n_children); } -static inline ArrowErrorCode ArrowArrayStartAppending( - struct ArrowArray* array) { +static inline ArrowErrorCode ArrowArrayStartAppending(struct ArrowArray* array) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -3014,17 +2891,12 @@ static inline ArrowErrorCode ArrowArrayStartAppending( // Initialize any data offset buffer with a single zero for (int i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) { - if (private_data->layout.buffer_type[i] == - NANOARROW_BUFFER_TYPE_DATA_OFFSET && + if (private_data->layout.buffer_type[i] == NANOARROW_BUFFER_TYPE_DATA_OFFSET && private_data->layout.element_size_bits[i] == 64) { - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendInt64(ArrowArrayBuffer(array, i), 0)); - } else if ( - private_data->layout.buffer_type[i] == - NANOARROW_BUFFER_TYPE_DATA_OFFSET && - private_data->layout.element_size_bits[i] == 32) { - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendInt32(ArrowArrayBuffer(array, i), 0)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt64(ArrowArrayBuffer(array, i), 0)); + } else if (private_data->layout.buffer_type[i] == NANOARROW_BUFFER_TYPE_DATA_OFFSET && + private_data->layout.element_size_bits[i] == 32) { + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(ArrowArrayBuffer(array, i), 0)); } } @@ -3057,15 +2929,15 @@ static inline ArrowErrorCode ArrowArrayShrinkToFit(struct ArrowArray* array) { return NANOARROW_OK; } -static inline ArrowErrorCode _ArrowArrayAppendBits( - struct ArrowArray* array, int64_t buffer_i, uint8_t value, int64_t n) { +static inline ArrowErrorCode _ArrowArrayAppendBits(struct ArrowArray* array, + int64_t buffer_i, uint8_t value, + int64_t n) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; struct ArrowBuffer* buffer = ArrowArrayBuffer(array, buffer_i); int64_t bytes_required = - _ArrowRoundUpToMultipleOf8( - private_data->layout.element_size_bits[buffer_i] * - (array->length + 1)) / + _ArrowRoundUpToMultipleOf8(private_data->layout.element_size_bits[buffer_i] * + (array->length + 1)) / 8; if (bytes_required > buffer->size_bytes) { NANOARROW_RETURN_NOT_OK( @@ -3076,8 +2948,8 @@ static inline ArrowErrorCode _ArrowArrayAppendBits( return NANOARROW_OK; } -static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal( - struct ArrowArray* array, int64_t n, uint8_t is_valid) { +static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal(struct ArrowArray* array, + int64_t n, uint8_t is_valid) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -3102,11 +2974,10 @@ static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal( ArrowBufferAppendFill(ArrowArrayBuffer(array, 0), type_id, n)); for (int64_t i = 0; i < n; i++) { NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32( - ArrowArrayBuffer(array, 1), - (int32_t)array->children[0]->length - 1)); + ArrowArrayBuffer(array, 1), (int32_t)array->children[0]->length - 1)); } - // For the purposes of array->null_count, union elements are never - // considered "null" even if some children contain nulls. + // For the purposes of array->null_count, union elements are never considered "null" + // even if some children contain nulls. array->length += n; return NANOARROW_OK; } @@ -3122,8 +2993,8 @@ static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal( NANOARROW_RETURN_NOT_OK( ArrowBufferAppendFill(ArrowArrayBuffer(array, 0), type_id, n)); - // For the purposes of array->null_count, union elements are never - // considered "null" even if some children contain nulls. + // For the purposes of array->null_count, union elements are never considered "null" + // even if some children contain nulls. array->length += n; return NANOARROW_OK; } @@ -3142,11 +3013,10 @@ static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal( break; } - // Append n is_valid bits to the validity bitmap. If we haven't allocated a - // bitmap yet and we need to append nulls, do it now. + // Append n is_valid bits to the validity bitmap. If we haven't allocated a bitmap yet + // and we need to append nulls, do it now. if (!is_valid && private_data->bitmap.buffer.data == NULL) { - NANOARROW_RETURN_NOT_OK( - ArrowBitmapReserve(&private_data->bitmap, array->length + n)); + NANOARROW_RETURN_NOT_OK(ArrowBitmapReserve(&private_data->bitmap, array->length + n)); ArrowBitmapAppendUnsafe(&private_data->bitmap, 1, array->length); ArrowBitmapAppendUnsafe(&private_data->bitmap, is_valid, n); } else if (private_data->bitmap.buffer.data != NULL) { @@ -3167,15 +3037,12 @@ static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal( case NANOARROW_BUFFER_TYPE_VALIDITY: continue; case NANOARROW_BUFFER_TYPE_DATA_OFFSET: - // Append the current value at the end of the offset buffer for each - // element + // Append the current value at the end of the offset buffer for each element NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, size_bytes * n)); for (int64_t j = 0; j < n; j++) { - ArrowBufferAppendUnsafe( - buffer, - buffer->data + size_bytes * (array->length + j), - size_bytes); + ArrowBufferAppendUnsafe(buffer, buffer->data + size_bytes * (array->length + j), + size_bytes); } // Skip the data buffer @@ -3184,8 +3051,7 @@ static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal( case NANOARROW_BUFFER_TYPE_DATA: // Zero out the next bit of memory if (private_data->layout.element_size_bits[i] % 8 == 0) { - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendFill(buffer, 0, size_bytes * n)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendFill(buffer, 0, size_bytes * n)); } else { NANOARROW_RETURN_NOT_OK(_ArrowArrayAppendBits(array, i, 0, n)); } @@ -3203,18 +3069,16 @@ static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal( return NANOARROW_OK; } -static inline ArrowErrorCode ArrowArrayAppendNull( - struct ArrowArray* array, int64_t n) { +static inline ArrowErrorCode ArrowArrayAppendNull(struct ArrowArray* array, int64_t n) { return _ArrowArrayAppendEmptyInternal(array, n, 0); } -static inline ArrowErrorCode ArrowArrayAppendEmpty( - struct ArrowArray* array, int64_t n) { +static inline ArrowErrorCode ArrowArrayAppendEmpty(struct ArrowArray* array, int64_t n) { return _ArrowArrayAppendEmptyInternal(array, n, 1); } -static inline ArrowErrorCode ArrowArrayAppendInt( - struct ArrowArray* array, int64_t value) { +static inline ArrowErrorCode ArrowArrayAppendInt(struct ArrowArray* array, + int64_t value) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -3222,23 +3086,19 @@ static inline ArrowErrorCode ArrowArrayAppendInt( switch (private_data->storage_type) { case NANOARROW_TYPE_INT64: - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppend(data_buffer, &value, sizeof(int64_t))); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(data_buffer, &value, sizeof(int64_t))); break; case NANOARROW_TYPE_INT32: _NANOARROW_CHECK_RANGE(value, INT32_MIN, INT32_MAX); - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendInt32(data_buffer, (int32_t)value)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(data_buffer, (int32_t)value)); break; case NANOARROW_TYPE_INT16: _NANOARROW_CHECK_RANGE(value, INT16_MIN, INT16_MAX); - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendInt16(data_buffer, (int16_t)value)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt16(data_buffer, (int16_t)value)); break; case NANOARROW_TYPE_INT8: _NANOARROW_CHECK_RANGE(value, INT8_MIN, INT8_MAX); - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendInt8(data_buffer, (int8_t)value)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt8(data_buffer, (int8_t)value)); break; case NANOARROW_TYPE_UINT64: case NANOARROW_TYPE_UINT32: @@ -3247,12 +3107,10 @@ static inline ArrowErrorCode ArrowArrayAppendInt( _NANOARROW_CHECK_RANGE(value, 0, INT64_MAX); return ArrowArrayAppendUInt(array, value); case NANOARROW_TYPE_DOUBLE: - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendDouble(data_buffer, (double)value)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendDouble(data_buffer, (double)value)); break; case NANOARROW_TYPE_FLOAT: - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendFloat(data_buffer, (float)value)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendFloat(data_buffer, (float)value)); break; case NANOARROW_TYPE_BOOL: NANOARROW_RETURN_NOT_OK(_ArrowArrayAppendBits(array, 1, value != 0, 1)); @@ -3262,16 +3120,15 @@ static inline ArrowErrorCode ArrowArrayAppendInt( } if (private_data->bitmap.buffer.data != NULL) { - NANOARROW_RETURN_NOT_OK( - ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); + NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); } array->length++; return NANOARROW_OK; } -static inline ArrowErrorCode ArrowArrayAppendUInt( - struct ArrowArray* array, uint64_t value) { +static inline ArrowErrorCode ArrowArrayAppendUInt(struct ArrowArray* array, + uint64_t value) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -3279,23 +3136,19 @@ static inline ArrowErrorCode ArrowArrayAppendUInt( switch (private_data->storage_type) { case NANOARROW_TYPE_UINT64: - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppend(data_buffer, &value, sizeof(uint64_t))); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(data_buffer, &value, sizeof(uint64_t))); break; case NANOARROW_TYPE_UINT32: _NANOARROW_CHECK_UPPER_LIMIT(value, UINT32_MAX); - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendUInt32(data_buffer, (uint32_t)value)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendUInt32(data_buffer, (uint32_t)value)); break; case NANOARROW_TYPE_UINT16: _NANOARROW_CHECK_UPPER_LIMIT(value, UINT16_MAX); - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendUInt16(data_buffer, (uint16_t)value)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendUInt16(data_buffer, (uint16_t)value)); break; case NANOARROW_TYPE_UINT8: _NANOARROW_CHECK_UPPER_LIMIT(value, UINT8_MAX); - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendUInt8(data_buffer, (uint8_t)value)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendUInt8(data_buffer, (uint8_t)value)); break; case NANOARROW_TYPE_INT64: case NANOARROW_TYPE_INT32: @@ -3304,12 +3157,10 @@ static inline ArrowErrorCode ArrowArrayAppendUInt( _NANOARROW_CHECK_UPPER_LIMIT(value, INT64_MAX); return ArrowArrayAppendInt(array, value); case NANOARROW_TYPE_DOUBLE: - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendDouble(data_buffer, (double)value)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendDouble(data_buffer, (double)value)); break; case NANOARROW_TYPE_FLOAT: - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendFloat(data_buffer, (float)value)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendFloat(data_buffer, (float)value)); break; case NANOARROW_TYPE_BOOL: NANOARROW_RETURN_NOT_OK(_ArrowArrayAppendBits(array, 1, value != 0, 1)); @@ -3319,16 +3170,15 @@ static inline ArrowErrorCode ArrowArrayAppendUInt( } if (private_data->bitmap.buffer.data != NULL) { - NANOARROW_RETURN_NOT_OK( - ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); + NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); } array->length++; return NANOARROW_OK; } -static inline ArrowErrorCode ArrowArrayAppendDouble( - struct ArrowArray* array, double value) { +static inline ArrowErrorCode ArrowArrayAppendDouble(struct ArrowArray* array, + double value) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -3336,35 +3186,31 @@ static inline ArrowErrorCode ArrowArrayAppendDouble( switch (private_data->storage_type) { case NANOARROW_TYPE_DOUBLE: - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppend(data_buffer, &value, sizeof(double))); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(data_buffer, &value, sizeof(double))); break; case NANOARROW_TYPE_FLOAT: - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendFloat(data_buffer, (float)value)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendFloat(data_buffer, (float)value)); break; default: return EINVAL; } if (private_data->bitmap.buffer.data != NULL) { - NANOARROW_RETURN_NOT_OK( - ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); + NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); } array->length++; return NANOARROW_OK; } -static inline ArrowErrorCode ArrowArrayAppendBytes( - struct ArrowArray* array, struct ArrowBufferView value) { +static inline ArrowErrorCode ArrowArrayAppendBytes(struct ArrowArray* array, + struct ArrowBufferView value) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; struct ArrowBuffer* offset_buffer = ArrowArrayBuffer(array, 1); struct ArrowBuffer* data_buffer = ArrowArrayBuffer( - array, - 1 + (private_data->storage_type != NANOARROW_TYPE_FIXED_SIZE_BINARY)); + array, 1 + (private_data->storage_type != NANOARROW_TYPE_FIXED_SIZE_BINARY)); int32_t offset; int64_t large_offset; int64_t fixed_size_bytes = private_data->layout.element_size_bits[1] / 8; @@ -3378,8 +3224,7 @@ static inline ArrowErrorCode ArrowArrayAppendBytes( } offset += (int32_t)value.size_bytes; - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppend(offset_buffer, &offset, sizeof(int32_t))); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(offset_buffer, &offset, sizeof(int32_t))); NANOARROW_RETURN_NOT_OK( ArrowBufferAppend(data_buffer, value.data.data, value.size_bytes)); break; @@ -3407,16 +3252,15 @@ static inline ArrowErrorCode ArrowArrayAppendBytes( } if (private_data->bitmap.buffer.data != NULL) { - NANOARROW_RETURN_NOT_OK( - ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); + NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); } array->length++; return NANOARROW_OK; } -static inline ArrowErrorCode ArrowArrayAppendString( - struct ArrowArray* array, struct ArrowStringView value) { +static inline ArrowErrorCode ArrowArrayAppendString(struct ArrowArray* array, + struct ArrowStringView value) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -3435,8 +3279,8 @@ static inline ArrowErrorCode ArrowArrayAppendString( } } -static inline ArrowErrorCode ArrowArrayAppendInterval( - struct ArrowArray* array, const struct ArrowInterval* value) { +static inline ArrowErrorCode ArrowArrayAppendInterval(struct ArrowArray* array, + const struct ArrowInterval* value) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -3448,8 +3292,7 @@ static inline ArrowErrorCode ArrowArrayAppendInterval( return EINVAL; } - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendInt32(data_buffer, value->months)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(data_buffer, value->months)); break; } case NANOARROW_TYPE_INTERVAL_DAY_TIME: { @@ -3466,8 +3309,7 @@ static inline ArrowErrorCode ArrowArrayAppendInterval( return EINVAL; } - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendInt32(data_buffer, value->months)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(data_buffer, value->months)); NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(data_buffer, value->days)); NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt64(data_buffer, value->ns)); break; @@ -3477,16 +3319,15 @@ static inline ArrowErrorCode ArrowArrayAppendInterval( } if (private_data->bitmap.buffer.data != NULL) { - NANOARROW_RETURN_NOT_OK( - ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); + NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); } array->length++; return NANOARROW_OK; } -static inline ArrowErrorCode ArrowArrayAppendDecimal( - struct ArrowArray* array, const struct ArrowDecimal* value) { +static inline ArrowErrorCode ArrowArrayAppendDecimal(struct ArrowArray* array, + const struct ArrowDecimal* value) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; struct ArrowBuffer* data_buffer = ArrowArrayBuffer(array, 1); @@ -3513,8 +3354,7 @@ static inline ArrowErrorCode ArrowArrayAppendDecimal( } if (private_data->bitmap.buffer.data != NULL) { - NANOARROW_RETURN_NOT_OK( - ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); + NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); } array->length++; @@ -3534,8 +3374,8 @@ static inline ArrowErrorCode ArrowArrayFinishElement(struct ArrowArray* array) { if (child_length > INT32_MAX) { return EOVERFLOW; } - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32( - ArrowArrayBuffer(array, 1), (int32_t)child_length)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendInt32(ArrowArrayBuffer(array, 1), (int32_t)child_length)); break; case NANOARROW_TYPE_LARGE_LIST: child_length = array->children[0]->length; @@ -3562,16 +3402,15 @@ static inline ArrowErrorCode ArrowArrayFinishElement(struct ArrowArray* array) { } if (private_data->bitmap.buffer.data != NULL) { - NANOARROW_RETURN_NOT_OK( - ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); + NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); } array->length++; return NANOARROW_OK; } -static inline ArrowErrorCode ArrowArrayFinishUnionElement( - struct ArrowArray* array, int8_t type_id) { +static inline ArrowErrorCode ArrowArrayFinishUnionElement(struct ArrowArray* array, + int8_t type_id) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -3583,19 +3422,15 @@ static inline ArrowErrorCode ArrowArrayFinishUnionElement( switch (private_data->storage_type) { case NANOARROW_TYPE_DENSE_UNION: // Append the target child length to the union offsets buffer - _NANOARROW_CHECK_RANGE( - array->children[child_index]->length, 0, INT32_MAX); + _NANOARROW_CHECK_RANGE(array->children[child_index]->length, 0, INT32_MAX); NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32( - ArrowArrayBuffer(array, 1), - (int32_t)array->children[child_index]->length - 1)); + ArrowArrayBuffer(array, 1), (int32_t)array->children[child_index]->length - 1)); break; case NANOARROW_TYPE_SPARSE_UNION: - // Append one empty to any non-target column that isn't already the right - // length or abort if appending a null will result in a column with - // invalid length + // Append one empty to any non-target column that isn't already the right length + // or abort if appending a null will result in a column with invalid length for (int64_t i = 0; i < array->n_children; i++) { - if (i == child_index || - array->children[i]->length == (array->length + 1)) { + if (i == child_index || array->children[i]->length == (array->length + 1)) { continue; } @@ -3618,14 +3453,14 @@ static inline ArrowErrorCode ArrowArrayFinishUnionElement( return NANOARROW_OK; } -static inline void ArrowArrayViewMove( - struct ArrowArrayView* src, struct ArrowArrayView* dst) { +static inline void ArrowArrayViewMove(struct ArrowArrayView* src, + struct ArrowArrayView* dst) { memcpy(dst, src, sizeof(struct ArrowArrayView)); ArrowArrayViewInitFromType(src, NANOARROW_TYPE_UNINITIALIZED); } -static inline int8_t ArrowArrayViewIsNull( - const struct ArrowArrayView* array_view, int64_t i) { +static inline int8_t ArrowArrayViewIsNull(const struct ArrowArrayView* array_view, + int64_t i) { const uint8_t* validity_buffer = array_view->buffer_views[0].data.as_uint8; i += array_view->offset; switch (array_view->storage_type) { @@ -3640,8 +3475,8 @@ static inline int8_t ArrowArrayViewIsNull( } } -static inline int8_t ArrowArrayViewUnionTypeId( - const struct ArrowArrayView* array_view, int64_t i) { +static inline int8_t ArrowArrayViewUnionTypeId(const struct ArrowArrayView* array_view, + int64_t i) { switch (array_view->storage_type) { case NANOARROW_TYPE_DENSE_UNION: case NANOARROW_TYPE_SPARSE_UNION: @@ -3685,8 +3520,8 @@ static inline int64_t ArrowArrayViewListChildOffset( } } -static inline int64_t ArrowArrayViewGetIntUnsafe( - const struct ArrowArrayView* array_view, int64_t i) { +static inline int64_t ArrowArrayViewGetIntUnsafe(const struct ArrowArrayView* array_view, + int64_t i) { const struct ArrowBufferView* data_view = &array_view->buffer_views[1]; i += array_view->offset; switch (array_view->storage_type) { @@ -3805,8 +3640,7 @@ static inline struct ArrowStringView ArrowArrayViewGetStringUnsafe( break; case NANOARROW_TYPE_FIXED_SIZE_BINARY: view.size_bytes = array_view->layout.element_size_bits[1] / 8; - view.data = - array_view->buffer_views[1].data.as_char + (i * view.size_bytes); + view.data = array_view->buffer_views[1].data.as_char + (i * view.size_bytes); break; default: view.data = NULL; @@ -3852,9 +3686,7 @@ static inline struct ArrowBufferView ArrowArrayViewGetBytesUnsafe( } static inline void ArrowArrayViewGetIntervalUnsafe( - const struct ArrowArrayView* array_view, - int64_t i, - struct ArrowInterval* out) { + const struct ArrowArrayView* array_view, int64_t i, struct ArrowInterval* out) { const uint8_t* data_view = array_view->buffer_views[1].data.as_uint8; switch (array_view->storage_type) { case NANOARROW_TYPE_INTERVAL_MONTHS: { @@ -3880,10 +3712,8 @@ static inline void ArrowArrayViewGetIntervalUnsafe( } } -static inline void ArrowArrayViewGetDecimalUnsafe( - const struct ArrowArrayView* array_view, - int64_t i, - struct ArrowDecimal* out) { +static inline void ArrowArrayViewGetDecimalUnsafe(const struct ArrowArrayView* array_view, + int64_t i, struct ArrowDecimal* out) { i += array_view->offset; const uint8_t* data_view = array_view->buffer_views[1].data.as_uint8; switch (array_view->storage_type) { From 0a48261092f2a531449384bc2f18182210ae61ff Mon Sep 17 00:00:00 2001 From: Vivian Nguyen Date: Tue, 9 Apr 2024 12:11:22 -0500 Subject: [PATCH 03/10] Add unit tests --- libtiledbsoma/test/common.h | 66 +++++++++++++++ libtiledbsoma/test/test_indexer.cc | 5 +- libtiledbsoma/test/unit_soma_array.cc | 10 +-- libtiledbsoma/test/unit_soma_collection.cc | 81 +++++-------------- libtiledbsoma/test/unit_soma_dataframe.cc | 81 +++++-------------- libtiledbsoma/test/unit_soma_dense_ndarray.cc | 41 +++++----- .../test/unit_soma_sparse_ndarray.cc | 30 ++++--- 7 files changed, 152 insertions(+), 162 deletions(-) create mode 100644 libtiledbsoma/test/common.h diff --git a/libtiledbsoma/test/common.h b/libtiledbsoma/test/common.h new file mode 100644 index 0000000000..16ce7e4bbd --- /dev/null +++ b/libtiledbsoma/test/common.h @@ -0,0 +1,66 @@ +/** + * @file common.h + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file manages common headers and helper classes for the unit test files. + */ + +#ifndef UNIT_TEST_COMMON_H +#define UNIT_TEST_COMMON_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include "utils/util.h" + +using namespace tiledb; +using namespace tiledbsoma; +using namespace Catch::Matchers; + +#ifndef TILEDBSOMA_SOURCE_ROOT +#define TILEDBSOMA_SOURCE_ROOT "not_defined" +#endif + +static const std::string src_path = TILEDBSOMA_SOURCE_ROOT; + +namespace helper { +ArraySchema create_schema(Context& ctx, bool allow_duplicates = false); +std::pair, ColumnIndexInfo> create_arrow_schema(); +} // namespace helper +#endif \ No newline at end of file diff --git a/libtiledbsoma/test/test_indexer.cc b/libtiledbsoma/test/test_indexer.cc index 4f507940b6..c99d13ff61 100644 --- a/libtiledbsoma/test/test_indexer.cc +++ b/libtiledbsoma/test/test_indexer.cc @@ -31,10 +31,8 @@ */ #include -#include #include #include -#include #include #include #include @@ -59,9 +57,8 @@ bool run_test(int id, std::vector keys, std::vector lookups) { try { std::vector indexer_results; indexer_results.resize(lookups.size()); - auto context = std::make_shared(); - tiledbsoma::IntIndexer indexer(context); + tiledbsoma::IntIndexer indexer; indexer.map_locations(keys); auto* hash = kh_init(m64); int ret; diff --git a/libtiledbsoma/test/unit_soma_array.cc b/libtiledbsoma/test/unit_soma_array.cc index 610bc1f0bb..baaa0492d5 100644 --- a/libtiledbsoma/test/unit_soma_array.cc +++ b/libtiledbsoma/test/unit_soma_array.cc @@ -138,14 +138,10 @@ std::tuple, std::vector> write_array( } std::vector a0(num_cells_per_fragment, frag_num); - auto array_buffer = std::make_shared(); - auto tdb_arr = std::make_shared( - *ctx->tiledb_ctx(), uri, TILEDB_READ); - array_buffer->emplace("a0", ColumnBuffer::create(tdb_arr, "a0", a0)); - array_buffer->emplace("d0", ColumnBuffer::create(tdb_arr, "d0", d0)); - // Write data to array - soma_array->write(array_buffer); + soma_array->set_column_data("a0", a0.size(), a0.data()); + soma_array->set_column_data("d0", d0.size(), d0.data()); + soma_array->write(); soma_array->close(); } diff --git a/libtiledbsoma/test/unit_soma_collection.cc b/libtiledbsoma/test/unit_soma_collection.cc index 38336e1b27..914e263c70 100644 --- a/libtiledbsoma/test/unit_soma_collection.cc +++ b/libtiledbsoma/test/unit_soma_collection.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -30,52 +30,7 @@ * This file manages unit tests for the SOMACollection class */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include "utils/util.h" - -using namespace tiledb; -using namespace tiledbsoma; -using namespace Catch::Matchers; - -#ifndef TILEDBSOMA_SOURCE_ROOT -#define TILEDBSOMA_SOURCE_ROOT "not_defined" -#endif - -const std::string src_path = TILEDBSOMA_SOURCE_ROOT; - -namespace { -ArraySchema create_schema( - Context& ctx, bool sparse = false, bool allow_duplicates = false) { - // Create schema - ArraySchema schema(ctx, sparse ? TILEDB_SPARSE : TILEDB_DENSE); - - auto dim = Dimension::create(ctx, "d0", {0, 1000}); - - Domain domain(ctx); - domain.add_dimension(dim); - schema.set_domain(domain); - - auto attr = Attribute::create(ctx, "a0"); - schema.add_attribute(attr); - schema.set_allows_dups(allow_duplicates); - schema.check(); - - return schema; -} -}; // namespace +#include "common.h" TEST_CASE("SOMACollection: basic") { auto ctx = std::make_shared(); @@ -95,7 +50,8 @@ TEST_CASE("SOMACollection: add SOMASparseNDArray") { std::string sub_uri = "mem://unit-test-add-sparse-ndarray/sub"; SOMACollection::create(base_uri, ctx); - auto schema = create_schema(*ctx->tiledb_ctx(), true); + auto [arrow_schema, index_columns] = helper::create_arrow_schema(); + auto schema = helper::create_schema(*ctx->tiledb_ctx(), true); std::map expected_map{ {"sparse_ndarray", sub_uri}}; @@ -124,7 +80,7 @@ TEST_CASE("SOMACollection: add SOMADenseNDArray") { std::string sub_uri = "mem://unit-test-add-dense-ndarray/sub"; SOMACollection::create(base_uri, ctx); - auto schema = create_schema(*ctx->tiledb_ctx(), false); + auto schema = helper::create_schema(*ctx->tiledb_ctx(), false); std::map expected_map{{"dense_ndarray", sub_uri}}; @@ -151,13 +107,13 @@ TEST_CASE("SOMACollection: add SOMADataFrame") { std::string sub_uri = "mem://unit-test-add-dataframe/sub"; SOMACollection::create(base_uri, ctx); - auto schema = create_schema(*ctx->tiledb_ctx(), true); + auto [schema, index_columns] = helper::create_arrow_schema(); std::map expected_map{{"dataframe", sub_uri}}; auto soma_collection = SOMACollection::open(base_uri, OpenMode::write, ctx); auto soma_dataframe = soma_collection->add_new_dataframe( - "dataframe", sub_uri, URIType::absolute, ctx, schema); + "dataframe", sub_uri, URIType::absolute, ctx, schema, index_columns); REQUIRE(soma_collection->member_to_uri_mapping() == expected_map); REQUIRE(soma_dataframe->uri() == sub_uri); REQUIRE(soma_dataframe->ctx() == ctx); @@ -179,7 +135,7 @@ TEST_CASE("SOMACollection: add SOMACollection") { std::string sub_uri = "mem://unit-test-add-collection/sub"; SOMACollection::create(base_uri, ctx); - auto schema = create_schema(*ctx->tiledb_ctx(), false); + auto schema = helper::create_schema(*ctx->tiledb_ctx(), false); std::map expected_map{{"subcollection", sub_uri}}; @@ -203,13 +159,13 @@ TEST_CASE("SOMACollection: add SOMAExperiment") { std::string sub_uri = "mem://unit-test-add-experiment/sub"; SOMACollection::create(base_uri, ctx); - auto schema = create_schema(*ctx->tiledb_ctx(), false); + auto [schema, index_columns] = helper::create_arrow_schema(); std::map expected_map{{"experiment", sub_uri}}; auto soma_collection = SOMACollection::open(base_uri, OpenMode::write, ctx); auto soma_experiment = soma_collection->add_new_experiment( - "experiment", sub_uri, URIType::absolute, ctx, schema); + "experiment", sub_uri, URIType::absolute, ctx, schema, index_columns); REQUIRE(soma_collection->member_to_uri_mapping() == expected_map); REQUIRE(soma_experiment->uri() == sub_uri); REQUIRE(soma_experiment->ctx() == ctx); @@ -228,13 +184,13 @@ TEST_CASE("SOMACollection: add SOMAMeasurement") { std::string sub_uri = "mem://unit-test-add-measurement/sub"; SOMACollection::create(base_uri, ctx); - auto schema = create_schema(*ctx->tiledb_ctx(), false); + auto [schema, index_columns] = helper::create_arrow_schema(); std::map expected_map{{"measurement", sub_uri}}; auto soma_collection = SOMACollection::open(base_uri, OpenMode::write, ctx); auto soma_measurement = soma_collection->add_new_measurement( - "measurement", sub_uri, URIType::absolute, ctx, schema); + "measurement", sub_uri, URIType::absolute, ctx, schema, index_columns); REQUIRE(soma_collection->member_to_uri_mapping() == expected_map); REQUIRE(soma_measurement->uri() == sub_uri); REQUIRE(soma_measurement->ctx() == ctx); @@ -253,7 +209,7 @@ TEST_CASE("SOMACollection: metadata") { std::string uri = "mem://unit-test-collection"; SOMACollection::create(uri, ctx, TimestampRange(0, 2)); auto soma_collection = SOMACollection::open( - uri, OpenMode::write, ctx, TimestampRange(1, 1)); + uri, OpenMode::write, ctx, std::pair(1, 1)); int32_t val = 100; soma_collection->set_metadata("md", TILEDB_INT32, 1, &val); @@ -304,10 +260,11 @@ TEST_CASE("SOMAExperiment: metadata") { auto ctx = std::make_shared(); std::string uri = "mem://unit-test-experiment"; + auto [schema, index_columns] = helper::create_arrow_schema(); SOMAExperiment::create( - uri, create_schema(*ctx->tiledb_ctx()), ctx, TimestampRange(0, 2)); + uri, schema, index_columns, ctx, std::nullopt, TimestampRange(0, 2)); auto soma_experiment = SOMAExperiment::open( - uri, OpenMode::write, ctx, TimestampRange(1, 1)); + uri, OpenMode::write, ctx, std::pair(1, 1)); int32_t val = 100; soma_experiment->set_metadata("md", TILEDB_INT32, 1, &val); @@ -357,10 +314,12 @@ TEST_CASE("SOMAExperiment: metadata") { TEST_CASE("SOMAMeasurement: metadata") { auto ctx = std::make_shared(); std::string uri = "mem://unit-test-measurement"; + auto [schema, index_columns] = helper::create_arrow_schema(); SOMAMeasurement::create( - uri, create_schema(*ctx->tiledb_ctx()), ctx, TimestampRange(0, 2)); + uri, schema, index_columns, ctx, std::nullopt, TimestampRange(0, 2)); + auto soma_measurement = SOMAMeasurement::open( - uri, OpenMode::write, ctx, TimestampRange(1, 1)); + uri, OpenMode::write, ctx, std::pair(1, 1)); int32_t val = 100; soma_measurement->set_metadata("md", TILEDB_INT32, 1, &val); diff --git a/libtiledbsoma/test/unit_soma_dataframe.cc b/libtiledbsoma/test/unit_soma_dataframe.cc index 9f50de5807..39bceda87c 100644 --- a/libtiledbsoma/test/unit_soma_dataframe.cc +++ b/libtiledbsoma/test/unit_soma_dataframe.cc @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022 TileDB, Inc. + * @copyright Copyright (c) 2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -30,77 +30,37 @@ * This file manages unit tests for the SOMADataFrame class */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include "utils/util.h" - -using namespace tiledb; -using namespace tiledbsoma; -using namespace Catch::Matchers; - -#ifndef TILEDBSOMA_SOURCE_ROOT -#define TILEDBSOMA_SOURCE_ROOT "not_defined" -#endif - -const std::string src_path = TILEDBSOMA_SOURCE_ROOT; - -namespace { -ArraySchema create_schema(Context& ctx, bool allow_duplicates = false) { - // Create schema - ArraySchema schema(ctx, TILEDB_SPARSE); - - auto dim = Dimension::create(ctx, "d0", {0, 1000}); - - Domain domain(ctx); - domain.add_dimension(dim); - schema.set_domain(domain); - - auto attr = Attribute::create(ctx, "a0"); - schema.add_attribute(attr); - schema.set_allows_dups(allow_duplicates); - schema.check(); - - return schema; -} -}; // namespace +#include "common.h" TEST_CASE("SOMADataFrame: basic") { auto ctx = std::make_shared(); std::string uri = "mem://unit-test-dataframe-basic"; - auto soma_dataframe = SOMADataFrame::create( - uri, create_schema(*ctx->tiledb_ctx()), ctx); + auto [schema, index_columns] = helper::create_arrow_schema(); + SOMADataFrame::create(uri, schema, index_columns, ctx); + + auto soma_dataframe = SOMADataFrame::open(uri, OpenMode::read, ctx); + REQUIRE(soma_dataframe->uri() == uri); + REQUIRE(soma_dataframe->ctx() == ctx); + REQUIRE(soma_dataframe->type() == "SOMADataFrame"); + std::vector expected_index_column_names = {"d0"}; + REQUIRE( + soma_dataframe->index_column_names() == expected_index_column_names); + REQUIRE(soma_dataframe->count() == 0); + soma_dataframe->close(); std::vector d0(10); for (int j = 0; j < 10; j++) d0[j] = j; std::vector a0(10, 1); - auto array_buffer = std::make_shared(); - auto tdb_arr = std::make_shared( - *ctx->tiledb_ctx(), uri, TILEDB_READ); - array_buffer->emplace("a0", ColumnBuffer::create(tdb_arr, "a0", a0)); - array_buffer->emplace("d0", ColumnBuffer::create(tdb_arr, "d0", d0)); - - soma_dataframe->write(array_buffer); + soma_dataframe = SOMADataFrame::open(uri, OpenMode::write, ctx); + soma_dataframe->set_column_data("a0", a0.size(), a0.data()); + soma_dataframe->set_column_data("d0", d0.size(), d0.data()); + soma_dataframe->write(); soma_dataframe->close(); - soma_dataframe->open(OpenMode::read); - REQUIRE(soma_dataframe->uri() == uri); - REQUIRE(soma_dataframe->ctx() == ctx); - REQUIRE(soma_dataframe->type() == "SOMADataFrame"); + soma_dataframe = SOMADataFrame::open(uri, OpenMode::read, ctx); while (auto batch = soma_dataframe->read_next()) { auto arrbuf = batch.value(); auto d0span = arrbuf->at("d0")->data(); @@ -119,8 +79,9 @@ TEST_CASE("SOMADataFrame: basic") { TEST_CASE("SOMADataFrame: metadata") { auto ctx = std::make_shared(); std::string uri = "mem://unit-test-collection"; + auto [schema, index_columns] = helper::create_arrow_schema(); SOMADataFrame::create( - uri, create_schema(*ctx->tiledb_ctx()), ctx, TimestampRange(0, 2)); + uri, schema, index_columns, ctx, std::nullopt, TimestampRange(0, 2)); auto soma_dataframe = SOMADataFrame::open( uri, diff --git a/libtiledbsoma/test/unit_soma_dense_ndarray.cc b/libtiledbsoma/test/unit_soma_dense_ndarray.cc index e2e9c50e28..97f6677034 100644 --- a/libtiledbsoma/test/unit_soma_dense_ndarray.cc +++ b/libtiledbsoma/test/unit_soma_dense_ndarray.cc @@ -58,8 +58,8 @@ const std::string src_path = TILEDBSOMA_SOURCE_ROOT; namespace { ArraySchema create_schema(Context& ctx, bool allow_duplicates = false) { - // SOMADenseNDArray is actually a TILEDB_SPARSE under the hood - ArraySchema schema(ctx, TILEDB_SPARSE); + // Create schema + ArraySchema schema(ctx, TILEDB_DENSE); auto dim = Dimension::create(ctx, "d0", {0, 1000}); @@ -80,32 +80,37 @@ TEST_CASE("SOMADenseNDArray: basic") { auto ctx = std::make_shared(); std::string uri = "mem://unit-test-dense-ndarray-basic"; - auto soma_dense = SOMADenseNDArray::create( - uri, create_schema(*ctx->tiledb_ctx()), ctx); + SOMADenseNDArray::create(uri, create_schema(*ctx->tiledb_ctx()), ctx); - std::vector d0(10); - for (int j = 0; j < 10; j++) - d0[j] = j; - std::vector a0(10, 1); + auto soma_dense = SOMADenseNDArray::open(uri, OpenMode::read, ctx); + REQUIRE(soma_dense->uri() == uri); + REQUIRE(soma_dense->ctx() == ctx); + REQUIRE(soma_dense->type() == "SOMADenseNDArray"); + REQUIRE(soma_dense->is_sparse() == false); + auto schema = soma_dense->tiledb_schema(); + REQUIRE(schema->has_attribute("a0")); + REQUIRE(schema->domain().has_dimension("d0")); + REQUIRE(soma_dense->ndim() == 1); + REQUIRE(soma_dense->shape() == std::vector{1001}); + soma_dense->close(); - auto array_buffer = std::make_shared(); - auto tdb_arr = std::make_shared( - *ctx->tiledb_ctx(), uri, TILEDB_READ); - array_buffer->emplace("a0", ColumnBuffer::create(tdb_arr, "a0", a0)); - array_buffer->emplace("d0", ColumnBuffer::create(tdb_arr, "d0", d0)); + std::vector d0{1, 10}; + std::vector a0(10, 1); - soma_dense->write(array_buffer); + soma_dense->open(OpenMode::write); + soma_dense->set_column_data("a0", a0.size(), a0.data()); + soma_dense->set_column_data("d0", d0.size(), d0.data()); + soma_dense->write(); soma_dense->close(); soma_dense->open(OpenMode::read); - REQUIRE(soma_dense->uri() == uri); - REQUIRE(soma_dense->ctx() == ctx); - REQUIRE(soma_dense->type() == "SOMADenseNDArray"); while (auto batch = soma_dense->read_next()) { auto arrbuf = batch.value(); auto d0span = arrbuf->at("d0")->data(); auto a0span = arrbuf->at("a0")->data(); - REQUIRE(d0 == std::vector(d0span.begin(), d0span.end())); + REQUIRE( + std::vector{1, 2, 3, 4, 5, 6, 7, 8, 9, 10} == + std::vector(d0span.begin(), d0span.end())); REQUIRE(a0 == std::vector(a0span.begin(), a0span.end())); } soma_dense->close(); diff --git a/libtiledbsoma/test/unit_soma_sparse_ndarray.cc b/libtiledbsoma/test/unit_soma_sparse_ndarray.cc index b9a37d5dc4..b8ac0a6075 100644 --- a/libtiledbsoma/test/unit_soma_sparse_ndarray.cc +++ b/libtiledbsoma/test/unit_soma_sparse_ndarray.cc @@ -80,27 +80,33 @@ TEST_CASE("SOMASparseNDArray: basic") { auto ctx = std::make_shared(); std::string uri = "mem://unit-test-sparse-ndarray-basic"; - auto soma_sparse = SOMASparseNDArray::create( - uri, create_schema(*ctx->tiledb_ctx()), ctx); + SOMASparseNDArray::create( + uri, create_schema(*ctx->tiledb_ctx()), ctx, TimestampRange(0, 2)); + + auto soma_sparse = SOMASparseNDArray::open(uri, OpenMode::read, ctx); + REQUIRE(soma_sparse->uri() == uri); + REQUIRE(soma_sparse->ctx() == ctx); + REQUIRE(soma_sparse->type() == "SOMASparseNDArray"); + REQUIRE(soma_sparse->is_sparse() == true); + auto schema = soma_sparse->tiledb_schema(); + REQUIRE(schema->has_attribute("a0")); + REQUIRE(schema->domain().has_dimension("d0")); + REQUIRE(soma_sparse->ndim() == 1); + REQUIRE(soma_sparse->nnz() == 0); + soma_sparse->close(); std::vector d0(10); for (int j = 0; j < 10; j++) d0[j] = j; std::vector a0(10, 1); - auto array_buffer = std::make_shared(); - auto tdb_arr = std::make_shared( - *ctx->tiledb_ctx(), uri, TILEDB_READ); - array_buffer->emplace("a0", ColumnBuffer::create(tdb_arr, "a0", a0)); - array_buffer->emplace("d0", ColumnBuffer::create(tdb_arr, "d0", d0)); - - soma_sparse->write(array_buffer); + soma_sparse->open(OpenMode::write); + soma_sparse->set_column_data("a0", a0.size(), a0.data()); + soma_sparse->set_column_data("d0", d0.size(), d0.data()); + soma_sparse->write(); soma_sparse->close(); soma_sparse->open(OpenMode::read); - REQUIRE(soma_sparse->uri() == uri); - REQUIRE(soma_sparse->ctx() == ctx); - REQUIRE(soma_sparse->type() == "SOMASparseNDArray"); while (auto batch = soma_sparse->read_next()) { auto arrbuf = batch.value(); auto d0span = arrbuf->at("d0")->data(); From 2641e03db475e7f4238de205dbb34caf065c30a2 Mon Sep 17 00:00:00 2001 From: Vivian Nguyen Date: Tue, 9 Apr 2024 14:06:12 -0500 Subject: [PATCH 04/10] Limit unit test to current changes only --- .../external/include/nanoarrow/nanoarrow.h | 1274 ++++++++++------- libtiledbsoma/src/soma/soma_collection.cc | 24 +- libtiledbsoma/src/soma/soma_collection.h | 17 +- libtiledbsoma/src/soma/soma_dataframe.cc | 18 +- libtiledbsoma/src/soma/soma_dataframe.h | 24 +- libtiledbsoma/src/soma/soma_dense_ndarray.cc | 13 +- libtiledbsoma/src/soma/soma_dense_ndarray.h | 19 +- libtiledbsoma/src/soma/soma_experiment.cc | 28 +- libtiledbsoma/src/soma/soma_experiment.h | 10 +- libtiledbsoma/src/soma/soma_measurement.cc | 35 +- libtiledbsoma/src/soma/soma_measurement.h | 10 +- libtiledbsoma/src/soma/soma_sparse_ndarray.cc | 12 +- libtiledbsoma/src/soma/soma_sparse_ndarray.h | 21 +- libtiledbsoma/test/CMakeLists.txt | 2 + libtiledbsoma/test/common.cc | 133 ++ libtiledbsoma/test/unit_soma_array.cc | 11 +- libtiledbsoma/test/unit_soma_dataframe.cc | 10 +- libtiledbsoma/test/unit_soma_dense_ndarray.cc | 10 +- .../test/unit_soma_sparse_ndarray.cc | 10 +- 19 files changed, 1015 insertions(+), 666 deletions(-) create mode 100644 libtiledbsoma/test/common.cc diff --git a/libtiledbsoma/src/external/include/nanoarrow/nanoarrow.h b/libtiledbsoma/src/external/include/nanoarrow/nanoarrow.h index e338560f1a..fa99f12f25 100644 --- a/libtiledbsoma/src/external/include/nanoarrow/nanoarrow.h +++ b/libtiledbsoma/src/external/include/nanoarrow/nanoarrow.h @@ -53,8 +53,6 @@ #include #include - - #if defined(NANOARROW_DEBUG) && !defined(NANOARROW_PRINT_AND_DIE) #include #include @@ -70,11 +68,11 @@ extern "C" { /// \defgroup nanoarrow-arrow-cdata Arrow C Data interface /// /// The Arrow C Data (https://arrow.apache.org/docs/format/CDataInterface.html) -/// and Arrow C Stream (https://arrow.apache.org/docs/format/CStreamInterface.html) -/// interfaces are part of the -/// Arrow Columnar Format specification -/// (https://arrow.apache.org/docs/format/Columnar.html). See the Arrow documentation for -/// documentation of these structures. +/// and Arrow C Stream +/// (https://arrow.apache.org/docs/format/CStreamInterface.html) interfaces are +/// part of the Arrow Columnar Format specification +/// (https://arrow.apache.org/docs/format/Columnar.html). See the Arrow +/// documentation for documentation of these structures. /// /// @{ @@ -129,7 +127,8 @@ struct ArrowArrayStream { // // Return value: 0 if successful, an `errno`-compatible error code otherwise. // - // If successful, the ArrowSchema must be released independently from the stream. + // If successful, the ArrowSchema must be released independently from the + // stream. int (*get_schema)(struct ArrowArrayStream*, struct ArrowSchema* out); // Callback to get the next array @@ -137,7 +136,8 @@ struct ArrowArrayStream { // // Return value: 0 if successful, an `errno`-compatible error code otherwise. // - // If successful, the ArrowArray must be released independently from the stream. + // If successful, the ArrowArray must be released independently from the + // stream. int (*get_next)(struct ArrowArrayStream*, struct ArrowArray* out); // Callback to get optional detailed error information. @@ -171,7 +171,8 @@ struct ArrowArrayStream { #define _NANOARROW_RETURN_NOT_OK_IMPL(NAME, EXPR) \ do { \ const int NAME = (EXPR); \ - if (NAME) return NAME; \ + if (NAME) \ + return NAME; \ } while (0) #define _NANOARROW_CHECK_RANGE(x_, min_, max_) \ @@ -181,29 +182,38 @@ struct ArrowArrayStream { NANOARROW_RETURN_NOT_OK((x_ <= max_) ? NANOARROW_OK : EINVAL) #if defined(NANOARROW_DEBUG) -#define _NANOARROW_RETURN_NOT_OK_WITH_ERROR_IMPL(NAME, EXPR, ERROR_PTR_EXPR, EXPR_STR) \ - do { \ - const int NAME = (EXPR); \ - if (NAME) { \ - ArrowErrorSet((ERROR_PTR_EXPR), "%s failed with errno %d\n* %s:%d", EXPR_STR, \ - NAME, __FILE__, __LINE__); \ - return NAME; \ - } \ +#define _NANOARROW_RETURN_NOT_OK_WITH_ERROR_IMPL( \ + NAME, EXPR, ERROR_PTR_EXPR, EXPR_STR) \ + do { \ + const int NAME = (EXPR); \ + if (NAME) { \ + ArrowErrorSet( \ + (ERROR_PTR_EXPR), \ + "%s failed with errno %d\n* %s:%d", \ + EXPR_STR, \ + NAME, \ + __FILE__, \ + __LINE__); \ + return NAME; \ + } \ } while (0) #else -#define _NANOARROW_RETURN_NOT_OK_WITH_ERROR_IMPL(NAME, EXPR, ERROR_PTR_EXPR, EXPR_STR) \ - do { \ - const int NAME = (EXPR); \ - if (NAME) { \ - ArrowErrorSet((ERROR_PTR_EXPR), "%s failed with errno %d", EXPR_STR, NAME); \ - return NAME; \ - } \ +#define _NANOARROW_RETURN_NOT_OK_WITH_ERROR_IMPL( \ + NAME, EXPR, ERROR_PTR_EXPR, EXPR_STR) \ + do { \ + const int NAME = (EXPR); \ + if (NAME) { \ + ArrowErrorSet( \ + (ERROR_PTR_EXPR), "%s failed with errno %d", EXPR_STR, NAME); \ + return NAME; \ + } \ } while (0) #endif #if defined(NANOARROW_DEBUG) // For checking ArrowErrorSet() calls for valid printf format strings/arguments -// If using mingw's c99-compliant printf, we need a different format-checking attribute +// If using mingw's c99-compliant printf, we need a different format-checking +// attribute #if defined(__USE_MINGW_ANSI_STDIO) && defined(__MINGW_PRINTF_FORMAT) #define NANOARROW_CHECK_PRINTF_ATTRIBUTE \ __attribute__((format(__MINGW_PRINTF_FORMAT, 2, 3))) @@ -243,8 +253,9 @@ typedef int ArrowErrorCode; /// \brief Flags supported by ArrowSchemaViewInit() /// \ingroup nanoarrow-schema-view -#define NANOARROW_FLAG_ALL_SUPPORTED \ - (ARROW_FLAG_DICTIONARY_ORDERED | ARROW_FLAG_NULLABLE | ARROW_FLAG_MAP_KEYS_SORTED) +#define NANOARROW_FLAG_ALL_SUPPORTED \ + (ARROW_FLAG_DICTIONARY_ORDERED | ARROW_FLAG_NULLABLE | \ + ARROW_FLAG_MAP_KEYS_SORTED) /// \brief Error type containing a UTF-8 encoded message. /// \ingroup nanoarrow-errors @@ -253,8 +264,8 @@ struct ArrowError { char message[1024]; }; -/// \brief Ensure an ArrowError is null-terminated by zeroing the first character. -/// \ingroup nanoarrow-errors +/// \brief Ensure an ArrowError is null-terminated by zeroing the first +/// character. \ingroup nanoarrow-errors /// /// If error is NULL, this function does nothing. static inline void ArrowErrorInit(struct ArrowError* error) { @@ -280,7 +291,8 @@ static inline const char* ArrowErrorMessage(struct ArrowError* error) { /// \ingroup nanoarrow-errors /// /// If error is NULL, this function does nothing. -static inline void ArrowErrorSetString(struct ArrowError* error, const char* src) { +static inline void ArrowErrorSetString( + struct ArrowError* error, const char* src) { if (error == NULL) { return; } @@ -298,25 +310,34 @@ static inline void ArrowErrorSetString(struct ArrowError* error, const char* src /// \brief Check the result of an expression and return it if not NANOARROW_OK /// \ingroup nanoarrow-errors #define NANOARROW_RETURN_NOT_OK(EXPR) \ - _NANOARROW_RETURN_NOT_OK_IMPL(_NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), EXPR) + _NANOARROW_RETURN_NOT_OK_IMPL( \ + _NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), EXPR) /// \brief Check the result of an expression and return it if not NANOARROW_OK, /// adding an auto-generated message to an ArrowError. /// \ingroup nanoarrow-errors /// /// This macro is used to ensure that functions that accept an ArrowError -/// as input always set its message when returning an error code (e.g., when calling -/// a nanoarrow function that does *not* accept ArrowError). +/// as input always set its message when returning an error code (e.g., when +/// calling a nanoarrow function that does *not* accept ArrowError). #define NANOARROW_RETURN_NOT_OK_WITH_ERROR(EXPR, ERROR_EXPR) \ _NANOARROW_RETURN_NOT_OK_WITH_ERROR_IMPL( \ - _NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), EXPR, ERROR_EXPR, #EXPR) + _NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), \ + EXPR, \ + ERROR_EXPR, \ + #EXPR) #if defined(NANOARROW_DEBUG) && !defined(NANOARROW_PRINT_AND_DIE) -#define NANOARROW_PRINT_AND_DIE(VALUE, EXPR_STR) \ - do { \ - fprintf(stderr, "%s failed with code %d\n* %s:%d\n", EXPR_STR, (int)(VALUE), \ - __FILE__, (int)__LINE__); \ - abort(); \ +#define NANOARROW_PRINT_AND_DIE(VALUE, EXPR_STR) \ + do { \ + fprintf( \ + stderr, \ + "%s failed with code %d\n* %s:%d\n", \ + EXPR_STR, \ + (int)(VALUE), \ + __FILE__, \ + (int)__LINE__); \ + abort(); \ } while (0) #endif @@ -324,23 +345,27 @@ static inline void ArrowErrorSetString(struct ArrowError* error, const char* src #define _NANOARROW_ASSERT_OK_IMPL(NAME, EXPR, EXPR_STR) \ do { \ const int NAME = (EXPR); \ - if (NAME) NANOARROW_PRINT_AND_DIE(NAME, EXPR_STR); \ + if (NAME) \ + NANOARROW_PRINT_AND_DIE(NAME, EXPR_STR); \ } while (0) /// \brief Assert that an expression's value is NANOARROW_OK /// \ingroup nanoarrow-errors /// -/// If nanoarrow was built in debug mode (i.e., defined(NANOARROW_DEBUG) is true), -/// print a message to stderr and abort. If nanoarrow was built in release mode, -/// this statement has no effect. You can customize fatal error behaviour -/// be defining the NANOARROW_PRINT_AND_DIE macro before including nanoarrow.h -/// This macro is provided as a convenience for users and is not used internally. +/// If nanoarrow was built in debug mode (i.e., defined(NANOARROW_DEBUG) is +/// true), print a message to stderr and abort. If nanoarrow was built in +/// release mode, this statement has no effect. You can customize fatal error +/// behaviour be defining the NANOARROW_PRINT_AND_DIE macro before including +/// nanoarrow.h This macro is provided as a convenience for users and is not +/// used internally. #define NANOARROW_ASSERT_OK(EXPR) \ - _NANOARROW_ASSERT_OK_IMPL(_NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), EXPR, #EXPR) + _NANOARROW_ASSERT_OK_IMPL( \ + _NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), EXPR, #EXPR) -#define _NANOARROW_DCHECK_IMPL(EXPR, EXPR_STR) \ - do { \ - if (!(EXPR)) NANOARROW_PRINT_AND_DIE(-1, EXPR_STR); \ +#define _NANOARROW_DCHECK_IMPL(EXPR, EXPR_STR) \ + do { \ + if (!(EXPR)) \ + NANOARROW_PRINT_AND_DIE(-1, EXPR_STR); \ } while (0) #define NANOARROW_DCHECK(EXPR) _NANOARROW_DCHECK_IMPL(EXPR, #EXPR) @@ -349,7 +374,8 @@ static inline void ArrowErrorSetString(struct ArrowError* error, const char* src #define NANOARROW_DCHECK(EXPR) #endif -static inline void ArrowSchemaMove(struct ArrowSchema* src, struct ArrowSchema* dst) { +static inline void ArrowSchemaMove( + struct ArrowSchema* src, struct ArrowSchema* dst) { NANOARROW_DCHECK(src != NULL); NANOARROW_DCHECK(dst != NULL); @@ -363,7 +389,8 @@ static inline void ArrowSchemaRelease(struct ArrowSchema* schema) { NANOARROW_DCHECK(schema->release == NULL); } -static inline void ArrowArrayMove(struct ArrowArray* src, struct ArrowArray* dst) { +static inline void ArrowArrayMove( + struct ArrowArray* src, struct ArrowArray* dst) { NANOARROW_DCHECK(src != NULL); NANOARROW_DCHECK(dst != NULL); @@ -377,8 +404,8 @@ static inline void ArrowArrayRelease(struct ArrowArray* array) { NANOARROW_DCHECK(array->release == NULL); } -static inline void ArrowArrayStreamMove(struct ArrowArrayStream* src, - struct ArrowArrayStream* dst) { +static inline void ArrowArrayStreamMove( + struct ArrowArrayStream* src, struct ArrowArrayStream* dst) { NANOARROW_DCHECK(src != NULL); NANOARROW_DCHECK(dst != NULL); @@ -399,7 +426,8 @@ static inline const char* ArrowArrayStreamGetLastError( } static inline ArrowErrorCode ArrowArrayStreamGetSchema( - struct ArrowArrayStream* array_stream, struct ArrowSchema* out, + struct ArrowArrayStream* array_stream, + struct ArrowSchema* out, struct ArrowError* error) { NANOARROW_DCHECK(array_stream != NULL); @@ -412,7 +440,8 @@ static inline ArrowErrorCode ArrowArrayStreamGetSchema( } static inline ArrowErrorCode ArrowArrayStreamGetNext( - struct ArrowArrayStream* array_stream, struct ArrowArray* out, + struct ArrowArrayStream* array_stream, + struct ArrowArray* out, struct ArrowError* error) { NANOARROW_DCHECK(array_stream != NULL); @@ -424,7 +453,8 @@ static inline ArrowErrorCode ArrowArrayStreamGetNext( return result; } -static inline void ArrowArrayStreamRelease(struct ArrowArrayStream* array_stream) { +static inline void ArrowArrayStreamRelease( + struct ArrowArrayStream* array_stream) { NANOARROW_DCHECK(array_stream != NULL); array_stream->release(array_stream); NANOARROW_DCHECK(array_stream->release == NULL); @@ -592,16 +622,21 @@ enum ArrowValidationLevel { /// \brief Do not validate buffer sizes or content. NANOARROW_VALIDATION_LEVEL_NONE = 0, - /// \brief Validate buffer sizes that depend on array length but do not validate buffer + /// \brief Validate buffer sizes that depend on array length but do not + /// validate buffer /// sizes that depend on buffer data access. NANOARROW_VALIDATION_LEVEL_MINIMAL = 1, - /// \brief Validate all buffer sizes, including those that require buffer data access, - /// but do not perform any checks that are O(1) along the length of the buffers. + /// \brief Validate all buffer sizes, including those that require buffer data + /// access, + /// but do not perform any checks that are O(1) along the length of the + /// buffers. NANOARROW_VALIDATION_LEVEL_DEFAULT = 2, - /// \brief Validate all buffer sizes and all buffer content. This is useful in the - /// context of untrusted input or input that may have been corrupted in transit. + /// \brief Validate all buffer sizes and all buffer content. This is useful in + /// the + /// context of untrusted input or input that may have been corrupted in + /// transit. NANOARROW_VALIDATION_LEVEL_FULL = 3 }; @@ -626,8 +661,8 @@ static inline const char* ArrowTimeUnitString(enum ArrowTimeUnit time_unit) { } } -/// \brief Functional types of buffers as described in the Arrow Columnar Specification -/// \ingroup nanoarrow-array-view +/// \brief Functional types of buffers as described in the Arrow Columnar +/// Specification \ingroup nanoarrow-array-view enum ArrowBufferType { NANOARROW_BUFFER_TYPE_NONE, NANOARROW_BUFFER_TYPE_VALIDITY, @@ -642,7 +677,8 @@ enum ArrowBufferType { /// /// All currently supported types have 3 buffers or fewer; however, future types /// may involve a variable number of buffers (e.g., string view). These buffers -/// will be represented by separate members of the ArrowArrayView or ArrowLayout. +/// will be represented by separate members of the ArrowArrayView or +/// ArrowLayout. #define NANOARROW_MAX_FIXED_BUFFERS 3 /// \brief An non-owning view of a string @@ -711,11 +747,15 @@ struct ArrowBufferView { /// an ArrowArray. struct ArrowBufferAllocator { /// \brief Reallocate a buffer or return NULL if it cannot be reallocated - uint8_t* (*reallocate)(struct ArrowBufferAllocator* allocator, uint8_t* ptr, - int64_t old_size, int64_t new_size); + uint8_t* (*reallocate)( + struct ArrowBufferAllocator* allocator, + uint8_t* ptr, + int64_t old_size, + int64_t new_size); /// \brief Deallocate a buffer allocated by this allocator - void (*free)(struct ArrowBufferAllocator* allocator, uint8_t* ptr, int64_t size); + void (*free)( + struct ArrowBufferAllocator* allocator, uint8_t* ptr, int64_t size); /// \brief Opaque data specific to the allocator void* private_data; @@ -735,7 +775,8 @@ struct ArrowBuffer { /// \brief The capacity of the buffer in bytes int64_t capacity_bytes; - /// \brief The allocator that will be used to reallocate and/or free the buffer + /// \brief The allocator that will be used to reallocate and/or free the + /// buffer struct ArrowBufferAllocator allocator; }; @@ -762,11 +803,12 @@ struct ArrowLayout { /// \brief The data type of each buffer enum ArrowType buffer_data_type[NANOARROW_MAX_FIXED_BUFFERS]; - /// \brief The size of an element each buffer or 0 if this size is variable or unknown + /// \brief The size of an element each buffer or 0 if this size is variable or + /// unknown int64_t element_size_bits[NANOARROW_MAX_FIXED_BUFFERS]; - /// \brief The number of elements in the child array per element in this array for a - /// fixed-size list + /// \brief The number of elements in the child array per element in this array + /// for a fixed-size list int64_t child_size_elements; }; @@ -869,8 +911,8 @@ struct ArrowInterval { /// \brief Zero initialize an Interval with a given unit /// \ingroup nanoarrow-utils -static inline void ArrowIntervalInit(struct ArrowInterval* interval, - enum ArrowType type) { +static inline void ArrowIntervalInit( + struct ArrowInterval* interval, enum ArrowType type) { memset(interval, 0, sizeof(struct ArrowInterval)); interval->type = type; } @@ -882,7 +924,8 @@ static inline void ArrowIntervalInit(struct ArrowInterval* interval, /// values set using ArrowDecimalSetInt(), ArrowDecimalSetBytes128(), /// or ArrowDecimalSetBytes256(). struct ArrowDecimal { - /// \brief An array of 64-bit integers of n_words length defined in native-endian order + /// \brief An array of 64-bit integers of n_words length defined in + /// native-endian order uint64_t words[4]; /// \brief The number of significant digits this decimal number can represent @@ -903,8 +946,11 @@ struct ArrowDecimal { /// \brief Initialize a decimal with a given set of type parameters /// \ingroup nanoarrow-utils -static inline void ArrowDecimalInit(struct ArrowDecimal* decimal, int32_t bitwidth, - int32_t precision, int32_t scale) { +static inline void ArrowDecimalInit( + struct ArrowDecimal* decimal, + int32_t bitwidth, + int32_t precision, + int32_t scale) { memset(decimal->words, 0, sizeof(decimal->words)); decimal->precision = precision; decimal->scale = scale; @@ -924,14 +970,15 @@ static inline void ArrowDecimalInit(struct ArrowDecimal* decimal, int32_t bitwid /// This does not check if the decimal's precision sufficiently small to fit /// within the signed 64-bit integer range (A precision less than or equal /// to 18 is sufficiently small). -static inline int64_t ArrowDecimalGetIntUnsafe(const struct ArrowDecimal* decimal) { +static inline int64_t ArrowDecimalGetIntUnsafe( + const struct ArrowDecimal* decimal) { return (int64_t)decimal->words[decimal->low_word_index]; } /// \brief Copy the bytes of this decimal into a sufficiently large buffer /// \ingroup nanoarrow-utils -static inline void ArrowDecimalGetBytes(const struct ArrowDecimal* decimal, - uint8_t* out) { +static inline void ArrowDecimalGetBytes( + const struct ArrowDecimal* decimal, uint8_t* out) { memcpy(out, decimal->words, decimal->n_words * sizeof(uint64_t)); } @@ -943,7 +990,8 @@ static inline int64_t ArrowDecimalSign(const struct ArrowDecimal* decimal) { /// \brief Sets the integer value of this decimal /// \ingroup nanoarrow-utils -static inline void ArrowDecimalSetInt(struct ArrowDecimal* decimal, int64_t value) { +static inline void ArrowDecimalSetInt( + struct ArrowDecimal* decimal, int64_t value) { if (value < 0) { memset(decimal->words, 0xff, decimal->n_words * sizeof(uint64_t)); } else { @@ -977,8 +1025,8 @@ static inline void ArrowDecimalNegate(struct ArrowDecimal* decimal) { /// \brief Copy bytes from a buffer into this decimal /// \ingroup nanoarrow-utils -static inline void ArrowDecimalSetBytes(struct ArrowDecimal* decimal, - const uint8_t* value) { +static inline void ArrowDecimalSetBytes( + struct ArrowDecimal* decimal, const uint8_t* value) { memcpy(decimal->words, value, decimal->n_words * sizeof(uint64_t)); } @@ -1011,11 +1059,9 @@ static inline void ArrowDecimalSetBytes(struct ArrowDecimal* decimal, #include #include - - -// If using CMake, optionally pass -DNANOARROW_NAMESPACE=MyNamespace which will set this -// define in nanoarrow_config.h. If not, you can optionally #define NANOARROW_NAMESPACE -// MyNamespace here. +// If using CMake, optionally pass -DNANOARROW_NAMESPACE=MyNamespace which will +// set this define in nanoarrow_config.h. If not, you can optionally #define +// NANOARROW_NAMESPACE MyNamespace here. // This section remaps the non-prefixed symbols to the prefixed symbols so that // code written against this build can be used independent of the value of @@ -1024,7 +1070,8 @@ static inline void ArrowDecimalSetBytes(struct ArrowDecimal* decimal, #define NANOARROW_CAT(A, B) A##B #define NANOARROW_SYMBOL(A, B) NANOARROW_CAT(A, B) -#define ArrowNanoarrowVersion NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowNanoarrowVersion) +#define ArrowNanoarrowVersion \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowNanoarrowVersion) #define ArrowNanoarrowVersionInt \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowNanoarrowVersionInt) #define ArrowMalloc NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMalloc) @@ -1036,13 +1083,15 @@ static inline void ArrowDecimalSetBytes(struct ArrowDecimal* decimal, NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowBufferDeallocator) #define ArrowErrorSet NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowErrorSet) #define ArrowLayoutInit NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowLayoutInit) -#define ArrowDecimalSetDigits NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowDecimalSetDigits) +#define ArrowDecimalSetDigits \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowDecimalSetDigits) #define ArrowDecimalAppendDigitsToBuffer \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowDecimalAppendDigitsToBuffer) #define ArrowSchemaInit NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaInit) #define ArrowSchemaInitFromType \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaInitFromType) -#define ArrowSchemaSetType NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetType) +#define ArrowSchemaSetType \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetType) #define ArrowSchemaSetTypeStruct \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetTypeStruct) #define ArrowSchemaSetTypeFixedSize \ @@ -1053,9 +1102,12 @@ static inline void ArrowDecimalSetBytes(struct ArrowDecimal* decimal, NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetTypeDateTime) #define ArrowSchemaSetTypeUnion \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetTypeUnion) -#define ArrowSchemaDeepCopy NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaDeepCopy) -#define ArrowSchemaSetFormat NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetFormat) -#define ArrowSchemaSetName NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetName) +#define ArrowSchemaDeepCopy \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaDeepCopy) +#define ArrowSchemaSetFormat \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetFormat) +#define ArrowSchemaSetName \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetName) #define ArrowSchemaSetMetadata \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetMetadata) #define ArrowSchemaAllocateChildren \ @@ -1066,9 +1118,12 @@ static inline void ArrowDecimalSetBytes(struct ArrowDecimal* decimal, NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataReaderInit) #define ArrowMetadataReaderRead \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataReaderRead) -#define ArrowMetadataSizeOf NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataSizeOf) -#define ArrowMetadataHasKey NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataHasKey) -#define ArrowMetadataGetValue NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataGetValue) +#define ArrowMetadataSizeOf \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataSizeOf) +#define ArrowMetadataHasKey \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataHasKey) +#define ArrowMetadataGetValue \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataGetValue) #define ArrowMetadataBuilderInit \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataBuilderInit) #define ArrowMetadataBuilderAppend \ @@ -1077,8 +1132,10 @@ static inline void ArrowDecimalSetBytes(struct ArrowDecimal* decimal, NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataBuilderSet) #define ArrowMetadataBuilderRemove \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataBuilderRemove) -#define ArrowSchemaViewInit NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaViewInit) -#define ArrowSchemaToString NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaToString) +#define ArrowSchemaViewInit \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaViewInit) +#define ArrowSchemaToString \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaToString) #define ArrowArrayInitFromType \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayInitFromType) #define ArrowArrayInitFromSchema \ @@ -1093,8 +1150,10 @@ static inline void ArrowDecimalSetBytes(struct ArrowDecimal* decimal, NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayAllocateDictionary) #define ArrowArraySetValidityBitmap \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArraySetValidityBitmap) -#define ArrowArraySetBuffer NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArraySetBuffer) -#define ArrowArrayReserve NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayReserve) +#define ArrowArraySetBuffer \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArraySetBuffer) +#define ArrowArrayReserve \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayReserve) #define ArrowArrayFinishBuilding \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayFinishBuilding) #define ArrowArrayFinishBuildingDefault \ @@ -1115,7 +1174,8 @@ static inline void ArrowDecimalSetBytes(struct ArrowDecimal* decimal, NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewSetArrayMinimal) #define ArrowArrayViewValidate \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewValidate) -#define ArrowArrayViewReset NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewReset) +#define ArrowArrayViewReset \ + NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewReset) #define ArrowBasicArrayStreamInit \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowBasicArrayStreamInit) #define ArrowBasicArrayStreamSetArray \ @@ -1169,31 +1229,33 @@ struct ArrowBufferAllocator ArrowBufferAllocatorDefault(void); /// avoid copying an existing buffer that was not allocated using the /// infrastructure provided here (e.g., by an R or Python object). struct ArrowBufferAllocator ArrowBufferDeallocator( - void (*custom_free)(struct ArrowBufferAllocator* allocator, uint8_t* ptr, - int64_t size), + void (*custom_free)( + struct ArrowBufferAllocator* allocator, uint8_t* ptr, int64_t size), void* private_data); /// @} -/// \brief Move the contents of an src ArrowSchema into dst and set src->release to NULL -/// \ingroup nanoarrow-arrow-cdata -static inline void ArrowSchemaMove(struct ArrowSchema* src, struct ArrowSchema* dst); +/// \brief Move the contents of an src ArrowSchema into dst and set src->release +/// to NULL \ingroup nanoarrow-arrow-cdata +static inline void ArrowSchemaMove( + struct ArrowSchema* src, struct ArrowSchema* dst); /// \brief Call the release callback of an ArrowSchema /// \ingroup nanoarrow-arrow-cdata static inline void ArrowSchemaRelease(struct ArrowSchema* schema); -/// \brief Move the contents of an src ArrowArray into dst and set src->release to NULL -/// \ingroup nanoarrow-arrow-cdata -static inline void ArrowArrayMove(struct ArrowArray* src, struct ArrowArray* dst); +/// \brief Move the contents of an src ArrowArray into dst and set src->release +/// to NULL \ingroup nanoarrow-arrow-cdata +static inline void ArrowArrayMove( + struct ArrowArray* src, struct ArrowArray* dst); /// \brief Call the release callback of an ArrowArray static inline void ArrowArrayRelease(struct ArrowArray* array); -/// \brief Move the contents of an src ArrowArrayStream into dst and set src->release to -/// NULL \ingroup nanoarrow-arrow-cdata -static inline void ArrowArrayStreamMove(struct ArrowArrayStream* src, - struct ArrowArrayStream* dst); +/// \brief Move the contents of an src ArrowArrayStream into dst and set +/// src->release to NULL \ingroup nanoarrow-arrow-cdata +static inline void ArrowArrayStreamMove( + struct ArrowArrayStream* src, struct ArrowArrayStream* dst); /// \brief Call the get_schema callback of an ArrowArrayStream /// \ingroup nanoarrow-arrow-cdata @@ -1203,7 +1265,8 @@ static inline void ArrowArrayStreamMove(struct ArrowArrayStream* src, /// makes it significantly less verbose to iterate over array streams /// using NANOARROW_RETURN_NOT_OK()-style error handling. static inline ArrowErrorCode ArrowArrayStreamGetSchema( - struct ArrowArrayStream* array_stream, struct ArrowSchema* out, + struct ArrowArrayStream* array_stream, + struct ArrowSchema* out, struct ArrowError* error); /// \brief Call the get_schema callback of an ArrowArrayStream @@ -1214,20 +1277,22 @@ static inline ArrowErrorCode ArrowArrayStreamGetSchema( /// makes it significantly less verbose to iterate over array streams /// using NANOARROW_RETURN_NOT_OK()-style error handling. static inline ArrowErrorCode ArrowArrayStreamGetNext( - struct ArrowArrayStream* array_stream, struct ArrowArray* out, + struct ArrowArrayStream* array_stream, + struct ArrowArray* out, struct ArrowError* error); /// \brief Call the get_next callback of an ArrowArrayStream /// \ingroup nanoarrow-arrow-cdata /// /// Unlike the get_next callback, this function never returns NULL (i.e., its -/// result is safe to use in printf-style error formatters). Null values from the -/// original callback are reported as "". +/// result is safe to use in printf-style error formatters). Null values from +/// the original callback are reported as "". static inline const char* ArrowArrayStreamGetLastError( struct ArrowArrayStream* array_stream); /// \brief Call the release callback of an ArrowArrayStream -static inline void ArrowArrayStreamRelease(struct ArrowArrayStream* array_stream); +static inline void ArrowArrayStreamRelease( + struct ArrowArrayStream* array_stream); /// \defgroup nanoarrow-errors Error handling /// @@ -1235,24 +1300,24 @@ static inline void ArrowArrayStreamRelease(struct ArrowArrayStream* array_stream /// need to communicate more verbose error information accept a pointer /// to an ArrowError. This can be stack or statically allocated. The /// content of the message is undefined unless an error code has been -/// returned. If a nanoarrow function is passed a non-null ArrowError pointer, the -/// ArrowError pointed to by the argument will be propagated with a +/// returned. If a nanoarrow function is passed a non-null ArrowError pointer, +/// the ArrowError pointed to by the argument will be propagated with a /// null-terminated error message. It is safe to pass a NULL ArrowError anywhere /// in the nanoarrow API. /// /// Except where documented, it is generally not safe to continue after a -/// function has returned a non-zero ArrowErrorCode. The NANOARROW_RETURN_NOT_OK and -/// NANOARROW_ASSERT_OK macros are provided to help propagate errors. C++ clients can use -/// the helpers provided in the nanoarrow.hpp header to facilitate using C++ idioms -/// for memory management and error propgagtion. +/// function has returned a non-zero ArrowErrorCode. The NANOARROW_RETURN_NOT_OK +/// and NANOARROW_ASSERT_OK macros are provided to help propagate errors. C++ +/// clients can use the helpers provided in the nanoarrow.hpp header to +/// facilitate using C++ idioms for memory management and error propgagtion. /// /// @{ /// \brief Set the contents of an error using printf syntax. /// /// If error is NULL, this function does nothing and returns NANOARROW_OK. -NANOARROW_CHECK_PRINTF_ATTRIBUTE int ArrowErrorSet(struct ArrowError* error, - const char* fmt, ...); +NANOARROW_CHECK_PRINTF_ATTRIBUTE int ArrowErrorSet( + struct ArrowError* error, const char* fmt, ...); /// @} @@ -1273,12 +1338,12 @@ void ArrowLayoutInit(struct ArrowLayout* layout, enum ArrowType storage_type); static inline struct ArrowStringView ArrowCharView(const char* value); /// \brief Sets the integer value of an ArrowDecimal from a string -ArrowErrorCode ArrowDecimalSetDigits(struct ArrowDecimal* decimal, - struct ArrowStringView value); +ArrowErrorCode ArrowDecimalSetDigits( + struct ArrowDecimal* decimal, struct ArrowStringView value); /// \brief Get the integer value of an ArrowDecimal as string -ArrowErrorCode ArrowDecimalAppendDigitsToBuffer(const struct ArrowDecimal* decimal, - struct ArrowBuffer* buffer); +ArrowErrorCode ArrowDecimalAppendDigitsToBuffer( + const struct ArrowDecimal* decimal, struct ArrowBuffer* buffer); /// @} @@ -1299,9 +1364,10 @@ void ArrowSchemaInit(struct ArrowSchema* schema); /// /// A convenience constructor for that calls ArrowSchemaInit() and /// ArrowSchemaSetType() for the common case of constructing an -/// unparameterized type. The caller is responsible for calling the schema->release -/// callback if NANOARROW_OK is returned. -ArrowErrorCode ArrowSchemaInitFromType(struct ArrowSchema* schema, enum ArrowType type); +/// unparameterized type. The caller is responsible for calling the +/// schema->release callback if NANOARROW_OK is returned. +ArrowErrorCode ArrowSchemaInitFromType( + struct ArrowSchema* schema, enum ArrowType type); /// \brief Get a human-readable summary of a Schema /// @@ -1309,8 +1375,8 @@ ArrowErrorCode ArrowSchemaInitFromType(struct ArrowSchema* schema, enum ArrowTyp /// and returns the number of characters required for the output if /// n were sufficiently large. If recursive is non-zero, the result will /// also include children. -int64_t ArrowSchemaToString(const struct ArrowSchema* schema, char* out, int64_t n, - char recursive); +int64_t ArrowSchemaToString( + const struct ArrowSchema* schema, char* out, int64_t n, char recursive); /// \brief Set the format field of a schema from an ArrowType /// @@ -1318,16 +1384,19 @@ int64_t ArrowSchemaToString(const struct ArrowSchema* schema, char* out, int64_t /// NANOARROW_TYPE_LIST, NANOARROW_TYPE_LARGE_LIST, and /// NANOARROW_TYPE_MAP, the appropriate number of children are /// allocated, initialized, and named; however, the caller must -/// ArrowSchemaSetType() on the preinitialized children. Schema must have been initialized -/// using ArrowSchemaInit() or ArrowSchemaDeepCopy(). -ArrowErrorCode ArrowSchemaSetType(struct ArrowSchema* schema, enum ArrowType type); +/// ArrowSchemaSetType() on the preinitialized children. Schema must have been +/// initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy(). +ArrowErrorCode ArrowSchemaSetType( + struct ArrowSchema* schema, enum ArrowType type); /// \brief Set the format field and initialize children of a struct schema /// -/// The specified number of children are initialized; however, the caller is responsible -/// for calling ArrowSchemaSetType() and ArrowSchemaSetName() on each child. -/// Schema must have been initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy(). -ArrowErrorCode ArrowSchemaSetTypeStruct(struct ArrowSchema* schema, int64_t n_children); +/// The specified number of children are initialized; however, the caller is +/// responsible for calling ArrowSchemaSetType() and ArrowSchemaSetName() on +/// each child. Schema must have been initialized using ArrowSchemaInit() or +/// ArrowSchemaDeepCopy(). +ArrowErrorCode ArrowSchemaSetTypeStruct( + struct ArrowSchema* schema, int64_t n_children); /// \brief Set the format field of a fixed-size schema /// @@ -1335,50 +1404,55 @@ ArrowErrorCode ArrowSchemaSetTypeStruct(struct ArrowSchema* schema, int64_t n_ch /// NANOARROW_TYPE_FIXED_SIZE_BINARY or NANOARROW_TYPE_FIXED_SIZE_LIST. /// For NANOARROW_TYPE_FIXED_SIZE_LIST, the appropriate number of children are /// allocated, initialized, and named; however, the caller must -/// ArrowSchemaSetType() the first child. Schema must have been initialized using -/// ArrowSchemaInit() or ArrowSchemaDeepCopy(). -ArrowErrorCode ArrowSchemaSetTypeFixedSize(struct ArrowSchema* schema, - enum ArrowType type, int32_t fixed_size); +/// ArrowSchemaSetType() the first child. Schema must have been initialized +/// using ArrowSchemaInit() or ArrowSchemaDeepCopy(). +ArrowErrorCode ArrowSchemaSetTypeFixedSize( + struct ArrowSchema* schema, enum ArrowType type, int32_t fixed_size); /// \brief Set the format field of a decimal schema /// /// Returns EINVAL for scale <= 0 or for type that is not -/// NANOARROW_TYPE_DECIMAL128 or NANOARROW_TYPE_DECIMAL256. Schema must have been -/// initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy(). -ArrowErrorCode ArrowSchemaSetTypeDecimal(struct ArrowSchema* schema, enum ArrowType type, - int32_t decimal_precision, - int32_t decimal_scale); +/// NANOARROW_TYPE_DECIMAL128 or NANOARROW_TYPE_DECIMAL256. Schema must have +/// been initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy(). +ArrowErrorCode ArrowSchemaSetTypeDecimal( + struct ArrowSchema* schema, + enum ArrowType type, + int32_t decimal_precision, + int32_t decimal_scale); /// \brief Set the format field of a time, timestamp, or duration schema /// /// Returns EINVAL for type that is not /// NANOARROW_TYPE_TIME32, NANOARROW_TYPE_TIME64, /// NANOARROW_TYPE_TIMESTAMP, or NANOARROW_TYPE_DURATION. The -/// timezone parameter must be NULL for a non-timestamp type. Schema must have been -/// initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy(). -ArrowErrorCode ArrowSchemaSetTypeDateTime(struct ArrowSchema* schema, enum ArrowType type, - enum ArrowTimeUnit time_unit, - const char* timezone); +/// timezone parameter must be NULL for a non-timestamp type. Schema must have +/// been initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy(). +ArrowErrorCode ArrowSchemaSetTypeDateTime( + struct ArrowSchema* schema, + enum ArrowType type, + enum ArrowTimeUnit time_unit, + const char* timezone); /// \brief Seet the format field of a union schema /// /// Returns EINVAL for a type that is not NANOARROW_TYPE_DENSE_UNION /// or NANOARROW_TYPE_SPARSE_UNION. The specified number of children are /// allocated, and initialized. -ArrowErrorCode ArrowSchemaSetTypeUnion(struct ArrowSchema* schema, enum ArrowType type, - int64_t n_children); +ArrowErrorCode ArrowSchemaSetTypeUnion( + struct ArrowSchema* schema, enum ArrowType type, int64_t n_children); /// \brief Make a (recursive) copy of a schema /// /// Allocates and copies fields of schema into schema_out. -ArrowErrorCode ArrowSchemaDeepCopy(const struct ArrowSchema* schema, - struct ArrowSchema* schema_out); +ArrowErrorCode ArrowSchemaDeepCopy( + const struct ArrowSchema* schema, struct ArrowSchema* schema_out); /// \brief Copy format into schema->format /// /// schema must have been allocated using ArrowSchemaInitFromType() or /// ArrowSchemaDeepCopy(). -ArrowErrorCode ArrowSchemaSetFormat(struct ArrowSchema* schema, const char* format); +ArrowErrorCode ArrowSchemaSetFormat( + struct ArrowSchema* schema, const char* format); /// \brief Copy name into schema->name /// @@ -1390,15 +1464,16 @@ ArrowErrorCode ArrowSchemaSetName(struct ArrowSchema* schema, const char* name); /// /// schema must have been allocated using ArrowSchemaInitFromType() or /// ArrowSchemaDeepCopy. -ArrowErrorCode ArrowSchemaSetMetadata(struct ArrowSchema* schema, const char* metadata); +ArrowErrorCode ArrowSchemaSetMetadata( + struct ArrowSchema* schema, const char* metadata); /// \brief Allocate the schema->children array /// /// Includes the memory for each child struct ArrowSchema. /// schema must have been allocated using ArrowSchemaInitFromType() or /// ArrowSchemaDeepCopy(). -ArrowErrorCode ArrowSchemaAllocateChildren(struct ArrowSchema* schema, - int64_t n_children); +ArrowErrorCode ArrowSchemaAllocateChildren( + struct ArrowSchema* schema, int64_t n_children); /// \brief Allocate the schema->dictionary member /// @@ -1428,13 +1503,14 @@ struct ArrowMetadataReader { }; /// \brief Initialize an ArrowMetadataReader -ArrowErrorCode ArrowMetadataReaderInit(struct ArrowMetadataReader* reader, - const char* metadata); +ArrowErrorCode ArrowMetadataReaderInit( + struct ArrowMetadataReader* reader, const char* metadata); /// \brief Read the next key/value pair from an ArrowMetadataReader -ArrowErrorCode ArrowMetadataReaderRead(struct ArrowMetadataReader* reader, - struct ArrowStringView* key_out, - struct ArrowStringView* value_out); +ArrowErrorCode ArrowMetadataReaderRead( + struct ArrowMetadataReader* reader, + struct ArrowStringView* key_out, + struct ArrowStringView* value_out); /// \brief The number of bytes in in a key/value metadata string int64_t ArrowMetadataSizeOf(const char* metadata); @@ -1445,32 +1521,37 @@ char ArrowMetadataHasKey(const char* metadata, struct ArrowStringView key); /// \brief Extract a value from schema metadata /// /// If key does not exist in metadata, value_out is unmodified -ArrowErrorCode ArrowMetadataGetValue(const char* metadata, struct ArrowStringView key, - struct ArrowStringView* value_out); +ArrowErrorCode ArrowMetadataGetValue( + const char* metadata, + struct ArrowStringView key, + struct ArrowStringView* value_out); /// \brief Initialize a builder for schema metadata from key/value pairs /// /// metadata can be an existing metadata string or NULL to initialize /// an empty metadata string. -ArrowErrorCode ArrowMetadataBuilderInit(struct ArrowBuffer* buffer, const char* metadata); +ArrowErrorCode ArrowMetadataBuilderInit( + struct ArrowBuffer* buffer, const char* metadata); /// \brief Append a key/value pair to a buffer containing serialized metadata -ArrowErrorCode ArrowMetadataBuilderAppend(struct ArrowBuffer* buffer, - struct ArrowStringView key, - struct ArrowStringView value); +ArrowErrorCode ArrowMetadataBuilderAppend( + struct ArrowBuffer* buffer, + struct ArrowStringView key, + struct ArrowStringView value); /// \brief Set a key/value pair to a buffer containing serialized metadata /// /// Ensures that the only entry for key in the metadata is set to value. /// This function maintains the existing position of (the first instance of) /// key if present in the data. -ArrowErrorCode ArrowMetadataBuilderSet(struct ArrowBuffer* buffer, - struct ArrowStringView key, - struct ArrowStringView value); +ArrowErrorCode ArrowMetadataBuilderSet( + struct ArrowBuffer* buffer, + struct ArrowStringView key, + struct ArrowStringView value); /// \brief Remove a key from a buffer containing serialized metadata -ArrowErrorCode ArrowMetadataBuilderRemove(struct ArrowBuffer* buffer, - struct ArrowStringView key); +ArrowErrorCode ArrowMetadataBuilderRemove( + struct ArrowBuffer* buffer, struct ArrowStringView key); /// @} @@ -1499,9 +1580,9 @@ struct ArrowSchemaView { /// \brief The storage data type represented by the schema /// - /// This value will never be NANOARROW_TYPE_DICTIONARY, NANOARROW_TYPE_EXTENSION - /// or any datetime type. This value represents only the type required to - /// interpret the buffers in the array. + /// This value will never be NANOARROW_TYPE_DICTIONARY, + /// NANOARROW_TYPE_EXTENSION or any datetime type. This value represents only + /// the type required to interpret the buffers in the array. enum ArrowType storage_type; /// \brief The storage layout represented by the schema @@ -1568,9 +1649,10 @@ struct ArrowSchemaView { }; /// \brief Initialize an ArrowSchemaView -ArrowErrorCode ArrowSchemaViewInit(struct ArrowSchemaView* schema_view, - const struct ArrowSchema* schema, - struct ArrowError* error); +ArrowErrorCode ArrowSchemaViewInit( + struct ArrowSchemaView* schema_view, + const struct ArrowSchema* schema, + struct ArrowError* error); /// @} @@ -1601,7 +1683,8 @@ static inline void ArrowBufferReset(struct ArrowBuffer* buffer); /// /// Transfers the buffer data and lifecycle management to another /// address and resets buffer. -static inline void ArrowBufferMove(struct ArrowBuffer* src, struct ArrowBuffer* dst); +static inline void ArrowBufferMove( + struct ArrowBuffer* src, struct ArrowBuffer* dst); /// \brief Grow or shrink a buffer to a given capacity /// @@ -1609,85 +1692,84 @@ static inline void ArrowBufferMove(struct ArrowBuffer* src, struct ArrowBuffer* /// if shrink_to_fit is non-zero. Calling ArrowBufferResize() does not /// adjust the buffer's size member except to ensure that the invariant /// capacity >= size remains true. -static inline ArrowErrorCode ArrowBufferResize(struct ArrowBuffer* buffer, - int64_t new_capacity_bytes, - char shrink_to_fit); +static inline ArrowErrorCode ArrowBufferResize( + struct ArrowBuffer* buffer, int64_t new_capacity_bytes, char shrink_to_fit); /// \brief Ensure a buffer has at least a given additional capacity /// /// Ensures that the buffer has space to append at least /// additional_size_bytes, overallocating when required. -static inline ArrowErrorCode ArrowBufferReserve(struct ArrowBuffer* buffer, - int64_t additional_size_bytes); +static inline ArrowErrorCode ArrowBufferReserve( + struct ArrowBuffer* buffer, int64_t additional_size_bytes); /// \brief Write data to buffer and increment the buffer size /// /// This function does not check that buffer has the required capacity -static inline void ArrowBufferAppendUnsafe(struct ArrowBuffer* buffer, const void* data, - int64_t size_bytes); +static inline void ArrowBufferAppendUnsafe( + struct ArrowBuffer* buffer, const void* data, int64_t size_bytes); /// \brief Write data to buffer and increment the buffer size /// /// This function writes and ensures that the buffer has the required capacity, /// possibly by reallocating the buffer. Like ArrowBufferReserve, this will /// overallocate when reallocation is required. -static inline ArrowErrorCode ArrowBufferAppend(struct ArrowBuffer* buffer, - const void* data, int64_t size_bytes); +static inline ArrowErrorCode ArrowBufferAppend( + struct ArrowBuffer* buffer, const void* data, int64_t size_bytes); /// \brief Write fill to buffer and increment the buffer size /// /// This function writes the specified number of fill bytes and /// ensures that the buffer has the required capacity, -static inline ArrowErrorCode ArrowBufferAppendFill(struct ArrowBuffer* buffer, - uint8_t value, int64_t size_bytes); +static inline ArrowErrorCode ArrowBufferAppendFill( + struct ArrowBuffer* buffer, uint8_t value, int64_t size_bytes); /// \brief Write an 8-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendInt8(struct ArrowBuffer* buffer, - int8_t value); +static inline ArrowErrorCode ArrowBufferAppendInt8( + struct ArrowBuffer* buffer, int8_t value); /// \brief Write an unsigned 8-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendUInt8(struct ArrowBuffer* buffer, - uint8_t value); +static inline ArrowErrorCode ArrowBufferAppendUInt8( + struct ArrowBuffer* buffer, uint8_t value); /// \brief Write a 16-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendInt16(struct ArrowBuffer* buffer, - int16_t value); +static inline ArrowErrorCode ArrowBufferAppendInt16( + struct ArrowBuffer* buffer, int16_t value); /// \brief Write an unsigned 16-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendUInt16(struct ArrowBuffer* buffer, - uint16_t value); +static inline ArrowErrorCode ArrowBufferAppendUInt16( + struct ArrowBuffer* buffer, uint16_t value); /// \brief Write a 32-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendInt32(struct ArrowBuffer* buffer, - int32_t value); +static inline ArrowErrorCode ArrowBufferAppendInt32( + struct ArrowBuffer* buffer, int32_t value); /// \brief Write an unsigned 32-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendUInt32(struct ArrowBuffer* buffer, - uint32_t value); +static inline ArrowErrorCode ArrowBufferAppendUInt32( + struct ArrowBuffer* buffer, uint32_t value); /// \brief Write a 64-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendInt64(struct ArrowBuffer* buffer, - int64_t value); +static inline ArrowErrorCode ArrowBufferAppendInt64( + struct ArrowBuffer* buffer, int64_t value); /// \brief Write an unsigned 64-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendUInt64(struct ArrowBuffer* buffer, - uint64_t value); +static inline ArrowErrorCode ArrowBufferAppendUInt64( + struct ArrowBuffer* buffer, uint64_t value); /// \brief Write a double to a buffer -static inline ArrowErrorCode ArrowBufferAppendDouble(struct ArrowBuffer* buffer, - double value); +static inline ArrowErrorCode ArrowBufferAppendDouble( + struct ArrowBuffer* buffer, double value); /// \brief Write a float to a buffer -static inline ArrowErrorCode ArrowBufferAppendFloat(struct ArrowBuffer* buffer, - float value); +static inline ArrowErrorCode ArrowBufferAppendFloat( + struct ArrowBuffer* buffer, float value); /// \brief Write an ArrowStringView to a buffer -static inline ArrowErrorCode ArrowBufferAppendStringView(struct ArrowBuffer* buffer, - struct ArrowStringView value); +static inline ArrowErrorCode ArrowBufferAppendStringView( + struct ArrowBuffer* buffer, struct ArrowStringView value); /// \brief Write an ArrowBufferView to a buffer -static inline ArrowErrorCode ArrowBufferAppendBufferView(struct ArrowBuffer* buffer, - struct ArrowBufferView value); +static inline ArrowErrorCode ArrowBufferAppendBufferView( + struct ArrowBuffer* buffer, struct ArrowBufferView value); /// @} @@ -1708,19 +1790,20 @@ static inline void ArrowBitClear(uint8_t* bits, int64_t i); static inline void ArrowBitSetTo(uint8_t* bits, int64_t i, uint8_t value); /// \brief Set a boolean value to a range in a bitmap -static inline void ArrowBitsSetTo(uint8_t* bits, int64_t start_offset, int64_t length, - uint8_t bits_are_set); +static inline void ArrowBitsSetTo( + uint8_t* bits, int64_t start_offset, int64_t length, uint8_t bits_are_set); /// \brief Count true values in a bitmap -static inline int64_t ArrowBitCountSet(const uint8_t* bits, int64_t i_from, int64_t i_to); +static inline int64_t ArrowBitCountSet( + const uint8_t* bits, int64_t i_from, int64_t i_to); /// \brief Extract int8 boolean values from a range in a bitmap -static inline void ArrowBitsUnpackInt8(const uint8_t* bits, int64_t start_offset, - int64_t length, int8_t* out); +static inline void ArrowBitsUnpackInt8( + const uint8_t* bits, int64_t start_offset, int64_t length, int8_t* out); /// \brief Extract int32 boolean values from a range in a bitmap -static inline void ArrowBitsUnpackInt32(const uint8_t* bits, int64_t start_offset, - int64_t length, int32_t* out); +static inline void ArrowBitsUnpackInt32( + const uint8_t* bits, int64_t start_offset, int64_t length, int32_t* out); /// \brief Initialize an ArrowBitmap /// @@ -1731,14 +1814,15 @@ static inline void ArrowBitmapInit(struct ArrowBitmap* bitmap); /// /// Transfers the underlying buffer data and lifecycle management to another /// address and resets the bitmap. -static inline void ArrowBitmapMove(struct ArrowBitmap* src, struct ArrowBitmap* dst); +static inline void ArrowBitmapMove( + struct ArrowBitmap* src, struct ArrowBitmap* dst); /// \brief Ensure a bitmap builder has at least a given additional capacity /// /// Ensures that the buffer has space to append at least /// additional_size_bits, overallocating when required. -static inline ArrowErrorCode ArrowBitmapReserve(struct ArrowBitmap* bitmap, - int64_t additional_size_bits); +static inline ArrowErrorCode ArrowBitmapReserve( + struct ArrowBitmap* bitmap, int64_t additional_size_bits); /// \brief Grow or shrink a bitmap to a given capacity /// @@ -1746,33 +1830,34 @@ static inline ArrowErrorCode ArrowBitmapReserve(struct ArrowBitmap* bitmap, /// if shrink_to_fit is non-zero. Calling ArrowBitmapResize() does not /// adjust the buffer's size member except when shrinking new_capacity_bits /// to a value less than the current number of bits in the bitmap. -static inline ArrowErrorCode ArrowBitmapResize(struct ArrowBitmap* bitmap, - int64_t new_capacity_bits, - char shrink_to_fit); +static inline ArrowErrorCode ArrowBitmapResize( + struct ArrowBitmap* bitmap, int64_t new_capacity_bits, char shrink_to_fit); -/// \brief Reserve space for and append zero or more of the same boolean value to a bitmap -static inline ArrowErrorCode ArrowBitmapAppend(struct ArrowBitmap* bitmap, - uint8_t bits_are_set, int64_t length); +/// \brief Reserve space for and append zero or more of the same boolean value +/// to a bitmap +static inline ArrowErrorCode ArrowBitmapAppend( + struct ArrowBitmap* bitmap, uint8_t bits_are_set, int64_t length); /// \brief Append zero or more of the same boolean value to a bitmap -static inline void ArrowBitmapAppendUnsafe(struct ArrowBitmap* bitmap, - uint8_t bits_are_set, int64_t length); +static inline void ArrowBitmapAppendUnsafe( + struct ArrowBitmap* bitmap, uint8_t bits_are_set, int64_t length); /// \brief Append boolean values encoded as int8_t to a bitmap /// /// The values must all be 0 or 1. -static inline void ArrowBitmapAppendInt8Unsafe(struct ArrowBitmap* bitmap, - const int8_t* values, int64_t n_values); +static inline void ArrowBitmapAppendInt8Unsafe( + struct ArrowBitmap* bitmap, const int8_t* values, int64_t n_values); /// \brief Append boolean values encoded as int32_t to a bitmap /// /// The values must all be 0 or 1. -static inline void ArrowBitmapAppendInt32Unsafe(struct ArrowBitmap* bitmap, - const int32_t* values, int64_t n_values); +static inline void ArrowBitmapAppendInt32Unsafe( + struct ArrowBitmap* bitmap, const int32_t* values, int64_t n_values); /// \brief Reset a bitmap builder /// -/// Releases any memory held by buffer, empties the cache, and resets the size to zero +/// Releases any memory held by buffer, empties the cache, and resets the size +/// to zero static inline void ArrowBitmapReset(struct ArrowBitmap* bitmap); /// @} @@ -1791,24 +1876,26 @@ static inline void ArrowBitmapReset(struct ArrowBitmap* bitmap); /// Initializes the fields and release callback of array. Caller /// is responsible for calling the array->release callback if /// NANOARROW_OK is returned. -ArrowErrorCode ArrowArrayInitFromType(struct ArrowArray* array, - enum ArrowType storage_type); +ArrowErrorCode ArrowArrayInitFromType( + struct ArrowArray* array, enum ArrowType storage_type); /// \brief Initialize the contents of an ArrowArray from an ArrowSchema /// /// Caller is responsible for calling the array->release callback if /// NANOARROW_OK is returned. -ArrowErrorCode ArrowArrayInitFromSchema(struct ArrowArray* array, - const struct ArrowSchema* schema, - struct ArrowError* error); +ArrowErrorCode ArrowArrayInitFromSchema( + struct ArrowArray* array, + const struct ArrowSchema* schema, + struct ArrowError* error); /// \brief Initialize the contents of an ArrowArray from an ArrowArrayView /// /// Caller is responsible for calling the array->release callback if /// NANOARROW_OK is returned. -ArrowErrorCode ArrowArrayInitFromArrayView(struct ArrowArray* array, - const struct ArrowArrayView* array_view, - struct ArrowError* error); +ArrowErrorCode ArrowArrayInitFromArrayView( + struct ArrowArray* array, + const struct ArrowArrayView* array_view, + struct ArrowError* error); /// \brief Allocate the array->children array /// @@ -1816,7 +1903,8 @@ ArrowErrorCode ArrowArrayInitFromArrayView(struct ArrowArray* array, /// whose members are marked as released and may be subsequently initialized /// with ArrowArrayInitFromType() or moved from an existing ArrowArray. /// schema must have been allocated using ArrowArrayInitFromType(). -ArrowErrorCode ArrowArrayAllocateChildren(struct ArrowArray* array, int64_t n_children); +ArrowErrorCode ArrowArrayAllocateChildren( + struct ArrowArray* array, int64_t n_children); /// \brief Allocate the array->dictionary member /// @@ -1829,30 +1917,33 @@ ArrowErrorCode ArrowArrayAllocateDictionary(struct ArrowArray* array); /// \brief Set the validity bitmap of an ArrowArray /// /// array must have been allocated using ArrowArrayInitFromType() -void ArrowArraySetValidityBitmap(struct ArrowArray* array, struct ArrowBitmap* bitmap); +void ArrowArraySetValidityBitmap( + struct ArrowArray* array, struct ArrowBitmap* bitmap); /// \brief Set a buffer of an ArrowArray /// /// array must have been allocated using ArrowArrayInitFromType() -ArrowErrorCode ArrowArraySetBuffer(struct ArrowArray* array, int64_t i, - struct ArrowBuffer* buffer); +ArrowErrorCode ArrowArraySetBuffer( + struct ArrowArray* array, int64_t i, struct ArrowBuffer* buffer); /// \brief Get the validity bitmap of an ArrowArray /// /// array must have been allocated using ArrowArrayInitFromType() -static inline struct ArrowBitmap* ArrowArrayValidityBitmap(struct ArrowArray* array); +static inline struct ArrowBitmap* ArrowArrayValidityBitmap( + struct ArrowArray* array); /// \brief Get a buffer of an ArrowArray /// /// array must have been allocated using ArrowArrayInitFromType() -static inline struct ArrowBuffer* ArrowArrayBuffer(struct ArrowArray* array, int64_t i); +static inline struct ArrowBuffer* ArrowArrayBuffer( + struct ArrowArray* array, int64_t i); /// \brief Start element-wise appending to an ArrowArray /// /// Initializes any values needed to use ArrowArrayAppend*() functions. -/// All element-wise appenders append by value and return EINVAL if the exact value -/// cannot be represented by the underlying storage type. -/// array must have been allocated using ArrowArrayInitFromType() +/// All element-wise appenders append by value and return EINVAL if the exact +/// value cannot be represented by the underlying storage type. array must have +/// been allocated using ArrowArrayInitFromType() static inline ArrowErrorCode ArrowArrayStartAppending(struct ArrowArray* array); /// \brief Reserve space for future appends @@ -1861,29 +1952,32 @@ static inline ArrowErrorCode ArrowArrayStartAppending(struct ArrowArray* array); /// child array sizes for non-fixed-size arrays), recursively reserve space for /// additional elements. This is useful for reducing the number of reallocations /// that occur using the item-wise appenders. -ArrowErrorCode ArrowArrayReserve(struct ArrowArray* array, - int64_t additional_size_elements); +ArrowErrorCode ArrowArrayReserve( + struct ArrowArray* array, int64_t additional_size_elements); /// \brief Append a null value to an array -static inline ArrowErrorCode ArrowArrayAppendNull(struct ArrowArray* array, int64_t n); +static inline ArrowErrorCode ArrowArrayAppendNull( + struct ArrowArray* array, int64_t n); /// \brief Append an empty, non-null value to an array -static inline ArrowErrorCode ArrowArrayAppendEmpty(struct ArrowArray* array, int64_t n); +static inline ArrowErrorCode ArrowArrayAppendEmpty( + struct ArrowArray* array, int64_t n); /// \brief Append a signed integer value to an array /// /// Returns NANOARROW_OK if value can be exactly represented by /// the underlying storage type or EINVAL otherwise (e.g., value /// is outside the valid array range). -static inline ArrowErrorCode ArrowArrayAppendInt(struct ArrowArray* array, int64_t value); +static inline ArrowErrorCode ArrowArrayAppendInt( + struct ArrowArray* array, int64_t value); /// \brief Append an unsigned integer value to an array /// /// Returns NANOARROW_OK if value can be exactly represented by /// the underlying storage type or EINVAL otherwise (e.g., value /// is outside the valid array range). -static inline ArrowErrorCode ArrowArrayAppendUInt(struct ArrowArray* array, - uint64_t value); +static inline ArrowErrorCode ArrowArrayAppendUInt( + struct ArrowArray* array, uint64_t value); /// \brief Append a double value to an array /// @@ -1891,67 +1985,68 @@ static inline ArrowErrorCode ArrowArrayAppendUInt(struct ArrowArray* array, /// the underlying storage type or EINVAL otherwise (e.g., value /// is outside the valid array range or there is an attempt to append /// a non-integer to an array with an integer storage type). -static inline ArrowErrorCode ArrowArrayAppendDouble(struct ArrowArray* array, - double value); +static inline ArrowErrorCode ArrowArrayAppendDouble( + struct ArrowArray* array, double value); /// \brief Append a string of bytes to an array /// /// Returns NANOARROW_OK if value can be exactly represented by /// the underlying storage type, EOVERFLOW if appending value would overflow /// the offset type (e.g., if the data buffer would be larger than 2 GB for a -/// non-large string type), or EINVAL otherwise (e.g., the underlying array is not a -/// binary, string, large binary, large string, or fixed-size binary array, or value is -/// the wrong size for a fixed-size binary array). -static inline ArrowErrorCode ArrowArrayAppendBytes(struct ArrowArray* array, - struct ArrowBufferView value); +/// non-large string type), or EINVAL otherwise (e.g., the underlying array is +/// not a binary, string, large binary, large string, or fixed-size binary +/// array, or value is the wrong size for a fixed-size binary array). +static inline ArrowErrorCode ArrowArrayAppendBytes( + struct ArrowArray* array, struct ArrowBufferView value); /// \brief Append a string value to an array /// /// Returns NANOARROW_OK if value can be exactly represented by /// the underlying storage type, EOVERFLOW if appending value would overflow /// the offset type (e.g., if the data buffer would be larger than 2 GB for a -/// non-large string type), or EINVAL otherwise (e.g., the underlying array is not a -/// string or large string array). -static inline ArrowErrorCode ArrowArrayAppendString(struct ArrowArray* array, - struct ArrowStringView value); +/// non-large string type), or EINVAL otherwise (e.g., the underlying array is +/// not a string or large string array). +static inline ArrowErrorCode ArrowArrayAppendString( + struct ArrowArray* array, struct ArrowStringView value); /// \brief Append a Interval to an array /// /// Returns NANOARROW_OK if value can be exactly represented by /// the underlying storage type or EINVAL otherwise. -static inline ArrowErrorCode ArrowArrayAppendInterval(struct ArrowArray* array, - const struct ArrowInterval* value); +static inline ArrowErrorCode ArrowArrayAppendInterval( + struct ArrowArray* array, const struct ArrowInterval* value); /// \brief Append a decimal value to an array /// /// Returns NANOARROW_OK if array is a decimal array with the appropriate /// bitwidth or EINVAL otherwise. -static inline ArrowErrorCode ArrowArrayAppendDecimal(struct ArrowArray* array, - const struct ArrowDecimal* value); +static inline ArrowErrorCode ArrowArrayAppendDecimal( + struct ArrowArray* array, const struct ArrowDecimal* value); /// \brief Finish a nested array element /// /// Appends a non-null element to the array based on the first child's current /// length. Returns NANOARROW_OK if the item was successfully added, EOVERFLOW /// if the child of a list or map array would exceed INT_MAX elements, or EINVAL -/// if the underlying storage type is not a struct, list, large list, or fixed-size -/// list, or if there was an attempt to add a struct or fixed-size list element where the -/// length of the child array(s) did not match the expected length. +/// if the underlying storage type is not a struct, list, large list, or +/// fixed-size list, or if there was an attempt to add a struct or fixed-size +/// list element where the length of the child array(s) did not match the +/// expected length. static inline ArrowErrorCode ArrowArrayFinishElement(struct ArrowArray* array); /// \brief Finish a union array element /// -/// Appends an element to the union type ids buffer and increments array->length. -/// For sparse unions, up to one element is added to non type-id children. Returns -/// EINVAL if the underlying storage type is not a union, if type_id is not valid, -/// or if child sizes after appending are inconsistent. -static inline ArrowErrorCode ArrowArrayFinishUnionElement(struct ArrowArray* array, - int8_t type_id); +/// Appends an element to the union type ids buffer and increments +/// array->length. For sparse unions, up to one element is added to non type-id +/// children. Returns EINVAL if the underlying storage type is not a union, if +/// type_id is not valid, or if child sizes after appending are inconsistent. +static inline ArrowErrorCode ArrowArrayFinishUnionElement( + struct ArrowArray* array, int8_t type_id); /// \brief Shrink buffer capacity to the size required /// -/// Also applies shrinking to any child arrays. array must have been allocated using -/// ArrowArrayInitFromType +/// Also applies shrinking to any child arrays. array must have been allocated +/// using ArrowArrayInitFromType static inline ArrowErrorCode ArrowArrayShrinkToFit(struct ArrowArray* array); /// \brief Finish building an ArrowArray @@ -1960,19 +2055,20 @@ static inline ArrowErrorCode ArrowArrayShrinkToFit(struct ArrowArray* array); /// into array->buffers and checks the actual size of the buffers /// against the expected size based on the final length. /// array must have been allocated using ArrowArrayInitFromType() -ArrowErrorCode ArrowArrayFinishBuildingDefault(struct ArrowArray* array, - struct ArrowError* error); +ArrowErrorCode ArrowArrayFinishBuildingDefault( + struct ArrowArray* array, struct ArrowError* error); /// \brief Finish building an ArrowArray with explicit validation /// -/// Finish building with an explicit validation level. This could perform less validation -/// (i.e. NANOARROW_VALIDATION_LEVEL_NONE or NANOARROW_VALIDATION_LEVEL_MINIMAL) if CPU -/// buffer data access is not possible or more validation (i.e., -/// NANOARROW_VALIDATION_LEVEL_FULL) if buffer content was obtained from an untrusted or -/// corruptible source. -ArrowErrorCode ArrowArrayFinishBuilding(struct ArrowArray* array, - enum ArrowValidationLevel validation_level, - struct ArrowError* error); +/// Finish building with an explicit validation level. This could perform less +/// validation (i.e. NANOARROW_VALIDATION_LEVEL_NONE or +/// NANOARROW_VALIDATION_LEVEL_MINIMAL) if CPU buffer data access is not +/// possible or more validation (i.e., NANOARROW_VALIDATION_LEVEL_FULL) if +/// buffer content was obtained from an untrusted or corruptible source. +ArrowErrorCode ArrowArrayFinishBuilding( + struct ArrowArray* array, + enum ArrowValidationLevel validation_level, + struct ArrowError* error); /// @} @@ -1983,66 +2079,71 @@ ArrowErrorCode ArrowArrayFinishBuilding(struct ArrowArray* array, /// @{ /// \brief Initialize the contents of an ArrowArrayView -void ArrowArrayViewInitFromType(struct ArrowArrayView* array_view, - enum ArrowType storage_type); +void ArrowArrayViewInitFromType( + struct ArrowArrayView* array_view, enum ArrowType storage_type); /// \brief Move an ArrowArrayView /// /// Transfers the ArrowArrayView data and lifecycle management to another /// address and resets the contents of src. -static inline void ArrowArrayViewMove(struct ArrowArrayView* src, - struct ArrowArrayView* dst); +static inline void ArrowArrayViewMove( + struct ArrowArrayView* src, struct ArrowArrayView* dst); /// \brief Initialize the contents of an ArrowArrayView from an ArrowSchema -ArrowErrorCode ArrowArrayViewInitFromSchema(struct ArrowArrayView* array_view, - const struct ArrowSchema* schema, - struct ArrowError* error); +ArrowErrorCode ArrowArrayViewInitFromSchema( + struct ArrowArrayView* array_view, + const struct ArrowSchema* schema, + struct ArrowError* error); /// \brief Allocate the array_view->children array /// /// Includes the memory for each child struct ArrowArrayView -ArrowErrorCode ArrowArrayViewAllocateChildren(struct ArrowArrayView* array_view, - int64_t n_children); +ArrowErrorCode ArrowArrayViewAllocateChildren( + struct ArrowArrayView* array_view, int64_t n_children); /// \brief Allocate array_view->dictionary -ArrowErrorCode ArrowArrayViewAllocateDictionary(struct ArrowArrayView* array_view); +ArrowErrorCode ArrowArrayViewAllocateDictionary( + struct ArrowArrayView* array_view); /// \brief Set data-independent buffer sizes from length void ArrowArrayViewSetLength(struct ArrowArrayView* array_view, int64_t length); /// \brief Set buffer sizes and data pointers from an ArrowArray -ArrowErrorCode ArrowArrayViewSetArray(struct ArrowArrayView* array_view, - const struct ArrowArray* array, - struct ArrowError* error); +ArrowErrorCode ArrowArrayViewSetArray( + struct ArrowArrayView* array_view, + const struct ArrowArray* array, + struct ArrowError* error); -/// \brief Set buffer sizes and data pointers from an ArrowArray except for those -/// that require dereferencing buffer content. -ArrowErrorCode ArrowArrayViewSetArrayMinimal(struct ArrowArrayView* array_view, - const struct ArrowArray* array, - struct ArrowError* error); +/// \brief Set buffer sizes and data pointers from an ArrowArray except for +/// those that require dereferencing buffer content. +ArrowErrorCode ArrowArrayViewSetArrayMinimal( + struct ArrowArrayView* array_view, + const struct ArrowArray* array, + struct ArrowError* error); /// \brief Performs checks on the content of an ArrowArrayView /// /// If using ArrowArrayViewSetArray() to back array_view with an ArrowArray, /// the buffer sizes and some content (fist and last offset) have already /// been validated at the "default" level. If setting the buffer pointers -/// and sizes otherwise, you may wish to perform checks at a different level. See -/// documentation for ArrowValidationLevel for the details of checks performed -/// at each level. -ArrowErrorCode ArrowArrayViewValidate(struct ArrowArrayView* array_view, - enum ArrowValidationLevel validation_level, - struct ArrowError* error); +/// and sizes otherwise, you may wish to perform checks at a different level. +/// See documentation for ArrowValidationLevel for the details of checks +/// performed at each level. +ArrowErrorCode ArrowArrayViewValidate( + struct ArrowArrayView* array_view, + enum ArrowValidationLevel validation_level, + struct ArrowError* error); /// \brief Reset the contents of an ArrowArrayView and frees resources void ArrowArrayViewReset(struct ArrowArrayView* array_view); /// \brief Check for a null element in an ArrowArrayView -static inline int8_t ArrowArrayViewIsNull(const struct ArrowArrayView* array_view, - int64_t i); +static inline int8_t ArrowArrayViewIsNull( + const struct ArrowArrayView* array_view, int64_t i); /// \brief Get the type id of a union array element -static inline int8_t ArrowArrayViewUnionTypeId(const struct ArrowArrayView* array_view, - int64_t i); +static inline int8_t ArrowArrayViewUnionTypeId( + const struct ArrowArrayView* array_view, int64_t i); /// \brief Get the child index of a union array element static inline int8_t ArrowArrayViewUnionChildIndex( @@ -2054,15 +2155,15 @@ static inline int64_t ArrowArrayViewUnionChildOffset( /// \brief Get an element in an ArrowArrayView as an integer /// -/// This function does not check for null values, that values are actually integers, or -/// that values are within a valid range for an int64. -static inline int64_t ArrowArrayViewGetIntUnsafe(const struct ArrowArrayView* array_view, - int64_t i); +/// This function does not check for null values, that values are actually +/// integers, or that values are within a valid range for an int64. +static inline int64_t ArrowArrayViewGetIntUnsafe( + const struct ArrowArrayView* array_view, int64_t i); /// \brief Get an element in an ArrowArrayView as an unsigned integer /// -/// This function does not check for null values, that values are actually integers, or -/// that values are within a valid range for a uint64. +/// This function does not check for null values, that values are actually +/// integers, or that values are within a valid range for a uint64. static inline uint64_t ArrowArrayViewGetUIntUnsafe( const struct ArrowArrayView* array_view, int64_t i); @@ -2090,8 +2191,10 @@ static inline struct ArrowBufferView ArrowArrayViewGetBytesUnsafe( /// This function does not check for null values. The out parameter must /// be initialized with ArrowDecimalInit() with the proper parameters for this /// type before calling this for the first time. -static inline void ArrowArrayViewGetDecimalUnsafe(const struct ArrowArrayView* array_view, - int64_t i, struct ArrowDecimal* out); +static inline void ArrowArrayViewGetDecimalUnsafe( + const struct ArrowArrayView* array_view, + int64_t i, + struct ArrowDecimal* out); /// @} @@ -2109,8 +2212,10 @@ static inline void ArrowArrayViewGetDecimalUnsafe(const struct ArrowArrayView* a /// This function moves the ownership of schema to the array_stream. If /// this function returns NANOARROW_OK, the caller is responsible for /// releasing the ArrowArrayStream. -ArrowErrorCode ArrowBasicArrayStreamInit(struct ArrowArrayStream* array_stream, - struct ArrowSchema* schema, int64_t n_arrays); +ArrowErrorCode ArrowBasicArrayStreamInit( + struct ArrowArrayStream* array_stream, + struct ArrowSchema* schema, + int64_t n_arrays); /// \brief Set the ith ArrowArray in this ArrowArrayStream. /// @@ -2119,29 +2224,27 @@ ArrowErrorCode ArrowBasicArrayStreamInit(struct ArrowArrayStream* array_stream, /// be greater than zero and less than the value of n_arrays passed in /// ArrowBasicArrayStreamInit(). Callers are not required to fill all /// n_arrays members (i.e., n_arrays is a maximum bound). -void ArrowBasicArrayStreamSetArray(struct ArrowArrayStream* array_stream, int64_t i, - struct ArrowArray* array); +void ArrowBasicArrayStreamSetArray( + struct ArrowArrayStream* array_stream, int64_t i, struct ArrowArray* array); /// \brief Validate the contents of this ArrowArrayStream /// /// array_stream must have been initialized with ArrowBasicArrayStreamInit(). -/// This function uses ArrowArrayStreamInitFromSchema() and ArrowArrayStreamSetArray() -/// to validate the contents of the arrays. -ArrowErrorCode ArrowBasicArrayStreamValidate(const struct ArrowArrayStream* array_stream, - struct ArrowError* error); +/// This function uses ArrowArrayStreamInitFromSchema() and +/// ArrowArrayStreamSetArray() to validate the contents of the arrays. +ArrowErrorCode ArrowBasicArrayStreamValidate( + const struct ArrowArrayStream* array_stream, struct ArrowError* error); /// @} -// Undefine ArrowErrorCode, which may have been defined to annotate functions that return -// it to warn for an unused result. +// Undefine ArrowErrorCode, which may have been defined to annotate functions +// that return it to warn for an unused result. #if defined(ArrowErrorCode) #undef ArrowErrorCode #endif // Inline function definitions - - #ifdef __cplusplus } #endif @@ -2171,13 +2274,12 @@ ArrowErrorCode ArrowBasicArrayStreamValidate(const struct ArrowArrayStream* arra #include #include - - #ifdef __cplusplus extern "C" { #endif -static inline int64_t _ArrowGrowByFactor(int64_t current_capacity, int64_t new_capacity) { +static inline int64_t _ArrowGrowByFactor( + int64_t current_capacity, int64_t new_capacity) { int64_t doubled_capacity = current_capacity * 2; if (doubled_capacity > new_capacity) { return doubled_capacity; @@ -2205,8 +2307,8 @@ static inline ArrowErrorCode ArrowBufferSetAllocator( static inline void ArrowBufferReset(struct ArrowBuffer* buffer) { if (buffer->data != NULL) { - buffer->allocator.free(&buffer->allocator, (uint8_t*)buffer->data, - buffer->capacity_bytes); + buffer->allocator.free( + &buffer->allocator, (uint8_t*)buffer->data, buffer->capacity_bytes); buffer->data = NULL; } @@ -2214,22 +2316,27 @@ static inline void ArrowBufferReset(struct ArrowBuffer* buffer) { buffer->size_bytes = 0; } -static inline void ArrowBufferMove(struct ArrowBuffer* src, struct ArrowBuffer* dst) { +static inline void ArrowBufferMove( + struct ArrowBuffer* src, struct ArrowBuffer* dst) { memcpy(dst, src, sizeof(struct ArrowBuffer)); src->data = NULL; ArrowBufferReset(src); } -static inline ArrowErrorCode ArrowBufferResize(struct ArrowBuffer* buffer, - int64_t new_capacity_bytes, - char shrink_to_fit) { +static inline ArrowErrorCode ArrowBufferResize( + struct ArrowBuffer* buffer, + int64_t new_capacity_bytes, + char shrink_to_fit) { if (new_capacity_bytes < 0) { return EINVAL; } if (new_capacity_bytes > buffer->capacity_bytes || shrink_to_fit) { buffer->data = buffer->allocator.reallocate( - &buffer->allocator, buffer->data, buffer->capacity_bytes, new_capacity_bytes); + &buffer->allocator, + buffer->data, + buffer->capacity_bytes, + new_capacity_bytes); if (buffer->data == NULL && new_capacity_bytes > 0) { buffer->capacity_bytes = 0; buffer->size_bytes = 0; @@ -2247,95 +2354,97 @@ static inline ArrowErrorCode ArrowBufferResize(struct ArrowBuffer* buffer, return NANOARROW_OK; } -static inline ArrowErrorCode ArrowBufferReserve(struct ArrowBuffer* buffer, - int64_t additional_size_bytes) { +static inline ArrowErrorCode ArrowBufferReserve( + struct ArrowBuffer* buffer, int64_t additional_size_bytes) { int64_t min_capacity_bytes = buffer->size_bytes + additional_size_bytes; if (min_capacity_bytes <= buffer->capacity_bytes) { return NANOARROW_OK; } return ArrowBufferResize( - buffer, _ArrowGrowByFactor(buffer->capacity_bytes, min_capacity_bytes), 0); + buffer, + _ArrowGrowByFactor(buffer->capacity_bytes, min_capacity_bytes), + 0); } -static inline void ArrowBufferAppendUnsafe(struct ArrowBuffer* buffer, const void* data, - int64_t size_bytes) { +static inline void ArrowBufferAppendUnsafe( + struct ArrowBuffer* buffer, const void* data, int64_t size_bytes) { if (size_bytes > 0) { memcpy(buffer->data + buffer->size_bytes, data, size_bytes); buffer->size_bytes += size_bytes; } } -static inline ArrowErrorCode ArrowBufferAppend(struct ArrowBuffer* buffer, - const void* data, int64_t size_bytes) { +static inline ArrowErrorCode ArrowBufferAppend( + struct ArrowBuffer* buffer, const void* data, int64_t size_bytes) { NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, size_bytes)); ArrowBufferAppendUnsafe(buffer, data, size_bytes); return NANOARROW_OK; } -static inline ArrowErrorCode ArrowBufferAppendInt8(struct ArrowBuffer* buffer, - int8_t value) { +static inline ArrowErrorCode ArrowBufferAppendInt8( + struct ArrowBuffer* buffer, int8_t value) { return ArrowBufferAppend(buffer, &value, sizeof(int8_t)); } -static inline ArrowErrorCode ArrowBufferAppendUInt8(struct ArrowBuffer* buffer, - uint8_t value) { +static inline ArrowErrorCode ArrowBufferAppendUInt8( + struct ArrowBuffer* buffer, uint8_t value) { return ArrowBufferAppend(buffer, &value, sizeof(uint8_t)); } -static inline ArrowErrorCode ArrowBufferAppendInt16(struct ArrowBuffer* buffer, - int16_t value) { +static inline ArrowErrorCode ArrowBufferAppendInt16( + struct ArrowBuffer* buffer, int16_t value) { return ArrowBufferAppend(buffer, &value, sizeof(int16_t)); } -static inline ArrowErrorCode ArrowBufferAppendUInt16(struct ArrowBuffer* buffer, - uint16_t value) { +static inline ArrowErrorCode ArrowBufferAppendUInt16( + struct ArrowBuffer* buffer, uint16_t value) { return ArrowBufferAppend(buffer, &value, sizeof(uint16_t)); } -static inline ArrowErrorCode ArrowBufferAppendInt32(struct ArrowBuffer* buffer, - int32_t value) { +static inline ArrowErrorCode ArrowBufferAppendInt32( + struct ArrowBuffer* buffer, int32_t value) { return ArrowBufferAppend(buffer, &value, sizeof(int32_t)); } -static inline ArrowErrorCode ArrowBufferAppendUInt32(struct ArrowBuffer* buffer, - uint32_t value) { +static inline ArrowErrorCode ArrowBufferAppendUInt32( + struct ArrowBuffer* buffer, uint32_t value) { return ArrowBufferAppend(buffer, &value, sizeof(uint32_t)); } -static inline ArrowErrorCode ArrowBufferAppendInt64(struct ArrowBuffer* buffer, - int64_t value) { +static inline ArrowErrorCode ArrowBufferAppendInt64( + struct ArrowBuffer* buffer, int64_t value) { return ArrowBufferAppend(buffer, &value, sizeof(int64_t)); } -static inline ArrowErrorCode ArrowBufferAppendUInt64(struct ArrowBuffer* buffer, - uint64_t value) { +static inline ArrowErrorCode ArrowBufferAppendUInt64( + struct ArrowBuffer* buffer, uint64_t value) { return ArrowBufferAppend(buffer, &value, sizeof(uint64_t)); } -static inline ArrowErrorCode ArrowBufferAppendDouble(struct ArrowBuffer* buffer, - double value) { +static inline ArrowErrorCode ArrowBufferAppendDouble( + struct ArrowBuffer* buffer, double value) { return ArrowBufferAppend(buffer, &value, sizeof(double)); } -static inline ArrowErrorCode ArrowBufferAppendFloat(struct ArrowBuffer* buffer, - float value) { +static inline ArrowErrorCode ArrowBufferAppendFloat( + struct ArrowBuffer* buffer, float value) { return ArrowBufferAppend(buffer, &value, sizeof(float)); } -static inline ArrowErrorCode ArrowBufferAppendStringView(struct ArrowBuffer* buffer, - struct ArrowStringView value) { +static inline ArrowErrorCode ArrowBufferAppendStringView( + struct ArrowBuffer* buffer, struct ArrowStringView value) { return ArrowBufferAppend(buffer, value.data, value.size_bytes); } -static inline ArrowErrorCode ArrowBufferAppendBufferView(struct ArrowBuffer* buffer, - struct ArrowBufferView value) { +static inline ArrowErrorCode ArrowBufferAppendBufferView( + struct ArrowBuffer* buffer, struct ArrowBufferView value) { return ArrowBufferAppend(buffer, value.data.data, value.size_bytes); } -static inline ArrowErrorCode ArrowBufferAppendFill(struct ArrowBuffer* buffer, - uint8_t value, int64_t size_bytes) { +static inline ArrowErrorCode ArrowBufferAppendFill( + struct ArrowBuffer* buffer, uint8_t value, int64_t size_bytes) { NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, size_bytes)); memset(buffer->data + buffer->size_bytes, value, size_bytes); @@ -2344,20 +2453,24 @@ static inline ArrowErrorCode ArrowBufferAppendFill(struct ArrowBuffer* buffer, } static const uint8_t _ArrowkBitmask[] = {1, 2, 4, 8, 16, 32, 64, 128}; -static const uint8_t _ArrowkFlippedBitmask[] = {254, 253, 251, 247, 239, 223, 191, 127}; +static const uint8_t _ArrowkFlippedBitmask[] = { + 254, 253, 251, 247, 239, 223, 191, 127}; static const uint8_t _ArrowkPrecedingBitmask[] = {0, 1, 3, 7, 15, 31, 63, 127}; -static const uint8_t _ArrowkTrailingBitmask[] = {255, 254, 252, 248, 240, 224, 192, 128}; +static const uint8_t _ArrowkTrailingBitmask[] = { + 255, 254, 252, 248, 240, 224, 192, 128}; static const uint8_t _ArrowkBytePopcount[] = { - 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, - 4, 4, 5, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, - 4, 5, 4, 5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, - 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, - 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, - 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, - 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, - 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 3, 4, 4, 5, 4, 5, 5, 6, - 4, 5, 5, 6, 5, 6, 6, 7, 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8}; + 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, + 2, 3, 3, 4, 3, 4, 4, 5, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, + 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, + 4, 5, 5, 6, 5, 6, 6, 7, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, + 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, + 4, 5, 5, 6, 5, 6, 6, 7, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8}; static inline int64_t _ArrowRoundUpToMultipleOf8(int64_t value) { return (value + 7) & ~((int64_t)7); @@ -2394,25 +2507,25 @@ static inline void _ArrowBitsUnpackInt32(const uint8_t word, int32_t* out) { } static inline void _ArrowBitmapPackInt8(const int8_t* values, uint8_t* out) { - *out = (uint8_t)(values[0] | ((values[1] + 0x1) & 0x2) | ((values[2] + 0x3) & 0x4) | - ((values[3] + 0x7) & 0x8) | ((values[4] + 0xf) & 0x10) | - ((values[5] + 0x1f) & 0x20) | ((values[6] + 0x3f) & 0x40) | - ((values[7] + 0x7f) & 0x80)); + *out = (uint8_t)(values[0] | ((values[1] + 0x1) & 0x2) | + ((values[2] + 0x3) & 0x4) | ((values[3] + 0x7) & 0x8) | + ((values[4] + 0xf) & 0x10) | ((values[5] + 0x1f) & 0x20) | + ((values[6] + 0x3f) & 0x40) | ((values[7] + 0x7f) & 0x80)); } static inline void _ArrowBitmapPackInt32(const int32_t* values, uint8_t* out) { - *out = (uint8_t)(values[0] | ((values[1] + 0x1) & 0x2) | ((values[2] + 0x3) & 0x4) | - ((values[3] + 0x7) & 0x8) | ((values[4] + 0xf) & 0x10) | - ((values[5] + 0x1f) & 0x20) | ((values[6] + 0x3f) & 0x40) | - ((values[7] + 0x7f) & 0x80)); + *out = (uint8_t)(values[0] | ((values[1] + 0x1) & 0x2) | + ((values[2] + 0x3) & 0x4) | ((values[3] + 0x7) & 0x8) | + ((values[4] + 0xf) & 0x10) | ((values[5] + 0x1f) & 0x20) | + ((values[6] + 0x3f) & 0x40) | ((values[7] + 0x7f) & 0x80)); } static inline int8_t ArrowBitGet(const uint8_t* bits, int64_t i) { return (bits[i >> 3] >> (i & 0x07)) & 1; } -static inline void ArrowBitsUnpackInt8(const uint8_t* bits, int64_t start_offset, - int64_t length, int8_t* out) { +static inline void ArrowBitsUnpackInt8( + const uint8_t* bits, int64_t start_offset, int64_t length, int8_t* out) { if (length == 0) { return; } @@ -2450,8 +2563,8 @@ static inline void ArrowBitsUnpackInt8(const uint8_t* bits, int64_t start_offset } } -static inline void ArrowBitsUnpackInt32(const uint8_t* bits, int64_t start_offset, - int64_t length, int32_t* out) { +static inline void ArrowBitsUnpackInt32( + const uint8_t* bits, int64_t start_offset, int64_t length, int32_t* out) { if (length == 0) { return; } @@ -2498,12 +2611,12 @@ static inline void ArrowBitClear(uint8_t* bits, int64_t i) { } static inline void ArrowBitSetTo(uint8_t* bits, int64_t i, uint8_t bit_is_set) { - bits[i / 8] ^= - ((uint8_t)(-((uint8_t)(bit_is_set != 0)) ^ bits[i / 8])) & _ArrowkBitmask[i % 8]; + bits[i / 8] ^= ((uint8_t)(-((uint8_t)(bit_is_set != 0)) ^ bits[i / 8])) & + _ArrowkBitmask[i % 8]; } -static inline void ArrowBitsSetTo(uint8_t* bits, int64_t start_offset, int64_t length, - uint8_t bits_are_set) { +static inline void ArrowBitsSetTo( + uint8_t* bits, int64_t start_offset, int64_t length, uint8_t bits_are_set) { const int64_t i_begin = start_offset; const int64_t i_end = start_offset + length; const uint8_t fill_byte = (uint8_t)(-bits_are_set); @@ -2517,7 +2630,8 @@ static inline void ArrowBitsSetTo(uint8_t* bits, int64_t start_offset, int64_t l if (bytes_end == bytes_begin + 1) { // set bits within a single byte const uint8_t only_byte_mask = - i_end % 8 == 0 ? first_byte_mask : (uint8_t)(first_byte_mask | last_byte_mask); + i_end % 8 == 0 ? first_byte_mask : + (uint8_t)(first_byte_mask | last_byte_mask); bits[bytes_begin] &= only_byte_mask; bits[bytes_begin] |= (uint8_t)(fill_byte & ~only_byte_mask); return; @@ -2529,7 +2643,10 @@ static inline void ArrowBitsSetTo(uint8_t* bits, int64_t start_offset, int64_t l if (bytes_end - bytes_begin > 2) { // set/clear whole bytes - memset(bits + bytes_begin + 1, fill_byte, (size_t)(bytes_end - bytes_begin - 2)); + memset( + bits + bytes_begin + 1, + fill_byte, + (size_t)(bytes_end - bytes_begin - 2)); } if (i_end % 8 == 0) { @@ -2541,8 +2658,8 @@ static inline void ArrowBitsSetTo(uint8_t* bits, int64_t start_offset, int64_t l bits[bytes_end - 1] |= (uint8_t)(fill_byte & ~last_byte_mask); } -static inline int64_t ArrowBitCountSet(const uint8_t* bits, int64_t start_offset, - int64_t length) { +static inline int64_t ArrowBitCountSet( + const uint8_t* bits, int64_t start_offset, int64_t length) { if (length == 0) { return 0; } @@ -2560,14 +2677,16 @@ static inline int64_t ArrowBitCountSet(const uint8_t* bits, int64_t start_offset const uint8_t last_byte_mask = _ArrowkTrailingBitmask[i_begin % 8]; const uint8_t only_byte_mask = - i_end % 8 == 0 ? last_byte_mask : (uint8_t)(first_byte_mask & last_byte_mask); + i_end % 8 == 0 ? last_byte_mask : + (uint8_t)(first_byte_mask & last_byte_mask); const uint8_t byte_masked = bits[bytes_begin] & only_byte_mask; return _ArrowkBytePopcount[byte_masked]; } const uint8_t first_byte_mask = _ArrowkPrecedingBitmask[i_begin % 8]; - const uint8_t last_byte_mask = i_end % 8 == 0 ? 0 : _ArrowkTrailingBitmask[i_end % 8]; + const uint8_t last_byte_mask = + i_end % 8 == 0 ? 0 : _ArrowkTrailingBitmask[i_end % 8]; int64_t count = 0; // first byte @@ -2589,29 +2708,29 @@ static inline void ArrowBitmapInit(struct ArrowBitmap* bitmap) { bitmap->size_bits = 0; } -static inline void ArrowBitmapMove(struct ArrowBitmap* src, struct ArrowBitmap* dst) { +static inline void ArrowBitmapMove( + struct ArrowBitmap* src, struct ArrowBitmap* dst) { ArrowBufferMove(&src->buffer, &dst->buffer); dst->size_bits = src->size_bits; src->size_bits = 0; } -static inline ArrowErrorCode ArrowBitmapReserve(struct ArrowBitmap* bitmap, - int64_t additional_size_bits) { +static inline ArrowErrorCode ArrowBitmapReserve( + struct ArrowBitmap* bitmap, int64_t additional_size_bits) { int64_t min_capacity_bits = bitmap->size_bits + additional_size_bits; if (min_capacity_bits <= (bitmap->buffer.capacity_bytes * 8)) { return NANOARROW_OK; } - NANOARROW_RETURN_NOT_OK( - ArrowBufferReserve(&bitmap->buffer, _ArrowBytesForBits(additional_size_bits))); + NANOARROW_RETURN_NOT_OK(ArrowBufferReserve( + &bitmap->buffer, _ArrowBytesForBits(additional_size_bits))); bitmap->buffer.data[bitmap->buffer.capacity_bytes - 1] = 0; return NANOARROW_OK; } -static inline ArrowErrorCode ArrowBitmapResize(struct ArrowBitmap* bitmap, - int64_t new_capacity_bits, - char shrink_to_fit) { +static inline ArrowErrorCode ArrowBitmapResize( + struct ArrowBitmap* bitmap, int64_t new_capacity_bits, char shrink_to_fit) { if (new_capacity_bits < 0) { return EINVAL; } @@ -2627,23 +2746,23 @@ static inline ArrowErrorCode ArrowBitmapResize(struct ArrowBitmap* bitmap, return NANOARROW_OK; } -static inline ArrowErrorCode ArrowBitmapAppend(struct ArrowBitmap* bitmap, - uint8_t bits_are_set, int64_t length) { +static inline ArrowErrorCode ArrowBitmapAppend( + struct ArrowBitmap* bitmap, uint8_t bits_are_set, int64_t length) { NANOARROW_RETURN_NOT_OK(ArrowBitmapReserve(bitmap, length)); ArrowBitmapAppendUnsafe(bitmap, bits_are_set, length); return NANOARROW_OK; } -static inline void ArrowBitmapAppendUnsafe(struct ArrowBitmap* bitmap, - uint8_t bits_are_set, int64_t length) { +static inline void ArrowBitmapAppendUnsafe( + struct ArrowBitmap* bitmap, uint8_t bits_are_set, int64_t length) { ArrowBitsSetTo(bitmap->buffer.data, bitmap->size_bits, length, bits_are_set); bitmap->size_bits += length; bitmap->buffer.size_bytes = _ArrowBytesForBits(bitmap->size_bits); } -static inline void ArrowBitmapAppendInt8Unsafe(struct ArrowBitmap* bitmap, - const int8_t* values, int64_t n_values) { +static inline void ArrowBitmapAppendInt8Unsafe( + struct ArrowBitmap* bitmap, const int8_t* values, int64_t n_values) { if (n_values == 0) { return; } @@ -2655,7 +2774,8 @@ static inline void ArrowBitmapAppendInt8Unsafe(struct ArrowBitmap* bitmap, // First byte if ((out_i_cursor % 8) != 0) { - int64_t n_partial_bits = _ArrowRoundUpToMultipleOf8(out_i_cursor) - out_i_cursor; + int64_t n_partial_bits = + _ArrowRoundUpToMultipleOf8(out_i_cursor) - out_i_cursor; for (int i = 0; i < n_partial_bits; i++) { ArrowBitSetTo(bitmap->buffer.data, out_i_cursor++, values[i]); } @@ -2689,8 +2809,8 @@ static inline void ArrowBitmapAppendInt8Unsafe(struct ArrowBitmap* bitmap, bitmap->buffer.size_bytes = out_cursor - bitmap->buffer.data; } -static inline void ArrowBitmapAppendInt32Unsafe(struct ArrowBitmap* bitmap, - const int32_t* values, int64_t n_values) { +static inline void ArrowBitmapAppendInt32Unsafe( + struct ArrowBitmap* bitmap, const int32_t* values, int64_t n_values) { if (n_values == 0) { return; } @@ -2702,7 +2822,8 @@ static inline void ArrowBitmapAppendInt32Unsafe(struct ArrowBitmap* bitmap, // First byte if ((out_i_cursor % 8) != 0) { - int64_t n_partial_bits = _ArrowRoundUpToMultipleOf8(out_i_cursor) - out_i_cursor; + int64_t n_partial_bits = + _ArrowRoundUpToMultipleOf8(out_i_cursor) - out_i_cursor; for (int i = 0; i < n_partial_bits; i++) { ArrowBitSetTo(bitmap->buffer.data, out_i_cursor++, (uint8_t)values[i]); } @@ -2727,7 +2848,8 @@ static inline void ArrowBitmapAppendInt32Unsafe(struct ArrowBitmap* bitmap, // Zero out the last byte *out_cursor = 0x00; for (int i = 0; i < n_remaining; i++) { - ArrowBitSetTo(bitmap->buffer.data, out_i_cursor++, (uint8_t)values_cursor[i]); + ArrowBitSetTo( + bitmap->buffer.data, out_i_cursor++, (uint8_t)values_cursor[i]); } out_cursor++; } @@ -2772,20 +2894,19 @@ static inline void ArrowBitmapReset(struct ArrowBitmap* bitmap) { #include #include - - - #ifdef __cplusplus extern "C" { #endif -static inline struct ArrowBitmap* ArrowArrayValidityBitmap(struct ArrowArray* array) { +static inline struct ArrowBitmap* ArrowArrayValidityBitmap( + struct ArrowArray* array) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; return &private_data->bitmap; } -static inline struct ArrowBuffer* ArrowArrayBuffer(struct ArrowArray* array, int64_t i) { +static inline struct ArrowBuffer* ArrowArrayBuffer( + struct ArrowArray* array, int64_t i) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; switch (i) { @@ -2799,19 +2920,20 @@ static inline struct ArrowBuffer* ArrowArrayBuffer(struct ArrowArray* array, int // We don't currently support the case of unions where type_id != child_index; // however, these functions are used to keep track of where that assumption // is made. -static inline int8_t _ArrowArrayUnionChildIndex(struct ArrowArray* array, - int8_t type_id) { +static inline int8_t _ArrowArrayUnionChildIndex( + struct ArrowArray* array, int8_t type_id) { NANOARROW_UNUSED(array); return type_id; } -static inline int8_t _ArrowArrayUnionTypeId(struct ArrowArray* array, - int8_t child_index) { +static inline int8_t _ArrowArrayUnionTypeId( + struct ArrowArray* array, int8_t child_index) { NANOARROW_UNUSED(array); return child_index; } -static inline int32_t _ArrowParseUnionTypeIds(const char* type_ids, int8_t* out) { +static inline int32_t _ArrowParseUnionTypeIds( + const char* type_ids, int8_t* out) { if (*type_ids == '\0') { return 0; } @@ -2844,9 +2966,8 @@ static inline int32_t _ArrowParseUnionTypeIds(const char* type_ids, int8_t* out) return -1; } -static inline int8_t _ArrowParsedUnionTypeIdsWillEqualChildIndices(const int8_t* type_ids, - int64_t n_type_ids, - int64_t n_children) { +static inline int8_t _ArrowParsedUnionTypeIdsWillEqualChildIndices( + const int8_t* type_ids, int64_t n_type_ids, int64_t n_children) { if (n_type_ids != n_children) { return 0; } @@ -2860,14 +2981,16 @@ static inline int8_t _ArrowParsedUnionTypeIdsWillEqualChildIndices(const int8_t* return 1; } -static inline int8_t _ArrowUnionTypeIdsWillEqualChildIndices(const char* type_id_str, - int64_t n_children) { +static inline int8_t _ArrowUnionTypeIdsWillEqualChildIndices( + const char* type_id_str, int64_t n_children) { int8_t type_ids[128]; int32_t n_type_ids = _ArrowParseUnionTypeIds(type_id_str, type_ids); - return _ArrowParsedUnionTypeIdsWillEqualChildIndices(type_ids, n_type_ids, n_children); + return _ArrowParsedUnionTypeIdsWillEqualChildIndices( + type_ids, n_type_ids, n_children); } -static inline ArrowErrorCode ArrowArrayStartAppending(struct ArrowArray* array) { +static inline ArrowErrorCode ArrowArrayStartAppending( + struct ArrowArray* array) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -2891,12 +3014,17 @@ static inline ArrowErrorCode ArrowArrayStartAppending(struct ArrowArray* array) // Initialize any data offset buffer with a single zero for (int i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) { - if (private_data->layout.buffer_type[i] == NANOARROW_BUFFER_TYPE_DATA_OFFSET && + if (private_data->layout.buffer_type[i] == + NANOARROW_BUFFER_TYPE_DATA_OFFSET && private_data->layout.element_size_bits[i] == 64) { - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt64(ArrowArrayBuffer(array, i), 0)); - } else if (private_data->layout.buffer_type[i] == NANOARROW_BUFFER_TYPE_DATA_OFFSET && - private_data->layout.element_size_bits[i] == 32) { - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(ArrowArrayBuffer(array, i), 0)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendInt64(ArrowArrayBuffer(array, i), 0)); + } else if ( + private_data->layout.buffer_type[i] == + NANOARROW_BUFFER_TYPE_DATA_OFFSET && + private_data->layout.element_size_bits[i] == 32) { + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendInt32(ArrowArrayBuffer(array, i), 0)); } } @@ -2929,15 +3057,15 @@ static inline ArrowErrorCode ArrowArrayShrinkToFit(struct ArrowArray* array) { return NANOARROW_OK; } -static inline ArrowErrorCode _ArrowArrayAppendBits(struct ArrowArray* array, - int64_t buffer_i, uint8_t value, - int64_t n) { +static inline ArrowErrorCode _ArrowArrayAppendBits( + struct ArrowArray* array, int64_t buffer_i, uint8_t value, int64_t n) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; struct ArrowBuffer* buffer = ArrowArrayBuffer(array, buffer_i); int64_t bytes_required = - _ArrowRoundUpToMultipleOf8(private_data->layout.element_size_bits[buffer_i] * - (array->length + 1)) / + _ArrowRoundUpToMultipleOf8( + private_data->layout.element_size_bits[buffer_i] * + (array->length + 1)) / 8; if (bytes_required > buffer->size_bytes) { NANOARROW_RETURN_NOT_OK( @@ -2948,8 +3076,8 @@ static inline ArrowErrorCode _ArrowArrayAppendBits(struct ArrowArray* array, return NANOARROW_OK; } -static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal(struct ArrowArray* array, - int64_t n, uint8_t is_valid) { +static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal( + struct ArrowArray* array, int64_t n, uint8_t is_valid) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -2974,10 +3102,11 @@ static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal(struct ArrowArray* a ArrowBufferAppendFill(ArrowArrayBuffer(array, 0), type_id, n)); for (int64_t i = 0; i < n; i++) { NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32( - ArrowArrayBuffer(array, 1), (int32_t)array->children[0]->length - 1)); + ArrowArrayBuffer(array, 1), + (int32_t)array->children[0]->length - 1)); } - // For the purposes of array->null_count, union elements are never considered "null" - // even if some children contain nulls. + // For the purposes of array->null_count, union elements are never + // considered "null" even if some children contain nulls. array->length += n; return NANOARROW_OK; } @@ -2993,8 +3122,8 @@ static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal(struct ArrowArray* a NANOARROW_RETURN_NOT_OK( ArrowBufferAppendFill(ArrowArrayBuffer(array, 0), type_id, n)); - // For the purposes of array->null_count, union elements are never considered "null" - // even if some children contain nulls. + // For the purposes of array->null_count, union elements are never + // considered "null" even if some children contain nulls. array->length += n; return NANOARROW_OK; } @@ -3013,10 +3142,11 @@ static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal(struct ArrowArray* a break; } - // Append n is_valid bits to the validity bitmap. If we haven't allocated a bitmap yet - // and we need to append nulls, do it now. + // Append n is_valid bits to the validity bitmap. If we haven't allocated a + // bitmap yet and we need to append nulls, do it now. if (!is_valid && private_data->bitmap.buffer.data == NULL) { - NANOARROW_RETURN_NOT_OK(ArrowBitmapReserve(&private_data->bitmap, array->length + n)); + NANOARROW_RETURN_NOT_OK( + ArrowBitmapReserve(&private_data->bitmap, array->length + n)); ArrowBitmapAppendUnsafe(&private_data->bitmap, 1, array->length); ArrowBitmapAppendUnsafe(&private_data->bitmap, is_valid, n); } else if (private_data->bitmap.buffer.data != NULL) { @@ -3037,12 +3167,15 @@ static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal(struct ArrowArray* a case NANOARROW_BUFFER_TYPE_VALIDITY: continue; case NANOARROW_BUFFER_TYPE_DATA_OFFSET: - // Append the current value at the end of the offset buffer for each element + // Append the current value at the end of the offset buffer for each + // element NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, size_bytes * n)); for (int64_t j = 0; j < n; j++) { - ArrowBufferAppendUnsafe(buffer, buffer->data + size_bytes * (array->length + j), - size_bytes); + ArrowBufferAppendUnsafe( + buffer, + buffer->data + size_bytes * (array->length + j), + size_bytes); } // Skip the data buffer @@ -3051,7 +3184,8 @@ static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal(struct ArrowArray* a case NANOARROW_BUFFER_TYPE_DATA: // Zero out the next bit of memory if (private_data->layout.element_size_bits[i] % 8 == 0) { - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendFill(buffer, 0, size_bytes * n)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendFill(buffer, 0, size_bytes * n)); } else { NANOARROW_RETURN_NOT_OK(_ArrowArrayAppendBits(array, i, 0, n)); } @@ -3069,16 +3203,18 @@ static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal(struct ArrowArray* a return NANOARROW_OK; } -static inline ArrowErrorCode ArrowArrayAppendNull(struct ArrowArray* array, int64_t n) { +static inline ArrowErrorCode ArrowArrayAppendNull( + struct ArrowArray* array, int64_t n) { return _ArrowArrayAppendEmptyInternal(array, n, 0); } -static inline ArrowErrorCode ArrowArrayAppendEmpty(struct ArrowArray* array, int64_t n) { +static inline ArrowErrorCode ArrowArrayAppendEmpty( + struct ArrowArray* array, int64_t n) { return _ArrowArrayAppendEmptyInternal(array, n, 1); } -static inline ArrowErrorCode ArrowArrayAppendInt(struct ArrowArray* array, - int64_t value) { +static inline ArrowErrorCode ArrowArrayAppendInt( + struct ArrowArray* array, int64_t value) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -3086,19 +3222,23 @@ static inline ArrowErrorCode ArrowArrayAppendInt(struct ArrowArray* array, switch (private_data->storage_type) { case NANOARROW_TYPE_INT64: - NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(data_buffer, &value, sizeof(int64_t))); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppend(data_buffer, &value, sizeof(int64_t))); break; case NANOARROW_TYPE_INT32: _NANOARROW_CHECK_RANGE(value, INT32_MIN, INT32_MAX); - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(data_buffer, (int32_t)value)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendInt32(data_buffer, (int32_t)value)); break; case NANOARROW_TYPE_INT16: _NANOARROW_CHECK_RANGE(value, INT16_MIN, INT16_MAX); - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt16(data_buffer, (int16_t)value)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendInt16(data_buffer, (int16_t)value)); break; case NANOARROW_TYPE_INT8: _NANOARROW_CHECK_RANGE(value, INT8_MIN, INT8_MAX); - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt8(data_buffer, (int8_t)value)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendInt8(data_buffer, (int8_t)value)); break; case NANOARROW_TYPE_UINT64: case NANOARROW_TYPE_UINT32: @@ -3107,10 +3247,12 @@ static inline ArrowErrorCode ArrowArrayAppendInt(struct ArrowArray* array, _NANOARROW_CHECK_RANGE(value, 0, INT64_MAX); return ArrowArrayAppendUInt(array, value); case NANOARROW_TYPE_DOUBLE: - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendDouble(data_buffer, (double)value)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendDouble(data_buffer, (double)value)); break; case NANOARROW_TYPE_FLOAT: - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendFloat(data_buffer, (float)value)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendFloat(data_buffer, (float)value)); break; case NANOARROW_TYPE_BOOL: NANOARROW_RETURN_NOT_OK(_ArrowArrayAppendBits(array, 1, value != 0, 1)); @@ -3120,15 +3262,16 @@ static inline ArrowErrorCode ArrowArrayAppendInt(struct ArrowArray* array, } if (private_data->bitmap.buffer.data != NULL) { - NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); + NANOARROW_RETURN_NOT_OK( + ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); } array->length++; return NANOARROW_OK; } -static inline ArrowErrorCode ArrowArrayAppendUInt(struct ArrowArray* array, - uint64_t value) { +static inline ArrowErrorCode ArrowArrayAppendUInt( + struct ArrowArray* array, uint64_t value) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -3136,19 +3279,23 @@ static inline ArrowErrorCode ArrowArrayAppendUInt(struct ArrowArray* array, switch (private_data->storage_type) { case NANOARROW_TYPE_UINT64: - NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(data_buffer, &value, sizeof(uint64_t))); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppend(data_buffer, &value, sizeof(uint64_t))); break; case NANOARROW_TYPE_UINT32: _NANOARROW_CHECK_UPPER_LIMIT(value, UINT32_MAX); - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendUInt32(data_buffer, (uint32_t)value)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendUInt32(data_buffer, (uint32_t)value)); break; case NANOARROW_TYPE_UINT16: _NANOARROW_CHECK_UPPER_LIMIT(value, UINT16_MAX); - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendUInt16(data_buffer, (uint16_t)value)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendUInt16(data_buffer, (uint16_t)value)); break; case NANOARROW_TYPE_UINT8: _NANOARROW_CHECK_UPPER_LIMIT(value, UINT8_MAX); - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendUInt8(data_buffer, (uint8_t)value)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendUInt8(data_buffer, (uint8_t)value)); break; case NANOARROW_TYPE_INT64: case NANOARROW_TYPE_INT32: @@ -3157,10 +3304,12 @@ static inline ArrowErrorCode ArrowArrayAppendUInt(struct ArrowArray* array, _NANOARROW_CHECK_UPPER_LIMIT(value, INT64_MAX); return ArrowArrayAppendInt(array, value); case NANOARROW_TYPE_DOUBLE: - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendDouble(data_buffer, (double)value)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendDouble(data_buffer, (double)value)); break; case NANOARROW_TYPE_FLOAT: - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendFloat(data_buffer, (float)value)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendFloat(data_buffer, (float)value)); break; case NANOARROW_TYPE_BOOL: NANOARROW_RETURN_NOT_OK(_ArrowArrayAppendBits(array, 1, value != 0, 1)); @@ -3170,15 +3319,16 @@ static inline ArrowErrorCode ArrowArrayAppendUInt(struct ArrowArray* array, } if (private_data->bitmap.buffer.data != NULL) { - NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); + NANOARROW_RETURN_NOT_OK( + ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); } array->length++; return NANOARROW_OK; } -static inline ArrowErrorCode ArrowArrayAppendDouble(struct ArrowArray* array, - double value) { +static inline ArrowErrorCode ArrowArrayAppendDouble( + struct ArrowArray* array, double value) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -3186,31 +3336,35 @@ static inline ArrowErrorCode ArrowArrayAppendDouble(struct ArrowArray* array, switch (private_data->storage_type) { case NANOARROW_TYPE_DOUBLE: - NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(data_buffer, &value, sizeof(double))); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppend(data_buffer, &value, sizeof(double))); break; case NANOARROW_TYPE_FLOAT: - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendFloat(data_buffer, (float)value)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendFloat(data_buffer, (float)value)); break; default: return EINVAL; } if (private_data->bitmap.buffer.data != NULL) { - NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); + NANOARROW_RETURN_NOT_OK( + ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); } array->length++; return NANOARROW_OK; } -static inline ArrowErrorCode ArrowArrayAppendBytes(struct ArrowArray* array, - struct ArrowBufferView value) { +static inline ArrowErrorCode ArrowArrayAppendBytes( + struct ArrowArray* array, struct ArrowBufferView value) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; struct ArrowBuffer* offset_buffer = ArrowArrayBuffer(array, 1); struct ArrowBuffer* data_buffer = ArrowArrayBuffer( - array, 1 + (private_data->storage_type != NANOARROW_TYPE_FIXED_SIZE_BINARY)); + array, + 1 + (private_data->storage_type != NANOARROW_TYPE_FIXED_SIZE_BINARY)); int32_t offset; int64_t large_offset; int64_t fixed_size_bytes = private_data->layout.element_size_bits[1] / 8; @@ -3224,7 +3378,8 @@ static inline ArrowErrorCode ArrowArrayAppendBytes(struct ArrowArray* array, } offset += (int32_t)value.size_bytes; - NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(offset_buffer, &offset, sizeof(int32_t))); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppend(offset_buffer, &offset, sizeof(int32_t))); NANOARROW_RETURN_NOT_OK( ArrowBufferAppend(data_buffer, value.data.data, value.size_bytes)); break; @@ -3252,15 +3407,16 @@ static inline ArrowErrorCode ArrowArrayAppendBytes(struct ArrowArray* array, } if (private_data->bitmap.buffer.data != NULL) { - NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); + NANOARROW_RETURN_NOT_OK( + ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); } array->length++; return NANOARROW_OK; } -static inline ArrowErrorCode ArrowArrayAppendString(struct ArrowArray* array, - struct ArrowStringView value) { +static inline ArrowErrorCode ArrowArrayAppendString( + struct ArrowArray* array, struct ArrowStringView value) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -3279,8 +3435,8 @@ static inline ArrowErrorCode ArrowArrayAppendString(struct ArrowArray* array, } } -static inline ArrowErrorCode ArrowArrayAppendInterval(struct ArrowArray* array, - const struct ArrowInterval* value) { +static inline ArrowErrorCode ArrowArrayAppendInterval( + struct ArrowArray* array, const struct ArrowInterval* value) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -3292,7 +3448,8 @@ static inline ArrowErrorCode ArrowArrayAppendInterval(struct ArrowArray* array, return EINVAL; } - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(data_buffer, value->months)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendInt32(data_buffer, value->months)); break; } case NANOARROW_TYPE_INTERVAL_DAY_TIME: { @@ -3309,7 +3466,8 @@ static inline ArrowErrorCode ArrowArrayAppendInterval(struct ArrowArray* array, return EINVAL; } - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(data_buffer, value->months)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendInt32(data_buffer, value->months)); NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(data_buffer, value->days)); NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt64(data_buffer, value->ns)); break; @@ -3319,15 +3477,16 @@ static inline ArrowErrorCode ArrowArrayAppendInterval(struct ArrowArray* array, } if (private_data->bitmap.buffer.data != NULL) { - NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); + NANOARROW_RETURN_NOT_OK( + ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); } array->length++; return NANOARROW_OK; } -static inline ArrowErrorCode ArrowArrayAppendDecimal(struct ArrowArray* array, - const struct ArrowDecimal* value) { +static inline ArrowErrorCode ArrowArrayAppendDecimal( + struct ArrowArray* array, const struct ArrowDecimal* value) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; struct ArrowBuffer* data_buffer = ArrowArrayBuffer(array, 1); @@ -3354,7 +3513,8 @@ static inline ArrowErrorCode ArrowArrayAppendDecimal(struct ArrowArray* array, } if (private_data->bitmap.buffer.data != NULL) { - NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); + NANOARROW_RETURN_NOT_OK( + ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); } array->length++; @@ -3374,8 +3534,8 @@ static inline ArrowErrorCode ArrowArrayFinishElement(struct ArrowArray* array) { if (child_length > INT32_MAX) { return EOVERFLOW; } - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendInt32(ArrowArrayBuffer(array, 1), (int32_t)child_length)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32( + ArrowArrayBuffer(array, 1), (int32_t)child_length)); break; case NANOARROW_TYPE_LARGE_LIST: child_length = array->children[0]->length; @@ -3402,15 +3562,16 @@ static inline ArrowErrorCode ArrowArrayFinishElement(struct ArrowArray* array) { } if (private_data->bitmap.buffer.data != NULL) { - NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); + NANOARROW_RETURN_NOT_OK( + ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); } array->length++; return NANOARROW_OK; } -static inline ArrowErrorCode ArrowArrayFinishUnionElement(struct ArrowArray* array, - int8_t type_id) { +static inline ArrowErrorCode ArrowArrayFinishUnionElement( + struct ArrowArray* array, int8_t type_id) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -3422,15 +3583,19 @@ static inline ArrowErrorCode ArrowArrayFinishUnionElement(struct ArrowArray* arr switch (private_data->storage_type) { case NANOARROW_TYPE_DENSE_UNION: // Append the target child length to the union offsets buffer - _NANOARROW_CHECK_RANGE(array->children[child_index]->length, 0, INT32_MAX); + _NANOARROW_CHECK_RANGE( + array->children[child_index]->length, 0, INT32_MAX); NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32( - ArrowArrayBuffer(array, 1), (int32_t)array->children[child_index]->length - 1)); + ArrowArrayBuffer(array, 1), + (int32_t)array->children[child_index]->length - 1)); break; case NANOARROW_TYPE_SPARSE_UNION: - // Append one empty to any non-target column that isn't already the right length - // or abort if appending a null will result in a column with invalid length + // Append one empty to any non-target column that isn't already the right + // length or abort if appending a null will result in a column with + // invalid length for (int64_t i = 0; i < array->n_children; i++) { - if (i == child_index || array->children[i]->length == (array->length + 1)) { + if (i == child_index || + array->children[i]->length == (array->length + 1)) { continue; } @@ -3453,14 +3618,14 @@ static inline ArrowErrorCode ArrowArrayFinishUnionElement(struct ArrowArray* arr return NANOARROW_OK; } -static inline void ArrowArrayViewMove(struct ArrowArrayView* src, - struct ArrowArrayView* dst) { +static inline void ArrowArrayViewMove( + struct ArrowArrayView* src, struct ArrowArrayView* dst) { memcpy(dst, src, sizeof(struct ArrowArrayView)); ArrowArrayViewInitFromType(src, NANOARROW_TYPE_UNINITIALIZED); } -static inline int8_t ArrowArrayViewIsNull(const struct ArrowArrayView* array_view, - int64_t i) { +static inline int8_t ArrowArrayViewIsNull( + const struct ArrowArrayView* array_view, int64_t i) { const uint8_t* validity_buffer = array_view->buffer_views[0].data.as_uint8; i += array_view->offset; switch (array_view->storage_type) { @@ -3475,8 +3640,8 @@ static inline int8_t ArrowArrayViewIsNull(const struct ArrowArrayView* array_vie } } -static inline int8_t ArrowArrayViewUnionTypeId(const struct ArrowArrayView* array_view, - int64_t i) { +static inline int8_t ArrowArrayViewUnionTypeId( + const struct ArrowArrayView* array_view, int64_t i) { switch (array_view->storage_type) { case NANOARROW_TYPE_DENSE_UNION: case NANOARROW_TYPE_SPARSE_UNION: @@ -3520,8 +3685,8 @@ static inline int64_t ArrowArrayViewListChildOffset( } } -static inline int64_t ArrowArrayViewGetIntUnsafe(const struct ArrowArrayView* array_view, - int64_t i) { +static inline int64_t ArrowArrayViewGetIntUnsafe( + const struct ArrowArrayView* array_view, int64_t i) { const struct ArrowBufferView* data_view = &array_view->buffer_views[1]; i += array_view->offset; switch (array_view->storage_type) { @@ -3640,7 +3805,8 @@ static inline struct ArrowStringView ArrowArrayViewGetStringUnsafe( break; case NANOARROW_TYPE_FIXED_SIZE_BINARY: view.size_bytes = array_view->layout.element_size_bits[1] / 8; - view.data = array_view->buffer_views[1].data.as_char + (i * view.size_bytes); + view.data = + array_view->buffer_views[1].data.as_char + (i * view.size_bytes); break; default: view.data = NULL; @@ -3686,7 +3852,9 @@ static inline struct ArrowBufferView ArrowArrayViewGetBytesUnsafe( } static inline void ArrowArrayViewGetIntervalUnsafe( - const struct ArrowArrayView* array_view, int64_t i, struct ArrowInterval* out) { + const struct ArrowArrayView* array_view, + int64_t i, + struct ArrowInterval* out) { const uint8_t* data_view = array_view->buffer_views[1].data.as_uint8; switch (array_view->storage_type) { case NANOARROW_TYPE_INTERVAL_MONTHS: { @@ -3712,8 +3880,10 @@ static inline void ArrowArrayViewGetIntervalUnsafe( } } -static inline void ArrowArrayViewGetDecimalUnsafe(const struct ArrowArrayView* array_view, - int64_t i, struct ArrowDecimal* out) { +static inline void ArrowArrayViewGetDecimalUnsafe( + const struct ArrowArrayView* array_view, + int64_t i, + struct ArrowDecimal* out) { i += array_view->offset; const uint8_t* data_view = array_view->buffer_views[1].data.as_uint8; switch (array_view->storage_type) { diff --git a/libtiledbsoma/src/soma/soma_collection.cc b/libtiledbsoma/src/soma/soma_collection.cc index 5bf74c62bb..dff721027f 100644 --- a/libtiledbsoma/src/soma/soma_collection.cc +++ b/libtiledbsoma/src/soma/soma_collection.cc @@ -41,19 +41,18 @@ using namespace tiledb; //= public static //=================================================================== -std::unique_ptr SOMACollection::create( +void SOMACollection::create( std::string_view uri, std::shared_ptr ctx, std::optional timestamp) { - auto soma_group = SOMAGroup::create(ctx, uri, "SOMACollection", timestamp); - return std::make_unique(*soma_group); + SOMAGroup::create(ctx, uri, "SOMACollection", timestamp); } std::unique_ptr SOMACollection::open( std::string_view uri, OpenMode mode, std::shared_ptr ctx, - std::optional timestamp) { + std::optional> timestamp) { return std::make_unique(mode, uri, ctx, timestamp); } @@ -111,8 +110,10 @@ std::shared_ptr SOMACollection::add_new_experiment( std::string_view uri, URIType uri_type, std::shared_ptr ctx, - ArraySchema schema) { - SOMAExperiment::create(uri, schema, ctx); + std::shared_ptr schema, + ColumnIndexInfo index_columns, + std::optional platform_config) { + SOMAExperiment::create(uri, schema, index_columns, ctx, platform_config); std::shared_ptr member = SOMAExperiment::open( uri, OpenMode::read, ctx); this->set(std::string(uri), uri_type, std::string(key)); @@ -125,8 +126,9 @@ std::shared_ptr SOMACollection::add_new_measurement( std::string_view uri, URIType uri_type, std::shared_ptr ctx, - ArraySchema schema) { - SOMAMeasurement::create(uri, schema, ctx); + std::shared_ptr schema, + ColumnIndexInfo index_columns) { + SOMAMeasurement::create(uri, schema, index_columns, ctx); std::shared_ptr member = SOMAMeasurement::open( uri, OpenMode::read, ctx); this->set(std::string(uri), uri_type, std::string(key)); @@ -139,8 +141,10 @@ std::shared_ptr SOMACollection::add_new_dataframe( std::string_view uri, URIType uri_type, std::shared_ptr ctx, - ArraySchema schema) { - SOMADataFrame::create(uri, schema, ctx); + std::shared_ptr schema, + ColumnIndexInfo index_columns, + std::optional platform_config) { + SOMADataFrame::create(uri, schema, index_columns, ctx, platform_config); std::shared_ptr member = SOMADataFrame::open( uri, OpenMode::read, ctx); this->set(std::string(uri), uri_type, std::string(key)); diff --git a/libtiledbsoma/src/soma/soma_collection.h b/libtiledbsoma/src/soma/soma_collection.h index 58fbf418b7..f022752ef5 100644 --- a/libtiledbsoma/src/soma/soma_collection.h +++ b/libtiledbsoma/src/soma/soma_collection.h @@ -61,7 +61,7 @@ class SOMACollection : public SOMAGroup { * @param ctx TileDB context * @param uri URI to create the SOMACollection */ - static std::unique_ptr create( + static void create( std::string_view uri, std::shared_ptr ctx, std::optional timestamp = std::nullopt); @@ -80,7 +80,7 @@ class SOMACollection : public SOMAGroup { std::string_view uri, OpenMode mode, std::shared_ptr ctx, - std::optional timestamp = std::nullopt); + std::optional> timestamp = std::nullopt); //=================================================================== //= public non-static @@ -99,7 +99,7 @@ class SOMACollection : public SOMAGroup { OpenMode mode, std::string_view uri, std::shared_ptr ctx, - std::optional timestamp) + std::optional> timestamp) : SOMAGroup( mode, uri, @@ -157,7 +157,9 @@ class SOMACollection : public SOMAGroup { std::string_view uri, URIType uri_type, std::shared_ptr ctx, - ArraySchema schema); + std::shared_ptr schema, + ColumnIndexInfo index_columns, + std::optional platform_config = std::nullopt); /** * Create and add a SOMAMeasurement to the SOMACollection. @@ -172,7 +174,8 @@ class SOMACollection : public SOMAGroup { std::string_view uri, URIType uri_type, std::shared_ptr ctx, - ArraySchema schema); + std::shared_ptr schema, + ColumnIndexInfo index_columns); /** * Create and add a SOMADataFrame to the SOMACollection. @@ -187,7 +190,9 @@ class SOMACollection : public SOMAGroup { std::string_view uri, URIType uri_type, std::shared_ptr ctx, - ArraySchema schema); + std::shared_ptr schema, + ColumnIndexInfo index_columns, + std::optional platform_config = std::nullopt); /** * Create and add a SOMADenseNDArray to the SOMACollection. diff --git a/libtiledbsoma/src/soma/soma_dataframe.cc b/libtiledbsoma/src/soma/soma_dataframe.cc index 441288f134..6058b81968 100644 --- a/libtiledbsoma/src/soma/soma_dataframe.cc +++ b/libtiledbsoma/src/soma/soma_dataframe.cc @@ -39,14 +39,16 @@ using namespace tiledb; //= public static //=================================================================== -std::unique_ptr SOMADataFrame::create( +void SOMADataFrame::create( std::string_view uri, - ArraySchema schema, + std::shared_ptr schema, + ColumnIndexInfo index_columns, std::shared_ptr ctx, - std::optional timestamp) { - auto soma_array = SOMAArray::create( - ctx, uri, schema, "SOMADataFrame", timestamp); - return std::make_unique(*soma_array); + std::optional platform_config, + std::optional> timestamp) { + auto tiledb_schema = ArrowAdapter::tiledb_schema_from_arrow_schema( + ctx->tiledb_ctx(), schema, index_columns, platform_config); + SOMAArray::create(ctx, uri, tiledb_schema, "SOMADataFrame", timestamp); } std::unique_ptr SOMADataFrame::open( @@ -55,7 +57,7 @@ std::unique_ptr SOMADataFrame::open( std::shared_ptr ctx, std::vector column_names, ResultOrder result_order, - std::optional timestamp) { + std::optional> timestamp) { return std::make_unique( mode, uri, ctx, column_names, result_order, timestamp); } @@ -74,7 +76,7 @@ bool SOMADataFrame::exists(std::string_view uri) { //= public non-static //=================================================================== -std::unique_ptr SOMADataFrame::schema() const { +std::shared_ptr SOMADataFrame::schema() const { return this->arrow_schema(); } diff --git a/libtiledbsoma/src/soma/soma_dataframe.h b/libtiledbsoma/src/soma/soma_dataframe.h index 8583f16fb2..6f163d9f2c 100644 --- a/libtiledbsoma/src/soma/soma_dataframe.h +++ b/libtiledbsoma/src/soma/soma_dataframe.h @@ -52,17 +52,17 @@ class SOMADataFrame : public SOMAArray { /** * @brief Create a SOMADataFrame object at the given URI. * - * @param uri URI to create the SOMAArray + * @param uri URI to create the SOMADataFrame * @param schema TileDB ArraySchema - * @param ctx SOMAContext - * @param timestamp Optional pair indicating timestamp start and end - * @return std::unique_ptr + * @param platform_config Optional config parameter dictionary */ - static std::unique_ptr create( + static void create( std::string_view uri, - ArraySchema schema, + std::shared_ptr schema, + ColumnIndexInfo index_columns, std::shared_ptr ctx, - std::optional timestamp = std::nullopt); + std::optional platform_config = std::nullopt, + std::optional> timestamp = std::nullopt); /** * @brief Open and return a SOMADataFrame object at the given URI. @@ -78,7 +78,7 @@ class SOMADataFrame : public SOMAArray { * colmajor * @param timestamp If specified, overrides the default timestamp used to * open this object. If unset, uses the timestamp provided by the context. - * @return std::unique_ptr + * @return std::shared_ptr SOMADataFrame */ static std::unique_ptr open( std::string_view uri, @@ -86,7 +86,7 @@ class SOMADataFrame : public SOMAArray { std::shared_ptr ctx, std::vector column_names = {}, ResultOrder result_order = ResultOrder::automatic, - std::optional timestamp = std::nullopt); + std::optional> timestamp = std::nullopt); /** * @brief Check if the SOMADataFrame exists at the URI. @@ -116,7 +116,7 @@ class SOMADataFrame : public SOMAArray { std::shared_ptr ctx, std::vector column_names, ResultOrder result_order, - std::optional timestamp = std::nullopt) + std::optional> timestamp = std::nullopt) : SOMAArray( mode, uri, @@ -142,9 +142,9 @@ class SOMADataFrame : public SOMAArray { /** * Return the data schema, in the form of a ArrowSchema. * - * @return std::unique_ptr + * @return std::shared_ptr */ - std::unique_ptr schema() const; + std::shared_ptr schema() const; /** * Return the index (dimension) column names. diff --git a/libtiledbsoma/src/soma/soma_dense_ndarray.cc b/libtiledbsoma/src/soma/soma_dense_ndarray.cc index b82f8d3ace..1bcc1c4efd 100644 --- a/libtiledbsoma/src/soma/soma_dense_ndarray.cc +++ b/libtiledbsoma/src/soma/soma_dense_ndarray.cc @@ -29,7 +29,6 @@ * * This file defines the SOMADenseNDArray class. */ - #include "soma_dense_ndarray.h" namespace tiledbsoma { @@ -39,14 +38,12 @@ using namespace tiledb; //= public static //=================================================================== -std::unique_ptr SOMADenseNDArray::create( +void SOMADenseNDArray::create( std::string_view uri, ArraySchema schema, std::shared_ptr ctx, - std::optional timestamp) { - auto soma_array = SOMAArray::create( - ctx, uri, schema, "SOMADenseNDArray", timestamp); - return std::make_unique(*soma_array); + std::optional> timestamp) { + SOMAArray::create(ctx, uri, schema, "SOMADenseNDArray", timestamp); } std::unique_ptr SOMADenseNDArray::open( @@ -55,7 +52,7 @@ std::unique_ptr SOMADenseNDArray::open( std::shared_ptr ctx, std::vector column_names, ResultOrder result_order, - std::optional timestamp) { + std::optional> timestamp) { return std::make_unique( mode, uri, ctx, column_names, result_order, timestamp); } @@ -74,7 +71,7 @@ bool SOMADenseNDArray::exists(std::string_view uri) { //= public non-static //=================================================================== -std::unique_ptr SOMADenseNDArray::schema() const { +std::shared_ptr SOMADenseNDArray::schema() const { return this->arrow_schema(); } diff --git a/libtiledbsoma/src/soma/soma_dense_ndarray.h b/libtiledbsoma/src/soma/soma_dense_ndarray.h index 47a13f7bdb..a55ba630d3 100644 --- a/libtiledbsoma/src/soma/soma_dense_ndarray.h +++ b/libtiledbsoma/src/soma/soma_dense_ndarray.h @@ -52,17 +52,16 @@ class SOMADenseNDArray : public SOMAArray { /** * @brief Create a SOMADenseNDArray object at the given URI. * - * @param uri URI to create the SOMAArray + * @param uri URI to create the SOMADenseNDArray * @param schema TileDB ArraySchema - * @param ctx SOMAContext - * @param timestamp Optional pair indicating timestamp start and end - * @return std::unique_ptr + * @param platform_config Optional config parameter dictionary + * @return std::shared_ptr opened in read mode */ - static std::unique_ptr create( + static void create( std::string_view uri, ArraySchema schema, std::shared_ptr ctx, - std::optional timestamp = std::nullopt); + std::optional> timestamp = std::nullopt); /** * @brief Open and return a SOMADenseNDArray object at the given URI. @@ -78,7 +77,7 @@ class SOMADenseNDArray : public SOMAArray { * open this object. If unset, uses the timestamp provided by the context. * @param result_order Read result order: automatic (default), rowmajor, or * colmajor - * @return std::shared_ptr + * @return std::shared_ptr SOMADenseNDArray */ static std::unique_ptr open( std::string_view uri, @@ -86,7 +85,7 @@ class SOMADenseNDArray : public SOMAArray { std::shared_ptr ctx, std::vector column_names = {}, ResultOrder result_order = ResultOrder::automatic, - std::optional timestamp = std::nullopt); + std::optional> timestamp = std::nullopt); /** * @brief Check if the SOMADenseNDArray exists at the URI. @@ -115,7 +114,7 @@ class SOMADenseNDArray : public SOMAArray { std::shared_ptr ctx, std::vector column_names, ResultOrder result_order, - std::optional timestamp) + std::optional> timestamp) : SOMAArray( mode, uri, @@ -152,7 +151,7 @@ class SOMADenseNDArray : public SOMAArray { * * @return std::unique_ptr */ - std::unique_ptr schema() const; + std::shared_ptr schema() const; }; } // namespace tiledbsoma diff --git a/libtiledbsoma/src/soma/soma_experiment.cc b/libtiledbsoma/src/soma/soma_experiment.cc index b2bb3fa5ed..2c42aea68a 100644 --- a/libtiledbsoma/src/soma/soma_experiment.cc +++ b/libtiledbsoma/src/soma/soma_experiment.cc @@ -41,26 +41,38 @@ using namespace tiledb; //= public static //=================================================================== -std::unique_ptr SOMAExperiment::create( +void SOMAExperiment::create( std::string_view uri, - ArraySchema schema, + std::shared_ptr schema, + ColumnIndexInfo index_columns, std::shared_ptr ctx, + std::optional platform_config, std::optional timestamp) { std::string exp_uri(uri); - auto soma_group = SOMAGroup::create(ctx, uri, "SOMAExperiment", timestamp); - SOMADataFrame::create(exp_uri + "/obs", schema, ctx, timestamp); + SOMAGroup::create(ctx, exp_uri, "SOMAExperiment", timestamp); + SOMADataFrame::create( + exp_uri + "/obs", + schema, + index_columns, + ctx, + platform_config, + timestamp); SOMACollection::create(exp_uri + "/ms", ctx, timestamp); - soma_group->set(exp_uri + "/obs", URIType::absolute, "obs"); - soma_group->set(exp_uri + "/ms", URIType::absolute, "ms"); - return std::make_unique(*soma_group); + + auto name = std::string(std::filesystem::path(uri).filename()); + auto group = SOMAGroup::open( + OpenMode::write, exp_uri, ctx, name, timestamp); + group->set(exp_uri + "/obs", URIType::absolute, "obs"); + group->set(exp_uri + "/ms", URIType::absolute, "ms"); + group->close(); } std::unique_ptr SOMAExperiment::open( std::string_view uri, OpenMode mode, std::shared_ptr ctx, - std::optional timestamp) { + std::optional> timestamp) { return std::make_unique(mode, uri, ctx, timestamp); } } // namespace tiledbsoma diff --git a/libtiledbsoma/src/soma/soma_experiment.h b/libtiledbsoma/src/soma/soma_experiment.h index 9303d42b73..14b918a269 100644 --- a/libtiledbsoma/src/soma/soma_experiment.h +++ b/libtiledbsoma/src/soma/soma_experiment.h @@ -54,10 +54,12 @@ class SOMAExperiment : public SOMACollection { * @param schema TileDB ArraySchema * @param platform_config Optional config parameter dictionary */ - static std::unique_ptr create( + static void create( std::string_view uri, - ArraySchema schema, + std::shared_ptr schema, + ColumnIndexInfo index_columns, std::shared_ptr ctx, + std::optional platform_config = std::nullopt, std::optional timestamp = std::nullopt); /** @@ -74,7 +76,7 @@ class SOMAExperiment : public SOMACollection { std::string_view uri, OpenMode mode, std::shared_ptr ctx, - std::optional timestamp = std::nullopt); + std::optional> timestamp = std::nullopt); //=================================================================== //= public non-static @@ -84,7 +86,7 @@ class SOMAExperiment : public SOMACollection { OpenMode mode, std::string_view uri, std::shared_ptr ctx, - std::optional timestamp = std::nullopt) + std::optional> timestamp = std::nullopt) : SOMACollection(mode, uri, ctx, timestamp) { } diff --git a/libtiledbsoma/src/soma/soma_measurement.cc b/libtiledbsoma/src/soma/soma_measurement.cc index 38c892cc5a..d6605e7383 100644 --- a/libtiledbsoma/src/soma/soma_measurement.cc +++ b/libtiledbsoma/src/soma/soma_measurement.cc @@ -41,36 +41,45 @@ using namespace tiledb; //= public static //=================================================================== -std::unique_ptr SOMAMeasurement::create( +void SOMAMeasurement::create( std::string_view uri, - ArraySchema schema, + std::shared_ptr schema, + ColumnIndexInfo index_columns, std::shared_ptr ctx, + std::optional platform_config, std::optional timestamp) { std::string exp_uri(uri); - auto soma_group = SOMAGroup::create( - ctx, exp_uri, "SOMAMeasurement", timestamp); - SOMADataFrame::create(exp_uri + "/var", schema, ctx, timestamp); + SOMAGroup::create(ctx, exp_uri, "SOMAMeasurement", timestamp); + SOMADataFrame::create( + exp_uri + "/var", + schema, + index_columns, + ctx, + platform_config, + timestamp); SOMACollection::create(exp_uri + "/X", ctx, timestamp); SOMACollection::create(exp_uri + "/obsm", ctx, timestamp); SOMACollection::create(exp_uri + "/obsp", ctx, timestamp); SOMACollection::create(exp_uri + "/varm", ctx, timestamp); SOMACollection::create(exp_uri + "/varp", ctx, timestamp); - soma_group->set(exp_uri + "/var", URIType::absolute, "var"); - soma_group->set(exp_uri + "/X", URIType::absolute, "X"); - soma_group->set(exp_uri + "/obsm", URIType::absolute, "obsm"); - soma_group->set(exp_uri + "/obsp", URIType::absolute, "obsp"); - soma_group->set(exp_uri + "/varm", URIType::absolute, "varm"); - soma_group->set(exp_uri + "/varp", URIType::absolute, "varp"); - return std::make_unique(*soma_group); + auto name = std::string(std::filesystem::path(uri).filename()); + auto group = SOMAGroup::open(OpenMode::write, uri, ctx, name, timestamp); + group->set(exp_uri + "/var", URIType::absolute, "var"); + group->set(exp_uri + "/X", URIType::absolute, "X"); + group->set(exp_uri + "/obsm", URIType::absolute, "obsm"); + group->set(exp_uri + "/obsp", URIType::absolute, "obsp"); + group->set(exp_uri + "/varm", URIType::absolute, "varm"); + group->set(exp_uri + "/varp", URIType::absolute, "varp"); + group->close(); } std::unique_ptr SOMAMeasurement::open( std::string_view uri, OpenMode mode, std::shared_ptr ctx, - std::optional timestamp) { + std::optional> timestamp) { return std::make_unique(mode, uri, ctx, timestamp); } } // namespace tiledbsoma diff --git a/libtiledbsoma/src/soma/soma_measurement.h b/libtiledbsoma/src/soma/soma_measurement.h index dbfe3b2505..591c057751 100644 --- a/libtiledbsoma/src/soma/soma_measurement.h +++ b/libtiledbsoma/src/soma/soma_measurement.h @@ -55,10 +55,12 @@ class SOMAMeasurement : public SOMACollection { * @param schema TileDB ArraySchema * @param ctx TileDB context */ - static std::unique_ptr create( + static void create( std::string_view uri, - ArraySchema schema, + std::shared_ptr schema, + ColumnIndexInfo index_columns, std::shared_ptr ctx, + std::optional platform_config = std::nullopt, std::optional timestamp = std::nullopt); /** @@ -75,7 +77,7 @@ class SOMAMeasurement : public SOMACollection { std::string_view uri, OpenMode mode, std::shared_ptr ctx, - std::optional timestamp = std::nullopt); + std::optional> timestamp = std::nullopt); //=================================================================== //= public non-static @@ -84,7 +86,7 @@ class SOMAMeasurement : public SOMACollection { OpenMode mode, std::string_view uri, std::shared_ptr ctx, - std::optional timestamp = std::nullopt) + std::optional> timestamp = std::nullopt) : SOMACollection(mode, uri, ctx, timestamp) { } diff --git a/libtiledbsoma/src/soma/soma_sparse_ndarray.cc b/libtiledbsoma/src/soma/soma_sparse_ndarray.cc index ca3d91fc61..f7b0cd4ecd 100644 --- a/libtiledbsoma/src/soma/soma_sparse_ndarray.cc +++ b/libtiledbsoma/src/soma/soma_sparse_ndarray.cc @@ -39,14 +39,12 @@ using namespace tiledb; //= public static //=================================================================== -std::unique_ptr SOMASparseNDArray::create( +void SOMASparseNDArray::create( std::string_view uri, ArraySchema schema, std::shared_ptr ctx, - std::optional timestamp) { - auto soma_array = SOMAArray::create( - ctx, uri, schema, "SOMASparseNDArray", timestamp); - return std::make_unique(*soma_array); + std::optional> timestamp) { + SOMAArray::create(ctx, uri, schema, "SOMASparseNDArray", timestamp); } std::unique_ptr SOMASparseNDArray::open( @@ -55,7 +53,7 @@ std::unique_ptr SOMASparseNDArray::open( std::shared_ptr ctx, std::vector column_names, ResultOrder result_order, - std::optional timestamp) { + std::optional> timestamp) { return std::make_unique( mode, uri, ctx, column_names, result_order, timestamp); } @@ -74,7 +72,7 @@ bool SOMASparseNDArray::exists(std::string_view uri) { //= public non-static //=================================================================== -std::unique_ptr SOMASparseNDArray::schema() const { +std::shared_ptr SOMASparseNDArray::schema() const { return this->arrow_schema(); } } // namespace tiledbsoma diff --git a/libtiledbsoma/src/soma/soma_sparse_ndarray.h b/libtiledbsoma/src/soma/soma_sparse_ndarray.h index 4500b8870e..4bd9cddee7 100644 --- a/libtiledbsoma/src/soma/soma_sparse_ndarray.h +++ b/libtiledbsoma/src/soma/soma_sparse_ndarray.h @@ -52,17 +52,16 @@ class SOMASparseNDArray : public SOMAArray { /** * @brief Create a SOMASparseNDArray object at the given URI. * - * @param uri URI to create the SOMAArray + * @param uri URI to create the SOMASparseNDArray * @param schema TileDB ArraySchema - * @param ctx SOMAContext - * @param timestamp Optional pair indicating timestamp start and end - * @return std::unique_ptr + * @param platform_config Optional config parameter dictionary + * @return std::shared_ptr opened in read mode */ - static std::unique_ptr create( + static void create( std::string_view uri, ArraySchema schema, std::shared_ptr ctx, - std::optional timestamp = std::nullopt); + std::optional> timestamp = std::nullopt); /** * @brief Open and return a SOMASparseNDArray object at the given URI. @@ -78,7 +77,7 @@ class SOMASparseNDArray : public SOMAArray { * colmajor * @param timestamp If specified, overrides the default timestamp used to * open this object. If unset, uses the timestamp provided by the context. - * @return std::unique_ptr + * @return std::shared_ptr SOMASparseNDArray */ static std::unique_ptr open( std::string_view uri, @@ -86,7 +85,7 @@ class SOMASparseNDArray : public SOMAArray { std::shared_ptr ctx, std::vector column_names = {}, ResultOrder result_order = ResultOrder::automatic, - std::optional timestamp = std::nullopt); + std::optional> timestamp = std::nullopt); /** * @brief Check if the SOMASparseNDArray exists at the URI. @@ -115,7 +114,7 @@ class SOMASparseNDArray : public SOMAArray { std::shared_ptr ctx, std::vector column_names, ResultOrder result_order, - std::optional timestamp) + std::optional> timestamp) : SOMAArray( mode, uri, @@ -150,9 +149,9 @@ class SOMASparseNDArray : public SOMAArray { /** * Return the data schema, in the form of an ArrowSchema. * - * @return std::unique_ptr + * @return std::shared_ptr */ - std::unique_ptr schema() const; + std::shared_ptr schema() const; }; } // namespace tiledbsoma diff --git a/libtiledbsoma/test/CMakeLists.txt b/libtiledbsoma/test/CMakeLists.txt index b58a8952bc..4efbc890b1 100644 --- a/libtiledbsoma/test/CMakeLists.txt +++ b/libtiledbsoma/test/CMakeLists.txt @@ -27,6 +27,8 @@ find_package(Catch_EP REQUIRED) add_executable(unit_soma $ + common.cc + common.h unit_column_buffer.cc unit_managed_query.cc unit_soma_array.cc diff --git a/libtiledbsoma/test/common.cc b/libtiledbsoma/test/common.cc new file mode 100644 index 0000000000..f497908dd6 --- /dev/null +++ b/libtiledbsoma/test/common.cc @@ -0,0 +1,133 @@ +/** + * @file common.cc + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2024 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This file manages common headers and helper classes for the unit test files. + */ + +#include "common.h" + +namespace helper { +ArraySchema create_schema(Context& ctx, bool allow_duplicates) { + // Create schema + ArraySchema schema(ctx, TILEDB_SPARSE); + + auto dim = Dimension::create(ctx, "d0", {0, 1000}); + + Domain domain(ctx); + domain.add_dimension(dim); + schema.set_domain(domain); + + auto attr = Attribute::create(ctx, "a0"); + schema.add_attribute(attr); + schema.set_allows_dups(allow_duplicates); + schema.check(); + + return schema; +} + +std::pair, ColumnIndexInfo> create_arrow_schema() { + // Create ArrowSchema + auto arrow_schema = std::make_shared(); + arrow_schema->format = "+s"; + arrow_schema->n_children = 2; + arrow_schema->dictionary = nullptr; + arrow_schema->release = &ArrowAdapter::release_schema; + arrow_schema->children = new ArrowSchema*[arrow_schema->n_children]; + + ArrowSchema* dim = nullptr; + dim = arrow_schema->children[0] = new ArrowSchema; + dim->format = "l"; + dim->name = "d0"; + dim->n_children = 0; + dim->dictionary = nullptr; + dim->release = &ArrowAdapter::release_schema; + + ArrowSchema* attr = nullptr; + attr = arrow_schema->children[1] = new ArrowSchema; + attr->format = "l"; + attr->name = "a0"; + attr->n_children = 0; + attr->dictionary = nullptr; + attr->release = &ArrowAdapter::release_schema; + + // Create array for index columns + std::vector index_column_names = {"d0"}; + + auto domains = std::make_shared(); + domains->length = 0; + domains->null_count = 0; + domains->offset = 0; + domains->n_buffers = 0; + domains->buffers = nullptr; + domains->n_children = 2; + domains->release = &ArrowAdapter::release_array; + domains->children = new ArrowArray*[1]; + + auto d0_domain = domains->children[0] = new ArrowArray; + d0_domain->length = 2; + d0_domain->null_count = 0; + d0_domain->offset = 0; + d0_domain->n_buffers = 2; + d0_domain->release = &ArrowAdapter::release_array; + d0_domain->buffers = new const void*[2]; + d0_domain->buffers[0] = nullptr; + d0_domain->buffers[1] = malloc(sizeof(int64_t) * 2); + d0_domain->n_children = 0; + int64_t dom[] = {0, 1000}; + std::memcpy((void*)d0_domain->buffers[1], &dom, sizeof(int64_t) * 2); + + auto tiles = std::make_shared(); + tiles->length = 0; + tiles->null_count = 0; + tiles->offset = 0; + tiles->n_buffers = 0; + tiles->buffers = nullptr; + tiles->n_children = 2; + tiles->release = &ArrowAdapter::release_array; + tiles->children = new ArrowArray*[1]; + + ArrowArray* d0_tile = tiles->children[0] = new ArrowArray; + d0_tile->length = 1; + d0_tile->null_count = 0; + d0_tile->offset = 0; + d0_tile->n_buffers = 2; + d0_tile->release = &ArrowAdapter::release_array; + d0_tile->buffers = new const void*[2]; + d0_tile->buffers[0] = nullptr; + d0_tile->buffers[1] = malloc(sizeof(int64_t)); + d0_tile->n_children = 0; + int64_t tile = 1; + std::memcpy((void*)d0_tile->buffers[1], &tile, sizeof(int64_t)); + + ColumnIndexInfo index_columns_info = std::tuple( + index_column_names, domains, tiles); + + return std::pair(arrow_schema, index_columns_info); +} +} // namespace helper \ No newline at end of file diff --git a/libtiledbsoma/test/unit_soma_array.cc b/libtiledbsoma/test/unit_soma_array.cc index baaa0492d5..07c4b7abf5 100644 --- a/libtiledbsoma/test/unit_soma_array.cc +++ b/libtiledbsoma/test/unit_soma_array.cc @@ -138,11 +138,14 @@ std::tuple, std::vector> write_array( } std::vector a0(num_cells_per_fragment, frag_num); + auto array_buffer = std::make_shared(); + auto tdb_arr = std::make_shared( + *ctx->tiledb_ctx(), uri, TILEDB_READ); + array_buffer->emplace("a0", ColumnBuffer::create(tdb_arr, "a0", a0)); + array_buffer->emplace("d0", ColumnBuffer::create(tdb_arr, "d0", d0)); + // Write data to array - soma_array->set_column_data("a0", a0.size(), a0.data()); - soma_array->set_column_data("d0", d0.size(), d0.data()); - soma_array->write(); - soma_array->close(); + soma_array->write(array_buffer); } // Read from TileDB Array to get expected data diff --git a/libtiledbsoma/test/unit_soma_dataframe.cc b/libtiledbsoma/test/unit_soma_dataframe.cc index 39bceda87c..a5eb783aca 100644 --- a/libtiledbsoma/test/unit_soma_dataframe.cc +++ b/libtiledbsoma/test/unit_soma_dataframe.cc @@ -54,10 +54,14 @@ TEST_CASE("SOMADataFrame: basic") { d0[j] = j; std::vector a0(10, 1); + auto array_buffer = std::make_shared(); + auto tdb_arr = std::make_shared( + *ctx->tiledb_ctx(), uri, TILEDB_READ); + array_buffer->emplace("a0", ColumnBuffer::create(tdb_arr, "a0", a0)); + array_buffer->emplace("d0", ColumnBuffer::create(tdb_arr, "d0", d0)); + soma_dataframe = SOMADataFrame::open(uri, OpenMode::write, ctx); - soma_dataframe->set_column_data("a0", a0.size(), a0.data()); - soma_dataframe->set_column_data("d0", d0.size(), d0.data()); - soma_dataframe->write(); + soma_dataframe->write(array_buffer); soma_dataframe->close(); soma_dataframe = SOMADataFrame::open(uri, OpenMode::read, ctx); diff --git a/libtiledbsoma/test/unit_soma_dense_ndarray.cc b/libtiledbsoma/test/unit_soma_dense_ndarray.cc index 97f6677034..bbac4f47b9 100644 --- a/libtiledbsoma/test/unit_soma_dense_ndarray.cc +++ b/libtiledbsoma/test/unit_soma_dense_ndarray.cc @@ -97,10 +97,14 @@ TEST_CASE("SOMADenseNDArray: basic") { std::vector d0{1, 10}; std::vector a0(10, 1); + auto array_buffer = std::make_shared(); + auto tdb_arr = std::make_shared( + *ctx->tiledb_ctx(), uri, TILEDB_READ); + array_buffer->emplace("a0", ColumnBuffer::create(tdb_arr, "a0", a0)); + array_buffer->emplace("d0", ColumnBuffer::create(tdb_arr, "d0", d0)); + soma_dense->open(OpenMode::write); - soma_dense->set_column_data("a0", a0.size(), a0.data()); - soma_dense->set_column_data("d0", d0.size(), d0.data()); - soma_dense->write(); + soma_dense->write(array_buffer); soma_dense->close(); soma_dense->open(OpenMode::read); diff --git a/libtiledbsoma/test/unit_soma_sparse_ndarray.cc b/libtiledbsoma/test/unit_soma_sparse_ndarray.cc index b8ac0a6075..de457197d2 100644 --- a/libtiledbsoma/test/unit_soma_sparse_ndarray.cc +++ b/libtiledbsoma/test/unit_soma_sparse_ndarray.cc @@ -100,10 +100,14 @@ TEST_CASE("SOMASparseNDArray: basic") { d0[j] = j; std::vector a0(10, 1); + auto array_buffer = std::make_shared(); + auto tdb_arr = std::make_shared( + *ctx->tiledb_ctx(), uri, TILEDB_READ); + array_buffer->emplace("a0", ColumnBuffer::create(tdb_arr, "a0", a0)); + array_buffer->emplace("d0", ColumnBuffer::create(tdb_arr, "d0", d0)); + soma_sparse->open(OpenMode::write); - soma_sparse->set_column_data("a0", a0.size(), a0.data()); - soma_sparse->set_column_data("d0", d0.size(), d0.data()); - soma_sparse->write(); + soma_sparse->write(array_buffer); soma_sparse->close(); soma_sparse->open(OpenMode::read); From fc5084f2c9ee1ecbbdf41277b0c7cb239fbbb4ad Mon Sep 17 00:00:00 2001 From: Vivian Nguyen Date: Tue, 9 Apr 2024 14:07:49 -0500 Subject: [PATCH 05/10] Restore nanoarrow.h --- .../external/include/nanoarrow/nanoarrow.h | 1274 +++++++---------- 1 file changed, 552 insertions(+), 722 deletions(-) diff --git a/libtiledbsoma/src/external/include/nanoarrow/nanoarrow.h b/libtiledbsoma/src/external/include/nanoarrow/nanoarrow.h index fa99f12f25..e338560f1a 100644 --- a/libtiledbsoma/src/external/include/nanoarrow/nanoarrow.h +++ b/libtiledbsoma/src/external/include/nanoarrow/nanoarrow.h @@ -53,6 +53,8 @@ #include #include + + #if defined(NANOARROW_DEBUG) && !defined(NANOARROW_PRINT_AND_DIE) #include #include @@ -68,11 +70,11 @@ extern "C" { /// \defgroup nanoarrow-arrow-cdata Arrow C Data interface /// /// The Arrow C Data (https://arrow.apache.org/docs/format/CDataInterface.html) -/// and Arrow C Stream -/// (https://arrow.apache.org/docs/format/CStreamInterface.html) interfaces are -/// part of the Arrow Columnar Format specification -/// (https://arrow.apache.org/docs/format/Columnar.html). See the Arrow -/// documentation for documentation of these structures. +/// and Arrow C Stream (https://arrow.apache.org/docs/format/CStreamInterface.html) +/// interfaces are part of the +/// Arrow Columnar Format specification +/// (https://arrow.apache.org/docs/format/Columnar.html). See the Arrow documentation for +/// documentation of these structures. /// /// @{ @@ -127,8 +129,7 @@ struct ArrowArrayStream { // // Return value: 0 if successful, an `errno`-compatible error code otherwise. // - // If successful, the ArrowSchema must be released independently from the - // stream. + // If successful, the ArrowSchema must be released independently from the stream. int (*get_schema)(struct ArrowArrayStream*, struct ArrowSchema* out); // Callback to get the next array @@ -136,8 +137,7 @@ struct ArrowArrayStream { // // Return value: 0 if successful, an `errno`-compatible error code otherwise. // - // If successful, the ArrowArray must be released independently from the - // stream. + // If successful, the ArrowArray must be released independently from the stream. int (*get_next)(struct ArrowArrayStream*, struct ArrowArray* out); // Callback to get optional detailed error information. @@ -171,8 +171,7 @@ struct ArrowArrayStream { #define _NANOARROW_RETURN_NOT_OK_IMPL(NAME, EXPR) \ do { \ const int NAME = (EXPR); \ - if (NAME) \ - return NAME; \ + if (NAME) return NAME; \ } while (0) #define _NANOARROW_CHECK_RANGE(x_, min_, max_) \ @@ -182,38 +181,29 @@ struct ArrowArrayStream { NANOARROW_RETURN_NOT_OK((x_ <= max_) ? NANOARROW_OK : EINVAL) #if defined(NANOARROW_DEBUG) -#define _NANOARROW_RETURN_NOT_OK_WITH_ERROR_IMPL( \ - NAME, EXPR, ERROR_PTR_EXPR, EXPR_STR) \ - do { \ - const int NAME = (EXPR); \ - if (NAME) { \ - ArrowErrorSet( \ - (ERROR_PTR_EXPR), \ - "%s failed with errno %d\n* %s:%d", \ - EXPR_STR, \ - NAME, \ - __FILE__, \ - __LINE__); \ - return NAME; \ - } \ +#define _NANOARROW_RETURN_NOT_OK_WITH_ERROR_IMPL(NAME, EXPR, ERROR_PTR_EXPR, EXPR_STR) \ + do { \ + const int NAME = (EXPR); \ + if (NAME) { \ + ArrowErrorSet((ERROR_PTR_EXPR), "%s failed with errno %d\n* %s:%d", EXPR_STR, \ + NAME, __FILE__, __LINE__); \ + return NAME; \ + } \ } while (0) #else -#define _NANOARROW_RETURN_NOT_OK_WITH_ERROR_IMPL( \ - NAME, EXPR, ERROR_PTR_EXPR, EXPR_STR) \ - do { \ - const int NAME = (EXPR); \ - if (NAME) { \ - ArrowErrorSet( \ - (ERROR_PTR_EXPR), "%s failed with errno %d", EXPR_STR, NAME); \ - return NAME; \ - } \ +#define _NANOARROW_RETURN_NOT_OK_WITH_ERROR_IMPL(NAME, EXPR, ERROR_PTR_EXPR, EXPR_STR) \ + do { \ + const int NAME = (EXPR); \ + if (NAME) { \ + ArrowErrorSet((ERROR_PTR_EXPR), "%s failed with errno %d", EXPR_STR, NAME); \ + return NAME; \ + } \ } while (0) #endif #if defined(NANOARROW_DEBUG) // For checking ArrowErrorSet() calls for valid printf format strings/arguments -// If using mingw's c99-compliant printf, we need a different format-checking -// attribute +// If using mingw's c99-compliant printf, we need a different format-checking attribute #if defined(__USE_MINGW_ANSI_STDIO) && defined(__MINGW_PRINTF_FORMAT) #define NANOARROW_CHECK_PRINTF_ATTRIBUTE \ __attribute__((format(__MINGW_PRINTF_FORMAT, 2, 3))) @@ -253,9 +243,8 @@ typedef int ArrowErrorCode; /// \brief Flags supported by ArrowSchemaViewInit() /// \ingroup nanoarrow-schema-view -#define NANOARROW_FLAG_ALL_SUPPORTED \ - (ARROW_FLAG_DICTIONARY_ORDERED | ARROW_FLAG_NULLABLE | \ - ARROW_FLAG_MAP_KEYS_SORTED) +#define NANOARROW_FLAG_ALL_SUPPORTED \ + (ARROW_FLAG_DICTIONARY_ORDERED | ARROW_FLAG_NULLABLE | ARROW_FLAG_MAP_KEYS_SORTED) /// \brief Error type containing a UTF-8 encoded message. /// \ingroup nanoarrow-errors @@ -264,8 +253,8 @@ struct ArrowError { char message[1024]; }; -/// \brief Ensure an ArrowError is null-terminated by zeroing the first -/// character. \ingroup nanoarrow-errors +/// \brief Ensure an ArrowError is null-terminated by zeroing the first character. +/// \ingroup nanoarrow-errors /// /// If error is NULL, this function does nothing. static inline void ArrowErrorInit(struct ArrowError* error) { @@ -291,8 +280,7 @@ static inline const char* ArrowErrorMessage(struct ArrowError* error) { /// \ingroup nanoarrow-errors /// /// If error is NULL, this function does nothing. -static inline void ArrowErrorSetString( - struct ArrowError* error, const char* src) { +static inline void ArrowErrorSetString(struct ArrowError* error, const char* src) { if (error == NULL) { return; } @@ -310,34 +298,25 @@ static inline void ArrowErrorSetString( /// \brief Check the result of an expression and return it if not NANOARROW_OK /// \ingroup nanoarrow-errors #define NANOARROW_RETURN_NOT_OK(EXPR) \ - _NANOARROW_RETURN_NOT_OK_IMPL( \ - _NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), EXPR) + _NANOARROW_RETURN_NOT_OK_IMPL(_NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), EXPR) /// \brief Check the result of an expression and return it if not NANOARROW_OK, /// adding an auto-generated message to an ArrowError. /// \ingroup nanoarrow-errors /// /// This macro is used to ensure that functions that accept an ArrowError -/// as input always set its message when returning an error code (e.g., when -/// calling a nanoarrow function that does *not* accept ArrowError). +/// as input always set its message when returning an error code (e.g., when calling +/// a nanoarrow function that does *not* accept ArrowError). #define NANOARROW_RETURN_NOT_OK_WITH_ERROR(EXPR, ERROR_EXPR) \ _NANOARROW_RETURN_NOT_OK_WITH_ERROR_IMPL( \ - _NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), \ - EXPR, \ - ERROR_EXPR, \ - #EXPR) + _NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), EXPR, ERROR_EXPR, #EXPR) #if defined(NANOARROW_DEBUG) && !defined(NANOARROW_PRINT_AND_DIE) -#define NANOARROW_PRINT_AND_DIE(VALUE, EXPR_STR) \ - do { \ - fprintf( \ - stderr, \ - "%s failed with code %d\n* %s:%d\n", \ - EXPR_STR, \ - (int)(VALUE), \ - __FILE__, \ - (int)__LINE__); \ - abort(); \ +#define NANOARROW_PRINT_AND_DIE(VALUE, EXPR_STR) \ + do { \ + fprintf(stderr, "%s failed with code %d\n* %s:%d\n", EXPR_STR, (int)(VALUE), \ + __FILE__, (int)__LINE__); \ + abort(); \ } while (0) #endif @@ -345,27 +324,23 @@ static inline void ArrowErrorSetString( #define _NANOARROW_ASSERT_OK_IMPL(NAME, EXPR, EXPR_STR) \ do { \ const int NAME = (EXPR); \ - if (NAME) \ - NANOARROW_PRINT_AND_DIE(NAME, EXPR_STR); \ + if (NAME) NANOARROW_PRINT_AND_DIE(NAME, EXPR_STR); \ } while (0) /// \brief Assert that an expression's value is NANOARROW_OK /// \ingroup nanoarrow-errors /// -/// If nanoarrow was built in debug mode (i.e., defined(NANOARROW_DEBUG) is -/// true), print a message to stderr and abort. If nanoarrow was built in -/// release mode, this statement has no effect. You can customize fatal error -/// behaviour be defining the NANOARROW_PRINT_AND_DIE macro before including -/// nanoarrow.h This macro is provided as a convenience for users and is not -/// used internally. +/// If nanoarrow was built in debug mode (i.e., defined(NANOARROW_DEBUG) is true), +/// print a message to stderr and abort. If nanoarrow was built in release mode, +/// this statement has no effect. You can customize fatal error behaviour +/// be defining the NANOARROW_PRINT_AND_DIE macro before including nanoarrow.h +/// This macro is provided as a convenience for users and is not used internally. #define NANOARROW_ASSERT_OK(EXPR) \ - _NANOARROW_ASSERT_OK_IMPL( \ - _NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), EXPR, #EXPR) + _NANOARROW_ASSERT_OK_IMPL(_NANOARROW_MAKE_NAME(errno_status_, __COUNTER__), EXPR, #EXPR) -#define _NANOARROW_DCHECK_IMPL(EXPR, EXPR_STR) \ - do { \ - if (!(EXPR)) \ - NANOARROW_PRINT_AND_DIE(-1, EXPR_STR); \ +#define _NANOARROW_DCHECK_IMPL(EXPR, EXPR_STR) \ + do { \ + if (!(EXPR)) NANOARROW_PRINT_AND_DIE(-1, EXPR_STR); \ } while (0) #define NANOARROW_DCHECK(EXPR) _NANOARROW_DCHECK_IMPL(EXPR, #EXPR) @@ -374,8 +349,7 @@ static inline void ArrowErrorSetString( #define NANOARROW_DCHECK(EXPR) #endif -static inline void ArrowSchemaMove( - struct ArrowSchema* src, struct ArrowSchema* dst) { +static inline void ArrowSchemaMove(struct ArrowSchema* src, struct ArrowSchema* dst) { NANOARROW_DCHECK(src != NULL); NANOARROW_DCHECK(dst != NULL); @@ -389,8 +363,7 @@ static inline void ArrowSchemaRelease(struct ArrowSchema* schema) { NANOARROW_DCHECK(schema->release == NULL); } -static inline void ArrowArrayMove( - struct ArrowArray* src, struct ArrowArray* dst) { +static inline void ArrowArrayMove(struct ArrowArray* src, struct ArrowArray* dst) { NANOARROW_DCHECK(src != NULL); NANOARROW_DCHECK(dst != NULL); @@ -404,8 +377,8 @@ static inline void ArrowArrayRelease(struct ArrowArray* array) { NANOARROW_DCHECK(array->release == NULL); } -static inline void ArrowArrayStreamMove( - struct ArrowArrayStream* src, struct ArrowArrayStream* dst) { +static inline void ArrowArrayStreamMove(struct ArrowArrayStream* src, + struct ArrowArrayStream* dst) { NANOARROW_DCHECK(src != NULL); NANOARROW_DCHECK(dst != NULL); @@ -426,8 +399,7 @@ static inline const char* ArrowArrayStreamGetLastError( } static inline ArrowErrorCode ArrowArrayStreamGetSchema( - struct ArrowArrayStream* array_stream, - struct ArrowSchema* out, + struct ArrowArrayStream* array_stream, struct ArrowSchema* out, struct ArrowError* error) { NANOARROW_DCHECK(array_stream != NULL); @@ -440,8 +412,7 @@ static inline ArrowErrorCode ArrowArrayStreamGetSchema( } static inline ArrowErrorCode ArrowArrayStreamGetNext( - struct ArrowArrayStream* array_stream, - struct ArrowArray* out, + struct ArrowArrayStream* array_stream, struct ArrowArray* out, struct ArrowError* error) { NANOARROW_DCHECK(array_stream != NULL); @@ -453,8 +424,7 @@ static inline ArrowErrorCode ArrowArrayStreamGetNext( return result; } -static inline void ArrowArrayStreamRelease( - struct ArrowArrayStream* array_stream) { +static inline void ArrowArrayStreamRelease(struct ArrowArrayStream* array_stream) { NANOARROW_DCHECK(array_stream != NULL); array_stream->release(array_stream); NANOARROW_DCHECK(array_stream->release == NULL); @@ -622,21 +592,16 @@ enum ArrowValidationLevel { /// \brief Do not validate buffer sizes or content. NANOARROW_VALIDATION_LEVEL_NONE = 0, - /// \brief Validate buffer sizes that depend on array length but do not - /// validate buffer + /// \brief Validate buffer sizes that depend on array length but do not validate buffer /// sizes that depend on buffer data access. NANOARROW_VALIDATION_LEVEL_MINIMAL = 1, - /// \brief Validate all buffer sizes, including those that require buffer data - /// access, - /// but do not perform any checks that are O(1) along the length of the - /// buffers. + /// \brief Validate all buffer sizes, including those that require buffer data access, + /// but do not perform any checks that are O(1) along the length of the buffers. NANOARROW_VALIDATION_LEVEL_DEFAULT = 2, - /// \brief Validate all buffer sizes and all buffer content. This is useful in - /// the - /// context of untrusted input or input that may have been corrupted in - /// transit. + /// \brief Validate all buffer sizes and all buffer content. This is useful in the + /// context of untrusted input or input that may have been corrupted in transit. NANOARROW_VALIDATION_LEVEL_FULL = 3 }; @@ -661,8 +626,8 @@ static inline const char* ArrowTimeUnitString(enum ArrowTimeUnit time_unit) { } } -/// \brief Functional types of buffers as described in the Arrow Columnar -/// Specification \ingroup nanoarrow-array-view +/// \brief Functional types of buffers as described in the Arrow Columnar Specification +/// \ingroup nanoarrow-array-view enum ArrowBufferType { NANOARROW_BUFFER_TYPE_NONE, NANOARROW_BUFFER_TYPE_VALIDITY, @@ -677,8 +642,7 @@ enum ArrowBufferType { /// /// All currently supported types have 3 buffers or fewer; however, future types /// may involve a variable number of buffers (e.g., string view). These buffers -/// will be represented by separate members of the ArrowArrayView or -/// ArrowLayout. +/// will be represented by separate members of the ArrowArrayView or ArrowLayout. #define NANOARROW_MAX_FIXED_BUFFERS 3 /// \brief An non-owning view of a string @@ -747,15 +711,11 @@ struct ArrowBufferView { /// an ArrowArray. struct ArrowBufferAllocator { /// \brief Reallocate a buffer or return NULL if it cannot be reallocated - uint8_t* (*reallocate)( - struct ArrowBufferAllocator* allocator, - uint8_t* ptr, - int64_t old_size, - int64_t new_size); + uint8_t* (*reallocate)(struct ArrowBufferAllocator* allocator, uint8_t* ptr, + int64_t old_size, int64_t new_size); /// \brief Deallocate a buffer allocated by this allocator - void (*free)( - struct ArrowBufferAllocator* allocator, uint8_t* ptr, int64_t size); + void (*free)(struct ArrowBufferAllocator* allocator, uint8_t* ptr, int64_t size); /// \brief Opaque data specific to the allocator void* private_data; @@ -775,8 +735,7 @@ struct ArrowBuffer { /// \brief The capacity of the buffer in bytes int64_t capacity_bytes; - /// \brief The allocator that will be used to reallocate and/or free the - /// buffer + /// \brief The allocator that will be used to reallocate and/or free the buffer struct ArrowBufferAllocator allocator; }; @@ -803,12 +762,11 @@ struct ArrowLayout { /// \brief The data type of each buffer enum ArrowType buffer_data_type[NANOARROW_MAX_FIXED_BUFFERS]; - /// \brief The size of an element each buffer or 0 if this size is variable or - /// unknown + /// \brief The size of an element each buffer or 0 if this size is variable or unknown int64_t element_size_bits[NANOARROW_MAX_FIXED_BUFFERS]; - /// \brief The number of elements in the child array per element in this array - /// for a fixed-size list + /// \brief The number of elements in the child array per element in this array for a + /// fixed-size list int64_t child_size_elements; }; @@ -911,8 +869,8 @@ struct ArrowInterval { /// \brief Zero initialize an Interval with a given unit /// \ingroup nanoarrow-utils -static inline void ArrowIntervalInit( - struct ArrowInterval* interval, enum ArrowType type) { +static inline void ArrowIntervalInit(struct ArrowInterval* interval, + enum ArrowType type) { memset(interval, 0, sizeof(struct ArrowInterval)); interval->type = type; } @@ -924,8 +882,7 @@ static inline void ArrowIntervalInit( /// values set using ArrowDecimalSetInt(), ArrowDecimalSetBytes128(), /// or ArrowDecimalSetBytes256(). struct ArrowDecimal { - /// \brief An array of 64-bit integers of n_words length defined in - /// native-endian order + /// \brief An array of 64-bit integers of n_words length defined in native-endian order uint64_t words[4]; /// \brief The number of significant digits this decimal number can represent @@ -946,11 +903,8 @@ struct ArrowDecimal { /// \brief Initialize a decimal with a given set of type parameters /// \ingroup nanoarrow-utils -static inline void ArrowDecimalInit( - struct ArrowDecimal* decimal, - int32_t bitwidth, - int32_t precision, - int32_t scale) { +static inline void ArrowDecimalInit(struct ArrowDecimal* decimal, int32_t bitwidth, + int32_t precision, int32_t scale) { memset(decimal->words, 0, sizeof(decimal->words)); decimal->precision = precision; decimal->scale = scale; @@ -970,15 +924,14 @@ static inline void ArrowDecimalInit( /// This does not check if the decimal's precision sufficiently small to fit /// within the signed 64-bit integer range (A precision less than or equal /// to 18 is sufficiently small). -static inline int64_t ArrowDecimalGetIntUnsafe( - const struct ArrowDecimal* decimal) { +static inline int64_t ArrowDecimalGetIntUnsafe(const struct ArrowDecimal* decimal) { return (int64_t)decimal->words[decimal->low_word_index]; } /// \brief Copy the bytes of this decimal into a sufficiently large buffer /// \ingroup nanoarrow-utils -static inline void ArrowDecimalGetBytes( - const struct ArrowDecimal* decimal, uint8_t* out) { +static inline void ArrowDecimalGetBytes(const struct ArrowDecimal* decimal, + uint8_t* out) { memcpy(out, decimal->words, decimal->n_words * sizeof(uint64_t)); } @@ -990,8 +943,7 @@ static inline int64_t ArrowDecimalSign(const struct ArrowDecimal* decimal) { /// \brief Sets the integer value of this decimal /// \ingroup nanoarrow-utils -static inline void ArrowDecimalSetInt( - struct ArrowDecimal* decimal, int64_t value) { +static inline void ArrowDecimalSetInt(struct ArrowDecimal* decimal, int64_t value) { if (value < 0) { memset(decimal->words, 0xff, decimal->n_words * sizeof(uint64_t)); } else { @@ -1025,8 +977,8 @@ static inline void ArrowDecimalNegate(struct ArrowDecimal* decimal) { /// \brief Copy bytes from a buffer into this decimal /// \ingroup nanoarrow-utils -static inline void ArrowDecimalSetBytes( - struct ArrowDecimal* decimal, const uint8_t* value) { +static inline void ArrowDecimalSetBytes(struct ArrowDecimal* decimal, + const uint8_t* value) { memcpy(decimal->words, value, decimal->n_words * sizeof(uint64_t)); } @@ -1059,9 +1011,11 @@ static inline void ArrowDecimalSetBytes( #include #include -// If using CMake, optionally pass -DNANOARROW_NAMESPACE=MyNamespace which will -// set this define in nanoarrow_config.h. If not, you can optionally #define -// NANOARROW_NAMESPACE MyNamespace here. + + +// If using CMake, optionally pass -DNANOARROW_NAMESPACE=MyNamespace which will set this +// define in nanoarrow_config.h. If not, you can optionally #define NANOARROW_NAMESPACE +// MyNamespace here. // This section remaps the non-prefixed symbols to the prefixed symbols so that // code written against this build can be used independent of the value of @@ -1070,8 +1024,7 @@ static inline void ArrowDecimalSetBytes( #define NANOARROW_CAT(A, B) A##B #define NANOARROW_SYMBOL(A, B) NANOARROW_CAT(A, B) -#define ArrowNanoarrowVersion \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowNanoarrowVersion) +#define ArrowNanoarrowVersion NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowNanoarrowVersion) #define ArrowNanoarrowVersionInt \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowNanoarrowVersionInt) #define ArrowMalloc NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMalloc) @@ -1083,15 +1036,13 @@ static inline void ArrowDecimalSetBytes( NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowBufferDeallocator) #define ArrowErrorSet NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowErrorSet) #define ArrowLayoutInit NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowLayoutInit) -#define ArrowDecimalSetDigits \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowDecimalSetDigits) +#define ArrowDecimalSetDigits NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowDecimalSetDigits) #define ArrowDecimalAppendDigitsToBuffer \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowDecimalAppendDigitsToBuffer) #define ArrowSchemaInit NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaInit) #define ArrowSchemaInitFromType \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaInitFromType) -#define ArrowSchemaSetType \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetType) +#define ArrowSchemaSetType NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetType) #define ArrowSchemaSetTypeStruct \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetTypeStruct) #define ArrowSchemaSetTypeFixedSize \ @@ -1102,12 +1053,9 @@ static inline void ArrowDecimalSetBytes( NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetTypeDateTime) #define ArrowSchemaSetTypeUnion \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetTypeUnion) -#define ArrowSchemaDeepCopy \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaDeepCopy) -#define ArrowSchemaSetFormat \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetFormat) -#define ArrowSchemaSetName \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetName) +#define ArrowSchemaDeepCopy NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaDeepCopy) +#define ArrowSchemaSetFormat NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetFormat) +#define ArrowSchemaSetName NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetName) #define ArrowSchemaSetMetadata \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaSetMetadata) #define ArrowSchemaAllocateChildren \ @@ -1118,12 +1066,9 @@ static inline void ArrowDecimalSetBytes( NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataReaderInit) #define ArrowMetadataReaderRead \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataReaderRead) -#define ArrowMetadataSizeOf \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataSizeOf) -#define ArrowMetadataHasKey \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataHasKey) -#define ArrowMetadataGetValue \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataGetValue) +#define ArrowMetadataSizeOf NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataSizeOf) +#define ArrowMetadataHasKey NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataHasKey) +#define ArrowMetadataGetValue NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataGetValue) #define ArrowMetadataBuilderInit \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataBuilderInit) #define ArrowMetadataBuilderAppend \ @@ -1132,10 +1077,8 @@ static inline void ArrowDecimalSetBytes( NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataBuilderSet) #define ArrowMetadataBuilderRemove \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowMetadataBuilderRemove) -#define ArrowSchemaViewInit \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaViewInit) -#define ArrowSchemaToString \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaToString) +#define ArrowSchemaViewInit NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaViewInit) +#define ArrowSchemaToString NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowSchemaToString) #define ArrowArrayInitFromType \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayInitFromType) #define ArrowArrayInitFromSchema \ @@ -1150,10 +1093,8 @@ static inline void ArrowDecimalSetBytes( NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayAllocateDictionary) #define ArrowArraySetValidityBitmap \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArraySetValidityBitmap) -#define ArrowArraySetBuffer \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArraySetBuffer) -#define ArrowArrayReserve \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayReserve) +#define ArrowArraySetBuffer NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArraySetBuffer) +#define ArrowArrayReserve NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayReserve) #define ArrowArrayFinishBuilding \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayFinishBuilding) #define ArrowArrayFinishBuildingDefault \ @@ -1174,8 +1115,7 @@ static inline void ArrowDecimalSetBytes( NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewSetArrayMinimal) #define ArrowArrayViewValidate \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewValidate) -#define ArrowArrayViewReset \ - NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewReset) +#define ArrowArrayViewReset NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowArrayViewReset) #define ArrowBasicArrayStreamInit \ NANOARROW_SYMBOL(NANOARROW_NAMESPACE, ArrowBasicArrayStreamInit) #define ArrowBasicArrayStreamSetArray \ @@ -1229,33 +1169,31 @@ struct ArrowBufferAllocator ArrowBufferAllocatorDefault(void); /// avoid copying an existing buffer that was not allocated using the /// infrastructure provided here (e.g., by an R or Python object). struct ArrowBufferAllocator ArrowBufferDeallocator( - void (*custom_free)( - struct ArrowBufferAllocator* allocator, uint8_t* ptr, int64_t size), + void (*custom_free)(struct ArrowBufferAllocator* allocator, uint8_t* ptr, + int64_t size), void* private_data); /// @} -/// \brief Move the contents of an src ArrowSchema into dst and set src->release -/// to NULL \ingroup nanoarrow-arrow-cdata -static inline void ArrowSchemaMove( - struct ArrowSchema* src, struct ArrowSchema* dst); +/// \brief Move the contents of an src ArrowSchema into dst and set src->release to NULL +/// \ingroup nanoarrow-arrow-cdata +static inline void ArrowSchemaMove(struct ArrowSchema* src, struct ArrowSchema* dst); /// \brief Call the release callback of an ArrowSchema /// \ingroup nanoarrow-arrow-cdata static inline void ArrowSchemaRelease(struct ArrowSchema* schema); -/// \brief Move the contents of an src ArrowArray into dst and set src->release -/// to NULL \ingroup nanoarrow-arrow-cdata -static inline void ArrowArrayMove( - struct ArrowArray* src, struct ArrowArray* dst); +/// \brief Move the contents of an src ArrowArray into dst and set src->release to NULL +/// \ingroup nanoarrow-arrow-cdata +static inline void ArrowArrayMove(struct ArrowArray* src, struct ArrowArray* dst); /// \brief Call the release callback of an ArrowArray static inline void ArrowArrayRelease(struct ArrowArray* array); -/// \brief Move the contents of an src ArrowArrayStream into dst and set -/// src->release to NULL \ingroup nanoarrow-arrow-cdata -static inline void ArrowArrayStreamMove( - struct ArrowArrayStream* src, struct ArrowArrayStream* dst); +/// \brief Move the contents of an src ArrowArrayStream into dst and set src->release to +/// NULL \ingroup nanoarrow-arrow-cdata +static inline void ArrowArrayStreamMove(struct ArrowArrayStream* src, + struct ArrowArrayStream* dst); /// \brief Call the get_schema callback of an ArrowArrayStream /// \ingroup nanoarrow-arrow-cdata @@ -1265,8 +1203,7 @@ static inline void ArrowArrayStreamMove( /// makes it significantly less verbose to iterate over array streams /// using NANOARROW_RETURN_NOT_OK()-style error handling. static inline ArrowErrorCode ArrowArrayStreamGetSchema( - struct ArrowArrayStream* array_stream, - struct ArrowSchema* out, + struct ArrowArrayStream* array_stream, struct ArrowSchema* out, struct ArrowError* error); /// \brief Call the get_schema callback of an ArrowArrayStream @@ -1277,22 +1214,20 @@ static inline ArrowErrorCode ArrowArrayStreamGetSchema( /// makes it significantly less verbose to iterate over array streams /// using NANOARROW_RETURN_NOT_OK()-style error handling. static inline ArrowErrorCode ArrowArrayStreamGetNext( - struct ArrowArrayStream* array_stream, - struct ArrowArray* out, + struct ArrowArrayStream* array_stream, struct ArrowArray* out, struct ArrowError* error); /// \brief Call the get_next callback of an ArrowArrayStream /// \ingroup nanoarrow-arrow-cdata /// /// Unlike the get_next callback, this function never returns NULL (i.e., its -/// result is safe to use in printf-style error formatters). Null values from -/// the original callback are reported as "". +/// result is safe to use in printf-style error formatters). Null values from the +/// original callback are reported as "". static inline const char* ArrowArrayStreamGetLastError( struct ArrowArrayStream* array_stream); /// \brief Call the release callback of an ArrowArrayStream -static inline void ArrowArrayStreamRelease( - struct ArrowArrayStream* array_stream); +static inline void ArrowArrayStreamRelease(struct ArrowArrayStream* array_stream); /// \defgroup nanoarrow-errors Error handling /// @@ -1300,24 +1235,24 @@ static inline void ArrowArrayStreamRelease( /// need to communicate more verbose error information accept a pointer /// to an ArrowError. This can be stack or statically allocated. The /// content of the message is undefined unless an error code has been -/// returned. If a nanoarrow function is passed a non-null ArrowError pointer, -/// the ArrowError pointed to by the argument will be propagated with a +/// returned. If a nanoarrow function is passed a non-null ArrowError pointer, the +/// ArrowError pointed to by the argument will be propagated with a /// null-terminated error message. It is safe to pass a NULL ArrowError anywhere /// in the nanoarrow API. /// /// Except where documented, it is generally not safe to continue after a -/// function has returned a non-zero ArrowErrorCode. The NANOARROW_RETURN_NOT_OK -/// and NANOARROW_ASSERT_OK macros are provided to help propagate errors. C++ -/// clients can use the helpers provided in the nanoarrow.hpp header to -/// facilitate using C++ idioms for memory management and error propgagtion. +/// function has returned a non-zero ArrowErrorCode. The NANOARROW_RETURN_NOT_OK and +/// NANOARROW_ASSERT_OK macros are provided to help propagate errors. C++ clients can use +/// the helpers provided in the nanoarrow.hpp header to facilitate using C++ idioms +/// for memory management and error propgagtion. /// /// @{ /// \brief Set the contents of an error using printf syntax. /// /// If error is NULL, this function does nothing and returns NANOARROW_OK. -NANOARROW_CHECK_PRINTF_ATTRIBUTE int ArrowErrorSet( - struct ArrowError* error, const char* fmt, ...); +NANOARROW_CHECK_PRINTF_ATTRIBUTE int ArrowErrorSet(struct ArrowError* error, + const char* fmt, ...); /// @} @@ -1338,12 +1273,12 @@ void ArrowLayoutInit(struct ArrowLayout* layout, enum ArrowType storage_type); static inline struct ArrowStringView ArrowCharView(const char* value); /// \brief Sets the integer value of an ArrowDecimal from a string -ArrowErrorCode ArrowDecimalSetDigits( - struct ArrowDecimal* decimal, struct ArrowStringView value); +ArrowErrorCode ArrowDecimalSetDigits(struct ArrowDecimal* decimal, + struct ArrowStringView value); /// \brief Get the integer value of an ArrowDecimal as string -ArrowErrorCode ArrowDecimalAppendDigitsToBuffer( - const struct ArrowDecimal* decimal, struct ArrowBuffer* buffer); +ArrowErrorCode ArrowDecimalAppendDigitsToBuffer(const struct ArrowDecimal* decimal, + struct ArrowBuffer* buffer); /// @} @@ -1364,10 +1299,9 @@ void ArrowSchemaInit(struct ArrowSchema* schema); /// /// A convenience constructor for that calls ArrowSchemaInit() and /// ArrowSchemaSetType() for the common case of constructing an -/// unparameterized type. The caller is responsible for calling the -/// schema->release callback if NANOARROW_OK is returned. -ArrowErrorCode ArrowSchemaInitFromType( - struct ArrowSchema* schema, enum ArrowType type); +/// unparameterized type. The caller is responsible for calling the schema->release +/// callback if NANOARROW_OK is returned. +ArrowErrorCode ArrowSchemaInitFromType(struct ArrowSchema* schema, enum ArrowType type); /// \brief Get a human-readable summary of a Schema /// @@ -1375,8 +1309,8 @@ ArrowErrorCode ArrowSchemaInitFromType( /// and returns the number of characters required for the output if /// n were sufficiently large. If recursive is non-zero, the result will /// also include children. -int64_t ArrowSchemaToString( - const struct ArrowSchema* schema, char* out, int64_t n, char recursive); +int64_t ArrowSchemaToString(const struct ArrowSchema* schema, char* out, int64_t n, + char recursive); /// \brief Set the format field of a schema from an ArrowType /// @@ -1384,19 +1318,16 @@ int64_t ArrowSchemaToString( /// NANOARROW_TYPE_LIST, NANOARROW_TYPE_LARGE_LIST, and /// NANOARROW_TYPE_MAP, the appropriate number of children are /// allocated, initialized, and named; however, the caller must -/// ArrowSchemaSetType() on the preinitialized children. Schema must have been -/// initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy(). -ArrowErrorCode ArrowSchemaSetType( - struct ArrowSchema* schema, enum ArrowType type); +/// ArrowSchemaSetType() on the preinitialized children. Schema must have been initialized +/// using ArrowSchemaInit() or ArrowSchemaDeepCopy(). +ArrowErrorCode ArrowSchemaSetType(struct ArrowSchema* schema, enum ArrowType type); /// \brief Set the format field and initialize children of a struct schema /// -/// The specified number of children are initialized; however, the caller is -/// responsible for calling ArrowSchemaSetType() and ArrowSchemaSetName() on -/// each child. Schema must have been initialized using ArrowSchemaInit() or -/// ArrowSchemaDeepCopy(). -ArrowErrorCode ArrowSchemaSetTypeStruct( - struct ArrowSchema* schema, int64_t n_children); +/// The specified number of children are initialized; however, the caller is responsible +/// for calling ArrowSchemaSetType() and ArrowSchemaSetName() on each child. +/// Schema must have been initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy(). +ArrowErrorCode ArrowSchemaSetTypeStruct(struct ArrowSchema* schema, int64_t n_children); /// \brief Set the format field of a fixed-size schema /// @@ -1404,55 +1335,50 @@ ArrowErrorCode ArrowSchemaSetTypeStruct( /// NANOARROW_TYPE_FIXED_SIZE_BINARY or NANOARROW_TYPE_FIXED_SIZE_LIST. /// For NANOARROW_TYPE_FIXED_SIZE_LIST, the appropriate number of children are /// allocated, initialized, and named; however, the caller must -/// ArrowSchemaSetType() the first child. Schema must have been initialized -/// using ArrowSchemaInit() or ArrowSchemaDeepCopy(). -ArrowErrorCode ArrowSchemaSetTypeFixedSize( - struct ArrowSchema* schema, enum ArrowType type, int32_t fixed_size); +/// ArrowSchemaSetType() the first child. Schema must have been initialized using +/// ArrowSchemaInit() or ArrowSchemaDeepCopy(). +ArrowErrorCode ArrowSchemaSetTypeFixedSize(struct ArrowSchema* schema, + enum ArrowType type, int32_t fixed_size); /// \brief Set the format field of a decimal schema /// /// Returns EINVAL for scale <= 0 or for type that is not -/// NANOARROW_TYPE_DECIMAL128 or NANOARROW_TYPE_DECIMAL256. Schema must have -/// been initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy(). -ArrowErrorCode ArrowSchemaSetTypeDecimal( - struct ArrowSchema* schema, - enum ArrowType type, - int32_t decimal_precision, - int32_t decimal_scale); +/// NANOARROW_TYPE_DECIMAL128 or NANOARROW_TYPE_DECIMAL256. Schema must have been +/// initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy(). +ArrowErrorCode ArrowSchemaSetTypeDecimal(struct ArrowSchema* schema, enum ArrowType type, + int32_t decimal_precision, + int32_t decimal_scale); /// \brief Set the format field of a time, timestamp, or duration schema /// /// Returns EINVAL for type that is not /// NANOARROW_TYPE_TIME32, NANOARROW_TYPE_TIME64, /// NANOARROW_TYPE_TIMESTAMP, or NANOARROW_TYPE_DURATION. The -/// timezone parameter must be NULL for a non-timestamp type. Schema must have -/// been initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy(). -ArrowErrorCode ArrowSchemaSetTypeDateTime( - struct ArrowSchema* schema, - enum ArrowType type, - enum ArrowTimeUnit time_unit, - const char* timezone); +/// timezone parameter must be NULL for a non-timestamp type. Schema must have been +/// initialized using ArrowSchemaInit() or ArrowSchemaDeepCopy(). +ArrowErrorCode ArrowSchemaSetTypeDateTime(struct ArrowSchema* schema, enum ArrowType type, + enum ArrowTimeUnit time_unit, + const char* timezone); /// \brief Seet the format field of a union schema /// /// Returns EINVAL for a type that is not NANOARROW_TYPE_DENSE_UNION /// or NANOARROW_TYPE_SPARSE_UNION. The specified number of children are /// allocated, and initialized. -ArrowErrorCode ArrowSchemaSetTypeUnion( - struct ArrowSchema* schema, enum ArrowType type, int64_t n_children); +ArrowErrorCode ArrowSchemaSetTypeUnion(struct ArrowSchema* schema, enum ArrowType type, + int64_t n_children); /// \brief Make a (recursive) copy of a schema /// /// Allocates and copies fields of schema into schema_out. -ArrowErrorCode ArrowSchemaDeepCopy( - const struct ArrowSchema* schema, struct ArrowSchema* schema_out); +ArrowErrorCode ArrowSchemaDeepCopy(const struct ArrowSchema* schema, + struct ArrowSchema* schema_out); /// \brief Copy format into schema->format /// /// schema must have been allocated using ArrowSchemaInitFromType() or /// ArrowSchemaDeepCopy(). -ArrowErrorCode ArrowSchemaSetFormat( - struct ArrowSchema* schema, const char* format); +ArrowErrorCode ArrowSchemaSetFormat(struct ArrowSchema* schema, const char* format); /// \brief Copy name into schema->name /// @@ -1464,16 +1390,15 @@ ArrowErrorCode ArrowSchemaSetName(struct ArrowSchema* schema, const char* name); /// /// schema must have been allocated using ArrowSchemaInitFromType() or /// ArrowSchemaDeepCopy. -ArrowErrorCode ArrowSchemaSetMetadata( - struct ArrowSchema* schema, const char* metadata); +ArrowErrorCode ArrowSchemaSetMetadata(struct ArrowSchema* schema, const char* metadata); /// \brief Allocate the schema->children array /// /// Includes the memory for each child struct ArrowSchema. /// schema must have been allocated using ArrowSchemaInitFromType() or /// ArrowSchemaDeepCopy(). -ArrowErrorCode ArrowSchemaAllocateChildren( - struct ArrowSchema* schema, int64_t n_children); +ArrowErrorCode ArrowSchemaAllocateChildren(struct ArrowSchema* schema, + int64_t n_children); /// \brief Allocate the schema->dictionary member /// @@ -1503,14 +1428,13 @@ struct ArrowMetadataReader { }; /// \brief Initialize an ArrowMetadataReader -ArrowErrorCode ArrowMetadataReaderInit( - struct ArrowMetadataReader* reader, const char* metadata); +ArrowErrorCode ArrowMetadataReaderInit(struct ArrowMetadataReader* reader, + const char* metadata); /// \brief Read the next key/value pair from an ArrowMetadataReader -ArrowErrorCode ArrowMetadataReaderRead( - struct ArrowMetadataReader* reader, - struct ArrowStringView* key_out, - struct ArrowStringView* value_out); +ArrowErrorCode ArrowMetadataReaderRead(struct ArrowMetadataReader* reader, + struct ArrowStringView* key_out, + struct ArrowStringView* value_out); /// \brief The number of bytes in in a key/value metadata string int64_t ArrowMetadataSizeOf(const char* metadata); @@ -1521,37 +1445,32 @@ char ArrowMetadataHasKey(const char* metadata, struct ArrowStringView key); /// \brief Extract a value from schema metadata /// /// If key does not exist in metadata, value_out is unmodified -ArrowErrorCode ArrowMetadataGetValue( - const char* metadata, - struct ArrowStringView key, - struct ArrowStringView* value_out); +ArrowErrorCode ArrowMetadataGetValue(const char* metadata, struct ArrowStringView key, + struct ArrowStringView* value_out); /// \brief Initialize a builder for schema metadata from key/value pairs /// /// metadata can be an existing metadata string or NULL to initialize /// an empty metadata string. -ArrowErrorCode ArrowMetadataBuilderInit( - struct ArrowBuffer* buffer, const char* metadata); +ArrowErrorCode ArrowMetadataBuilderInit(struct ArrowBuffer* buffer, const char* metadata); /// \brief Append a key/value pair to a buffer containing serialized metadata -ArrowErrorCode ArrowMetadataBuilderAppend( - struct ArrowBuffer* buffer, - struct ArrowStringView key, - struct ArrowStringView value); +ArrowErrorCode ArrowMetadataBuilderAppend(struct ArrowBuffer* buffer, + struct ArrowStringView key, + struct ArrowStringView value); /// \brief Set a key/value pair to a buffer containing serialized metadata /// /// Ensures that the only entry for key in the metadata is set to value. /// This function maintains the existing position of (the first instance of) /// key if present in the data. -ArrowErrorCode ArrowMetadataBuilderSet( - struct ArrowBuffer* buffer, - struct ArrowStringView key, - struct ArrowStringView value); +ArrowErrorCode ArrowMetadataBuilderSet(struct ArrowBuffer* buffer, + struct ArrowStringView key, + struct ArrowStringView value); /// \brief Remove a key from a buffer containing serialized metadata -ArrowErrorCode ArrowMetadataBuilderRemove( - struct ArrowBuffer* buffer, struct ArrowStringView key); +ArrowErrorCode ArrowMetadataBuilderRemove(struct ArrowBuffer* buffer, + struct ArrowStringView key); /// @} @@ -1580,9 +1499,9 @@ struct ArrowSchemaView { /// \brief The storage data type represented by the schema /// - /// This value will never be NANOARROW_TYPE_DICTIONARY, - /// NANOARROW_TYPE_EXTENSION or any datetime type. This value represents only - /// the type required to interpret the buffers in the array. + /// This value will never be NANOARROW_TYPE_DICTIONARY, NANOARROW_TYPE_EXTENSION + /// or any datetime type. This value represents only the type required to + /// interpret the buffers in the array. enum ArrowType storage_type; /// \brief The storage layout represented by the schema @@ -1649,10 +1568,9 @@ struct ArrowSchemaView { }; /// \brief Initialize an ArrowSchemaView -ArrowErrorCode ArrowSchemaViewInit( - struct ArrowSchemaView* schema_view, - const struct ArrowSchema* schema, - struct ArrowError* error); +ArrowErrorCode ArrowSchemaViewInit(struct ArrowSchemaView* schema_view, + const struct ArrowSchema* schema, + struct ArrowError* error); /// @} @@ -1683,8 +1601,7 @@ static inline void ArrowBufferReset(struct ArrowBuffer* buffer); /// /// Transfers the buffer data and lifecycle management to another /// address and resets buffer. -static inline void ArrowBufferMove( - struct ArrowBuffer* src, struct ArrowBuffer* dst); +static inline void ArrowBufferMove(struct ArrowBuffer* src, struct ArrowBuffer* dst); /// \brief Grow or shrink a buffer to a given capacity /// @@ -1692,84 +1609,85 @@ static inline void ArrowBufferMove( /// if shrink_to_fit is non-zero. Calling ArrowBufferResize() does not /// adjust the buffer's size member except to ensure that the invariant /// capacity >= size remains true. -static inline ArrowErrorCode ArrowBufferResize( - struct ArrowBuffer* buffer, int64_t new_capacity_bytes, char shrink_to_fit); +static inline ArrowErrorCode ArrowBufferResize(struct ArrowBuffer* buffer, + int64_t new_capacity_bytes, + char shrink_to_fit); /// \brief Ensure a buffer has at least a given additional capacity /// /// Ensures that the buffer has space to append at least /// additional_size_bytes, overallocating when required. -static inline ArrowErrorCode ArrowBufferReserve( - struct ArrowBuffer* buffer, int64_t additional_size_bytes); +static inline ArrowErrorCode ArrowBufferReserve(struct ArrowBuffer* buffer, + int64_t additional_size_bytes); /// \brief Write data to buffer and increment the buffer size /// /// This function does not check that buffer has the required capacity -static inline void ArrowBufferAppendUnsafe( - struct ArrowBuffer* buffer, const void* data, int64_t size_bytes); +static inline void ArrowBufferAppendUnsafe(struct ArrowBuffer* buffer, const void* data, + int64_t size_bytes); /// \brief Write data to buffer and increment the buffer size /// /// This function writes and ensures that the buffer has the required capacity, /// possibly by reallocating the buffer. Like ArrowBufferReserve, this will /// overallocate when reallocation is required. -static inline ArrowErrorCode ArrowBufferAppend( - struct ArrowBuffer* buffer, const void* data, int64_t size_bytes); +static inline ArrowErrorCode ArrowBufferAppend(struct ArrowBuffer* buffer, + const void* data, int64_t size_bytes); /// \brief Write fill to buffer and increment the buffer size /// /// This function writes the specified number of fill bytes and /// ensures that the buffer has the required capacity, -static inline ArrowErrorCode ArrowBufferAppendFill( - struct ArrowBuffer* buffer, uint8_t value, int64_t size_bytes); +static inline ArrowErrorCode ArrowBufferAppendFill(struct ArrowBuffer* buffer, + uint8_t value, int64_t size_bytes); /// \brief Write an 8-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendInt8( - struct ArrowBuffer* buffer, int8_t value); +static inline ArrowErrorCode ArrowBufferAppendInt8(struct ArrowBuffer* buffer, + int8_t value); /// \brief Write an unsigned 8-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendUInt8( - struct ArrowBuffer* buffer, uint8_t value); +static inline ArrowErrorCode ArrowBufferAppendUInt8(struct ArrowBuffer* buffer, + uint8_t value); /// \brief Write a 16-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendInt16( - struct ArrowBuffer* buffer, int16_t value); +static inline ArrowErrorCode ArrowBufferAppendInt16(struct ArrowBuffer* buffer, + int16_t value); /// \brief Write an unsigned 16-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendUInt16( - struct ArrowBuffer* buffer, uint16_t value); +static inline ArrowErrorCode ArrowBufferAppendUInt16(struct ArrowBuffer* buffer, + uint16_t value); /// \brief Write a 32-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendInt32( - struct ArrowBuffer* buffer, int32_t value); +static inline ArrowErrorCode ArrowBufferAppendInt32(struct ArrowBuffer* buffer, + int32_t value); /// \brief Write an unsigned 32-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendUInt32( - struct ArrowBuffer* buffer, uint32_t value); +static inline ArrowErrorCode ArrowBufferAppendUInt32(struct ArrowBuffer* buffer, + uint32_t value); /// \brief Write a 64-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendInt64( - struct ArrowBuffer* buffer, int64_t value); +static inline ArrowErrorCode ArrowBufferAppendInt64(struct ArrowBuffer* buffer, + int64_t value); /// \brief Write an unsigned 64-bit integer to a buffer -static inline ArrowErrorCode ArrowBufferAppendUInt64( - struct ArrowBuffer* buffer, uint64_t value); +static inline ArrowErrorCode ArrowBufferAppendUInt64(struct ArrowBuffer* buffer, + uint64_t value); /// \brief Write a double to a buffer -static inline ArrowErrorCode ArrowBufferAppendDouble( - struct ArrowBuffer* buffer, double value); +static inline ArrowErrorCode ArrowBufferAppendDouble(struct ArrowBuffer* buffer, + double value); /// \brief Write a float to a buffer -static inline ArrowErrorCode ArrowBufferAppendFloat( - struct ArrowBuffer* buffer, float value); +static inline ArrowErrorCode ArrowBufferAppendFloat(struct ArrowBuffer* buffer, + float value); /// \brief Write an ArrowStringView to a buffer -static inline ArrowErrorCode ArrowBufferAppendStringView( - struct ArrowBuffer* buffer, struct ArrowStringView value); +static inline ArrowErrorCode ArrowBufferAppendStringView(struct ArrowBuffer* buffer, + struct ArrowStringView value); /// \brief Write an ArrowBufferView to a buffer -static inline ArrowErrorCode ArrowBufferAppendBufferView( - struct ArrowBuffer* buffer, struct ArrowBufferView value); +static inline ArrowErrorCode ArrowBufferAppendBufferView(struct ArrowBuffer* buffer, + struct ArrowBufferView value); /// @} @@ -1790,20 +1708,19 @@ static inline void ArrowBitClear(uint8_t* bits, int64_t i); static inline void ArrowBitSetTo(uint8_t* bits, int64_t i, uint8_t value); /// \brief Set a boolean value to a range in a bitmap -static inline void ArrowBitsSetTo( - uint8_t* bits, int64_t start_offset, int64_t length, uint8_t bits_are_set); +static inline void ArrowBitsSetTo(uint8_t* bits, int64_t start_offset, int64_t length, + uint8_t bits_are_set); /// \brief Count true values in a bitmap -static inline int64_t ArrowBitCountSet( - const uint8_t* bits, int64_t i_from, int64_t i_to); +static inline int64_t ArrowBitCountSet(const uint8_t* bits, int64_t i_from, int64_t i_to); /// \brief Extract int8 boolean values from a range in a bitmap -static inline void ArrowBitsUnpackInt8( - const uint8_t* bits, int64_t start_offset, int64_t length, int8_t* out); +static inline void ArrowBitsUnpackInt8(const uint8_t* bits, int64_t start_offset, + int64_t length, int8_t* out); /// \brief Extract int32 boolean values from a range in a bitmap -static inline void ArrowBitsUnpackInt32( - const uint8_t* bits, int64_t start_offset, int64_t length, int32_t* out); +static inline void ArrowBitsUnpackInt32(const uint8_t* bits, int64_t start_offset, + int64_t length, int32_t* out); /// \brief Initialize an ArrowBitmap /// @@ -1814,15 +1731,14 @@ static inline void ArrowBitmapInit(struct ArrowBitmap* bitmap); /// /// Transfers the underlying buffer data and lifecycle management to another /// address and resets the bitmap. -static inline void ArrowBitmapMove( - struct ArrowBitmap* src, struct ArrowBitmap* dst); +static inline void ArrowBitmapMove(struct ArrowBitmap* src, struct ArrowBitmap* dst); /// \brief Ensure a bitmap builder has at least a given additional capacity /// /// Ensures that the buffer has space to append at least /// additional_size_bits, overallocating when required. -static inline ArrowErrorCode ArrowBitmapReserve( - struct ArrowBitmap* bitmap, int64_t additional_size_bits); +static inline ArrowErrorCode ArrowBitmapReserve(struct ArrowBitmap* bitmap, + int64_t additional_size_bits); /// \brief Grow or shrink a bitmap to a given capacity /// @@ -1830,34 +1746,33 @@ static inline ArrowErrorCode ArrowBitmapReserve( /// if shrink_to_fit is non-zero. Calling ArrowBitmapResize() does not /// adjust the buffer's size member except when shrinking new_capacity_bits /// to a value less than the current number of bits in the bitmap. -static inline ArrowErrorCode ArrowBitmapResize( - struct ArrowBitmap* bitmap, int64_t new_capacity_bits, char shrink_to_fit); +static inline ArrowErrorCode ArrowBitmapResize(struct ArrowBitmap* bitmap, + int64_t new_capacity_bits, + char shrink_to_fit); -/// \brief Reserve space for and append zero or more of the same boolean value -/// to a bitmap -static inline ArrowErrorCode ArrowBitmapAppend( - struct ArrowBitmap* bitmap, uint8_t bits_are_set, int64_t length); +/// \brief Reserve space for and append zero or more of the same boolean value to a bitmap +static inline ArrowErrorCode ArrowBitmapAppend(struct ArrowBitmap* bitmap, + uint8_t bits_are_set, int64_t length); /// \brief Append zero or more of the same boolean value to a bitmap -static inline void ArrowBitmapAppendUnsafe( - struct ArrowBitmap* bitmap, uint8_t bits_are_set, int64_t length); +static inline void ArrowBitmapAppendUnsafe(struct ArrowBitmap* bitmap, + uint8_t bits_are_set, int64_t length); /// \brief Append boolean values encoded as int8_t to a bitmap /// /// The values must all be 0 or 1. -static inline void ArrowBitmapAppendInt8Unsafe( - struct ArrowBitmap* bitmap, const int8_t* values, int64_t n_values); +static inline void ArrowBitmapAppendInt8Unsafe(struct ArrowBitmap* bitmap, + const int8_t* values, int64_t n_values); /// \brief Append boolean values encoded as int32_t to a bitmap /// /// The values must all be 0 or 1. -static inline void ArrowBitmapAppendInt32Unsafe( - struct ArrowBitmap* bitmap, const int32_t* values, int64_t n_values); +static inline void ArrowBitmapAppendInt32Unsafe(struct ArrowBitmap* bitmap, + const int32_t* values, int64_t n_values); /// \brief Reset a bitmap builder /// -/// Releases any memory held by buffer, empties the cache, and resets the size -/// to zero +/// Releases any memory held by buffer, empties the cache, and resets the size to zero static inline void ArrowBitmapReset(struct ArrowBitmap* bitmap); /// @} @@ -1876,26 +1791,24 @@ static inline void ArrowBitmapReset(struct ArrowBitmap* bitmap); /// Initializes the fields and release callback of array. Caller /// is responsible for calling the array->release callback if /// NANOARROW_OK is returned. -ArrowErrorCode ArrowArrayInitFromType( - struct ArrowArray* array, enum ArrowType storage_type); +ArrowErrorCode ArrowArrayInitFromType(struct ArrowArray* array, + enum ArrowType storage_type); /// \brief Initialize the contents of an ArrowArray from an ArrowSchema /// /// Caller is responsible for calling the array->release callback if /// NANOARROW_OK is returned. -ArrowErrorCode ArrowArrayInitFromSchema( - struct ArrowArray* array, - const struct ArrowSchema* schema, - struct ArrowError* error); +ArrowErrorCode ArrowArrayInitFromSchema(struct ArrowArray* array, + const struct ArrowSchema* schema, + struct ArrowError* error); /// \brief Initialize the contents of an ArrowArray from an ArrowArrayView /// /// Caller is responsible for calling the array->release callback if /// NANOARROW_OK is returned. -ArrowErrorCode ArrowArrayInitFromArrayView( - struct ArrowArray* array, - const struct ArrowArrayView* array_view, - struct ArrowError* error); +ArrowErrorCode ArrowArrayInitFromArrayView(struct ArrowArray* array, + const struct ArrowArrayView* array_view, + struct ArrowError* error); /// \brief Allocate the array->children array /// @@ -1903,8 +1816,7 @@ ArrowErrorCode ArrowArrayInitFromArrayView( /// whose members are marked as released and may be subsequently initialized /// with ArrowArrayInitFromType() or moved from an existing ArrowArray. /// schema must have been allocated using ArrowArrayInitFromType(). -ArrowErrorCode ArrowArrayAllocateChildren( - struct ArrowArray* array, int64_t n_children); +ArrowErrorCode ArrowArrayAllocateChildren(struct ArrowArray* array, int64_t n_children); /// \brief Allocate the array->dictionary member /// @@ -1917,33 +1829,30 @@ ArrowErrorCode ArrowArrayAllocateDictionary(struct ArrowArray* array); /// \brief Set the validity bitmap of an ArrowArray /// /// array must have been allocated using ArrowArrayInitFromType() -void ArrowArraySetValidityBitmap( - struct ArrowArray* array, struct ArrowBitmap* bitmap); +void ArrowArraySetValidityBitmap(struct ArrowArray* array, struct ArrowBitmap* bitmap); /// \brief Set a buffer of an ArrowArray /// /// array must have been allocated using ArrowArrayInitFromType() -ArrowErrorCode ArrowArraySetBuffer( - struct ArrowArray* array, int64_t i, struct ArrowBuffer* buffer); +ArrowErrorCode ArrowArraySetBuffer(struct ArrowArray* array, int64_t i, + struct ArrowBuffer* buffer); /// \brief Get the validity bitmap of an ArrowArray /// /// array must have been allocated using ArrowArrayInitFromType() -static inline struct ArrowBitmap* ArrowArrayValidityBitmap( - struct ArrowArray* array); +static inline struct ArrowBitmap* ArrowArrayValidityBitmap(struct ArrowArray* array); /// \brief Get a buffer of an ArrowArray /// /// array must have been allocated using ArrowArrayInitFromType() -static inline struct ArrowBuffer* ArrowArrayBuffer( - struct ArrowArray* array, int64_t i); +static inline struct ArrowBuffer* ArrowArrayBuffer(struct ArrowArray* array, int64_t i); /// \brief Start element-wise appending to an ArrowArray /// /// Initializes any values needed to use ArrowArrayAppend*() functions. -/// All element-wise appenders append by value and return EINVAL if the exact -/// value cannot be represented by the underlying storage type. array must have -/// been allocated using ArrowArrayInitFromType() +/// All element-wise appenders append by value and return EINVAL if the exact value +/// cannot be represented by the underlying storage type. +/// array must have been allocated using ArrowArrayInitFromType() static inline ArrowErrorCode ArrowArrayStartAppending(struct ArrowArray* array); /// \brief Reserve space for future appends @@ -1952,32 +1861,29 @@ static inline ArrowErrorCode ArrowArrayStartAppending(struct ArrowArray* array); /// child array sizes for non-fixed-size arrays), recursively reserve space for /// additional elements. This is useful for reducing the number of reallocations /// that occur using the item-wise appenders. -ArrowErrorCode ArrowArrayReserve( - struct ArrowArray* array, int64_t additional_size_elements); +ArrowErrorCode ArrowArrayReserve(struct ArrowArray* array, + int64_t additional_size_elements); /// \brief Append a null value to an array -static inline ArrowErrorCode ArrowArrayAppendNull( - struct ArrowArray* array, int64_t n); +static inline ArrowErrorCode ArrowArrayAppendNull(struct ArrowArray* array, int64_t n); /// \brief Append an empty, non-null value to an array -static inline ArrowErrorCode ArrowArrayAppendEmpty( - struct ArrowArray* array, int64_t n); +static inline ArrowErrorCode ArrowArrayAppendEmpty(struct ArrowArray* array, int64_t n); /// \brief Append a signed integer value to an array /// /// Returns NANOARROW_OK if value can be exactly represented by /// the underlying storage type or EINVAL otherwise (e.g., value /// is outside the valid array range). -static inline ArrowErrorCode ArrowArrayAppendInt( - struct ArrowArray* array, int64_t value); +static inline ArrowErrorCode ArrowArrayAppendInt(struct ArrowArray* array, int64_t value); /// \brief Append an unsigned integer value to an array /// /// Returns NANOARROW_OK if value can be exactly represented by /// the underlying storage type or EINVAL otherwise (e.g., value /// is outside the valid array range). -static inline ArrowErrorCode ArrowArrayAppendUInt( - struct ArrowArray* array, uint64_t value); +static inline ArrowErrorCode ArrowArrayAppendUInt(struct ArrowArray* array, + uint64_t value); /// \brief Append a double value to an array /// @@ -1985,68 +1891,67 @@ static inline ArrowErrorCode ArrowArrayAppendUInt( /// the underlying storage type or EINVAL otherwise (e.g., value /// is outside the valid array range or there is an attempt to append /// a non-integer to an array with an integer storage type). -static inline ArrowErrorCode ArrowArrayAppendDouble( - struct ArrowArray* array, double value); +static inline ArrowErrorCode ArrowArrayAppendDouble(struct ArrowArray* array, + double value); /// \brief Append a string of bytes to an array /// /// Returns NANOARROW_OK if value can be exactly represented by /// the underlying storage type, EOVERFLOW if appending value would overflow /// the offset type (e.g., if the data buffer would be larger than 2 GB for a -/// non-large string type), or EINVAL otherwise (e.g., the underlying array is -/// not a binary, string, large binary, large string, or fixed-size binary -/// array, or value is the wrong size for a fixed-size binary array). -static inline ArrowErrorCode ArrowArrayAppendBytes( - struct ArrowArray* array, struct ArrowBufferView value); +/// non-large string type), or EINVAL otherwise (e.g., the underlying array is not a +/// binary, string, large binary, large string, or fixed-size binary array, or value is +/// the wrong size for a fixed-size binary array). +static inline ArrowErrorCode ArrowArrayAppendBytes(struct ArrowArray* array, + struct ArrowBufferView value); /// \brief Append a string value to an array /// /// Returns NANOARROW_OK if value can be exactly represented by /// the underlying storage type, EOVERFLOW if appending value would overflow /// the offset type (e.g., if the data buffer would be larger than 2 GB for a -/// non-large string type), or EINVAL otherwise (e.g., the underlying array is -/// not a string or large string array). -static inline ArrowErrorCode ArrowArrayAppendString( - struct ArrowArray* array, struct ArrowStringView value); +/// non-large string type), or EINVAL otherwise (e.g., the underlying array is not a +/// string or large string array). +static inline ArrowErrorCode ArrowArrayAppendString(struct ArrowArray* array, + struct ArrowStringView value); /// \brief Append a Interval to an array /// /// Returns NANOARROW_OK if value can be exactly represented by /// the underlying storage type or EINVAL otherwise. -static inline ArrowErrorCode ArrowArrayAppendInterval( - struct ArrowArray* array, const struct ArrowInterval* value); +static inline ArrowErrorCode ArrowArrayAppendInterval(struct ArrowArray* array, + const struct ArrowInterval* value); /// \brief Append a decimal value to an array /// /// Returns NANOARROW_OK if array is a decimal array with the appropriate /// bitwidth or EINVAL otherwise. -static inline ArrowErrorCode ArrowArrayAppendDecimal( - struct ArrowArray* array, const struct ArrowDecimal* value); +static inline ArrowErrorCode ArrowArrayAppendDecimal(struct ArrowArray* array, + const struct ArrowDecimal* value); /// \brief Finish a nested array element /// /// Appends a non-null element to the array based on the first child's current /// length. Returns NANOARROW_OK if the item was successfully added, EOVERFLOW /// if the child of a list or map array would exceed INT_MAX elements, or EINVAL -/// if the underlying storage type is not a struct, list, large list, or -/// fixed-size list, or if there was an attempt to add a struct or fixed-size -/// list element where the length of the child array(s) did not match the -/// expected length. +/// if the underlying storage type is not a struct, list, large list, or fixed-size +/// list, or if there was an attempt to add a struct or fixed-size list element where the +/// length of the child array(s) did not match the expected length. static inline ArrowErrorCode ArrowArrayFinishElement(struct ArrowArray* array); /// \brief Finish a union array element /// -/// Appends an element to the union type ids buffer and increments -/// array->length. For sparse unions, up to one element is added to non type-id -/// children. Returns EINVAL if the underlying storage type is not a union, if -/// type_id is not valid, or if child sizes after appending are inconsistent. -static inline ArrowErrorCode ArrowArrayFinishUnionElement( - struct ArrowArray* array, int8_t type_id); +/// Appends an element to the union type ids buffer and increments array->length. +/// For sparse unions, up to one element is added to non type-id children. Returns +/// EINVAL if the underlying storage type is not a union, if type_id is not valid, +/// or if child sizes after appending are inconsistent. +static inline ArrowErrorCode ArrowArrayFinishUnionElement(struct ArrowArray* array, + int8_t type_id); /// \brief Shrink buffer capacity to the size required /// -/// Also applies shrinking to any child arrays. array must have been allocated -/// using ArrowArrayInitFromType +/// Also applies shrinking to any child arrays. array must have been allocated using +/// ArrowArrayInitFromType static inline ArrowErrorCode ArrowArrayShrinkToFit(struct ArrowArray* array); /// \brief Finish building an ArrowArray @@ -2055,20 +1960,19 @@ static inline ArrowErrorCode ArrowArrayShrinkToFit(struct ArrowArray* array); /// into array->buffers and checks the actual size of the buffers /// against the expected size based on the final length. /// array must have been allocated using ArrowArrayInitFromType() -ArrowErrorCode ArrowArrayFinishBuildingDefault( - struct ArrowArray* array, struct ArrowError* error); +ArrowErrorCode ArrowArrayFinishBuildingDefault(struct ArrowArray* array, + struct ArrowError* error); /// \brief Finish building an ArrowArray with explicit validation /// -/// Finish building with an explicit validation level. This could perform less -/// validation (i.e. NANOARROW_VALIDATION_LEVEL_NONE or -/// NANOARROW_VALIDATION_LEVEL_MINIMAL) if CPU buffer data access is not -/// possible or more validation (i.e., NANOARROW_VALIDATION_LEVEL_FULL) if -/// buffer content was obtained from an untrusted or corruptible source. -ArrowErrorCode ArrowArrayFinishBuilding( - struct ArrowArray* array, - enum ArrowValidationLevel validation_level, - struct ArrowError* error); +/// Finish building with an explicit validation level. This could perform less validation +/// (i.e. NANOARROW_VALIDATION_LEVEL_NONE or NANOARROW_VALIDATION_LEVEL_MINIMAL) if CPU +/// buffer data access is not possible or more validation (i.e., +/// NANOARROW_VALIDATION_LEVEL_FULL) if buffer content was obtained from an untrusted or +/// corruptible source. +ArrowErrorCode ArrowArrayFinishBuilding(struct ArrowArray* array, + enum ArrowValidationLevel validation_level, + struct ArrowError* error); /// @} @@ -2079,71 +1983,66 @@ ArrowErrorCode ArrowArrayFinishBuilding( /// @{ /// \brief Initialize the contents of an ArrowArrayView -void ArrowArrayViewInitFromType( - struct ArrowArrayView* array_view, enum ArrowType storage_type); +void ArrowArrayViewInitFromType(struct ArrowArrayView* array_view, + enum ArrowType storage_type); /// \brief Move an ArrowArrayView /// /// Transfers the ArrowArrayView data and lifecycle management to another /// address and resets the contents of src. -static inline void ArrowArrayViewMove( - struct ArrowArrayView* src, struct ArrowArrayView* dst); +static inline void ArrowArrayViewMove(struct ArrowArrayView* src, + struct ArrowArrayView* dst); /// \brief Initialize the contents of an ArrowArrayView from an ArrowSchema -ArrowErrorCode ArrowArrayViewInitFromSchema( - struct ArrowArrayView* array_view, - const struct ArrowSchema* schema, - struct ArrowError* error); +ArrowErrorCode ArrowArrayViewInitFromSchema(struct ArrowArrayView* array_view, + const struct ArrowSchema* schema, + struct ArrowError* error); /// \brief Allocate the array_view->children array /// /// Includes the memory for each child struct ArrowArrayView -ArrowErrorCode ArrowArrayViewAllocateChildren( - struct ArrowArrayView* array_view, int64_t n_children); +ArrowErrorCode ArrowArrayViewAllocateChildren(struct ArrowArrayView* array_view, + int64_t n_children); /// \brief Allocate array_view->dictionary -ArrowErrorCode ArrowArrayViewAllocateDictionary( - struct ArrowArrayView* array_view); +ArrowErrorCode ArrowArrayViewAllocateDictionary(struct ArrowArrayView* array_view); /// \brief Set data-independent buffer sizes from length void ArrowArrayViewSetLength(struct ArrowArrayView* array_view, int64_t length); /// \brief Set buffer sizes and data pointers from an ArrowArray -ArrowErrorCode ArrowArrayViewSetArray( - struct ArrowArrayView* array_view, - const struct ArrowArray* array, - struct ArrowError* error); +ArrowErrorCode ArrowArrayViewSetArray(struct ArrowArrayView* array_view, + const struct ArrowArray* array, + struct ArrowError* error); -/// \brief Set buffer sizes and data pointers from an ArrowArray except for -/// those that require dereferencing buffer content. -ArrowErrorCode ArrowArrayViewSetArrayMinimal( - struct ArrowArrayView* array_view, - const struct ArrowArray* array, - struct ArrowError* error); +/// \brief Set buffer sizes and data pointers from an ArrowArray except for those +/// that require dereferencing buffer content. +ArrowErrorCode ArrowArrayViewSetArrayMinimal(struct ArrowArrayView* array_view, + const struct ArrowArray* array, + struct ArrowError* error); /// \brief Performs checks on the content of an ArrowArrayView /// /// If using ArrowArrayViewSetArray() to back array_view with an ArrowArray, /// the buffer sizes and some content (fist and last offset) have already /// been validated at the "default" level. If setting the buffer pointers -/// and sizes otherwise, you may wish to perform checks at a different level. -/// See documentation for ArrowValidationLevel for the details of checks -/// performed at each level. -ArrowErrorCode ArrowArrayViewValidate( - struct ArrowArrayView* array_view, - enum ArrowValidationLevel validation_level, - struct ArrowError* error); +/// and sizes otherwise, you may wish to perform checks at a different level. See +/// documentation for ArrowValidationLevel for the details of checks performed +/// at each level. +ArrowErrorCode ArrowArrayViewValidate(struct ArrowArrayView* array_view, + enum ArrowValidationLevel validation_level, + struct ArrowError* error); /// \brief Reset the contents of an ArrowArrayView and frees resources void ArrowArrayViewReset(struct ArrowArrayView* array_view); /// \brief Check for a null element in an ArrowArrayView -static inline int8_t ArrowArrayViewIsNull( - const struct ArrowArrayView* array_view, int64_t i); +static inline int8_t ArrowArrayViewIsNull(const struct ArrowArrayView* array_view, + int64_t i); /// \brief Get the type id of a union array element -static inline int8_t ArrowArrayViewUnionTypeId( - const struct ArrowArrayView* array_view, int64_t i); +static inline int8_t ArrowArrayViewUnionTypeId(const struct ArrowArrayView* array_view, + int64_t i); /// \brief Get the child index of a union array element static inline int8_t ArrowArrayViewUnionChildIndex( @@ -2155,15 +2054,15 @@ static inline int64_t ArrowArrayViewUnionChildOffset( /// \brief Get an element in an ArrowArrayView as an integer /// -/// This function does not check for null values, that values are actually -/// integers, or that values are within a valid range for an int64. -static inline int64_t ArrowArrayViewGetIntUnsafe( - const struct ArrowArrayView* array_view, int64_t i); +/// This function does not check for null values, that values are actually integers, or +/// that values are within a valid range for an int64. +static inline int64_t ArrowArrayViewGetIntUnsafe(const struct ArrowArrayView* array_view, + int64_t i); /// \brief Get an element in an ArrowArrayView as an unsigned integer /// -/// This function does not check for null values, that values are actually -/// integers, or that values are within a valid range for a uint64. +/// This function does not check for null values, that values are actually integers, or +/// that values are within a valid range for a uint64. static inline uint64_t ArrowArrayViewGetUIntUnsafe( const struct ArrowArrayView* array_view, int64_t i); @@ -2191,10 +2090,8 @@ static inline struct ArrowBufferView ArrowArrayViewGetBytesUnsafe( /// This function does not check for null values. The out parameter must /// be initialized with ArrowDecimalInit() with the proper parameters for this /// type before calling this for the first time. -static inline void ArrowArrayViewGetDecimalUnsafe( - const struct ArrowArrayView* array_view, - int64_t i, - struct ArrowDecimal* out); +static inline void ArrowArrayViewGetDecimalUnsafe(const struct ArrowArrayView* array_view, + int64_t i, struct ArrowDecimal* out); /// @} @@ -2212,10 +2109,8 @@ static inline void ArrowArrayViewGetDecimalUnsafe( /// This function moves the ownership of schema to the array_stream. If /// this function returns NANOARROW_OK, the caller is responsible for /// releasing the ArrowArrayStream. -ArrowErrorCode ArrowBasicArrayStreamInit( - struct ArrowArrayStream* array_stream, - struct ArrowSchema* schema, - int64_t n_arrays); +ArrowErrorCode ArrowBasicArrayStreamInit(struct ArrowArrayStream* array_stream, + struct ArrowSchema* schema, int64_t n_arrays); /// \brief Set the ith ArrowArray in this ArrowArrayStream. /// @@ -2224,27 +2119,29 @@ ArrowErrorCode ArrowBasicArrayStreamInit( /// be greater than zero and less than the value of n_arrays passed in /// ArrowBasicArrayStreamInit(). Callers are not required to fill all /// n_arrays members (i.e., n_arrays is a maximum bound). -void ArrowBasicArrayStreamSetArray( - struct ArrowArrayStream* array_stream, int64_t i, struct ArrowArray* array); +void ArrowBasicArrayStreamSetArray(struct ArrowArrayStream* array_stream, int64_t i, + struct ArrowArray* array); /// \brief Validate the contents of this ArrowArrayStream /// /// array_stream must have been initialized with ArrowBasicArrayStreamInit(). -/// This function uses ArrowArrayStreamInitFromSchema() and -/// ArrowArrayStreamSetArray() to validate the contents of the arrays. -ArrowErrorCode ArrowBasicArrayStreamValidate( - const struct ArrowArrayStream* array_stream, struct ArrowError* error); +/// This function uses ArrowArrayStreamInitFromSchema() and ArrowArrayStreamSetArray() +/// to validate the contents of the arrays. +ArrowErrorCode ArrowBasicArrayStreamValidate(const struct ArrowArrayStream* array_stream, + struct ArrowError* error); /// @} -// Undefine ArrowErrorCode, which may have been defined to annotate functions -// that return it to warn for an unused result. +// Undefine ArrowErrorCode, which may have been defined to annotate functions that return +// it to warn for an unused result. #if defined(ArrowErrorCode) #undef ArrowErrorCode #endif // Inline function definitions + + #ifdef __cplusplus } #endif @@ -2274,12 +2171,13 @@ ArrowErrorCode ArrowBasicArrayStreamValidate( #include #include + + #ifdef __cplusplus extern "C" { #endif -static inline int64_t _ArrowGrowByFactor( - int64_t current_capacity, int64_t new_capacity) { +static inline int64_t _ArrowGrowByFactor(int64_t current_capacity, int64_t new_capacity) { int64_t doubled_capacity = current_capacity * 2; if (doubled_capacity > new_capacity) { return doubled_capacity; @@ -2307,8 +2205,8 @@ static inline ArrowErrorCode ArrowBufferSetAllocator( static inline void ArrowBufferReset(struct ArrowBuffer* buffer) { if (buffer->data != NULL) { - buffer->allocator.free( - &buffer->allocator, (uint8_t*)buffer->data, buffer->capacity_bytes); + buffer->allocator.free(&buffer->allocator, (uint8_t*)buffer->data, + buffer->capacity_bytes); buffer->data = NULL; } @@ -2316,27 +2214,22 @@ static inline void ArrowBufferReset(struct ArrowBuffer* buffer) { buffer->size_bytes = 0; } -static inline void ArrowBufferMove( - struct ArrowBuffer* src, struct ArrowBuffer* dst) { +static inline void ArrowBufferMove(struct ArrowBuffer* src, struct ArrowBuffer* dst) { memcpy(dst, src, sizeof(struct ArrowBuffer)); src->data = NULL; ArrowBufferReset(src); } -static inline ArrowErrorCode ArrowBufferResize( - struct ArrowBuffer* buffer, - int64_t new_capacity_bytes, - char shrink_to_fit) { +static inline ArrowErrorCode ArrowBufferResize(struct ArrowBuffer* buffer, + int64_t new_capacity_bytes, + char shrink_to_fit) { if (new_capacity_bytes < 0) { return EINVAL; } if (new_capacity_bytes > buffer->capacity_bytes || shrink_to_fit) { buffer->data = buffer->allocator.reallocate( - &buffer->allocator, - buffer->data, - buffer->capacity_bytes, - new_capacity_bytes); + &buffer->allocator, buffer->data, buffer->capacity_bytes, new_capacity_bytes); if (buffer->data == NULL && new_capacity_bytes > 0) { buffer->capacity_bytes = 0; buffer->size_bytes = 0; @@ -2354,97 +2247,95 @@ static inline ArrowErrorCode ArrowBufferResize( return NANOARROW_OK; } -static inline ArrowErrorCode ArrowBufferReserve( - struct ArrowBuffer* buffer, int64_t additional_size_bytes) { +static inline ArrowErrorCode ArrowBufferReserve(struct ArrowBuffer* buffer, + int64_t additional_size_bytes) { int64_t min_capacity_bytes = buffer->size_bytes + additional_size_bytes; if (min_capacity_bytes <= buffer->capacity_bytes) { return NANOARROW_OK; } return ArrowBufferResize( - buffer, - _ArrowGrowByFactor(buffer->capacity_bytes, min_capacity_bytes), - 0); + buffer, _ArrowGrowByFactor(buffer->capacity_bytes, min_capacity_bytes), 0); } -static inline void ArrowBufferAppendUnsafe( - struct ArrowBuffer* buffer, const void* data, int64_t size_bytes) { +static inline void ArrowBufferAppendUnsafe(struct ArrowBuffer* buffer, const void* data, + int64_t size_bytes) { if (size_bytes > 0) { memcpy(buffer->data + buffer->size_bytes, data, size_bytes); buffer->size_bytes += size_bytes; } } -static inline ArrowErrorCode ArrowBufferAppend( - struct ArrowBuffer* buffer, const void* data, int64_t size_bytes) { +static inline ArrowErrorCode ArrowBufferAppend(struct ArrowBuffer* buffer, + const void* data, int64_t size_bytes) { NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, size_bytes)); ArrowBufferAppendUnsafe(buffer, data, size_bytes); return NANOARROW_OK; } -static inline ArrowErrorCode ArrowBufferAppendInt8( - struct ArrowBuffer* buffer, int8_t value) { +static inline ArrowErrorCode ArrowBufferAppendInt8(struct ArrowBuffer* buffer, + int8_t value) { return ArrowBufferAppend(buffer, &value, sizeof(int8_t)); } -static inline ArrowErrorCode ArrowBufferAppendUInt8( - struct ArrowBuffer* buffer, uint8_t value) { +static inline ArrowErrorCode ArrowBufferAppendUInt8(struct ArrowBuffer* buffer, + uint8_t value) { return ArrowBufferAppend(buffer, &value, sizeof(uint8_t)); } -static inline ArrowErrorCode ArrowBufferAppendInt16( - struct ArrowBuffer* buffer, int16_t value) { +static inline ArrowErrorCode ArrowBufferAppendInt16(struct ArrowBuffer* buffer, + int16_t value) { return ArrowBufferAppend(buffer, &value, sizeof(int16_t)); } -static inline ArrowErrorCode ArrowBufferAppendUInt16( - struct ArrowBuffer* buffer, uint16_t value) { +static inline ArrowErrorCode ArrowBufferAppendUInt16(struct ArrowBuffer* buffer, + uint16_t value) { return ArrowBufferAppend(buffer, &value, sizeof(uint16_t)); } -static inline ArrowErrorCode ArrowBufferAppendInt32( - struct ArrowBuffer* buffer, int32_t value) { +static inline ArrowErrorCode ArrowBufferAppendInt32(struct ArrowBuffer* buffer, + int32_t value) { return ArrowBufferAppend(buffer, &value, sizeof(int32_t)); } -static inline ArrowErrorCode ArrowBufferAppendUInt32( - struct ArrowBuffer* buffer, uint32_t value) { +static inline ArrowErrorCode ArrowBufferAppendUInt32(struct ArrowBuffer* buffer, + uint32_t value) { return ArrowBufferAppend(buffer, &value, sizeof(uint32_t)); } -static inline ArrowErrorCode ArrowBufferAppendInt64( - struct ArrowBuffer* buffer, int64_t value) { +static inline ArrowErrorCode ArrowBufferAppendInt64(struct ArrowBuffer* buffer, + int64_t value) { return ArrowBufferAppend(buffer, &value, sizeof(int64_t)); } -static inline ArrowErrorCode ArrowBufferAppendUInt64( - struct ArrowBuffer* buffer, uint64_t value) { +static inline ArrowErrorCode ArrowBufferAppendUInt64(struct ArrowBuffer* buffer, + uint64_t value) { return ArrowBufferAppend(buffer, &value, sizeof(uint64_t)); } -static inline ArrowErrorCode ArrowBufferAppendDouble( - struct ArrowBuffer* buffer, double value) { +static inline ArrowErrorCode ArrowBufferAppendDouble(struct ArrowBuffer* buffer, + double value) { return ArrowBufferAppend(buffer, &value, sizeof(double)); } -static inline ArrowErrorCode ArrowBufferAppendFloat( - struct ArrowBuffer* buffer, float value) { +static inline ArrowErrorCode ArrowBufferAppendFloat(struct ArrowBuffer* buffer, + float value) { return ArrowBufferAppend(buffer, &value, sizeof(float)); } -static inline ArrowErrorCode ArrowBufferAppendStringView( - struct ArrowBuffer* buffer, struct ArrowStringView value) { +static inline ArrowErrorCode ArrowBufferAppendStringView(struct ArrowBuffer* buffer, + struct ArrowStringView value) { return ArrowBufferAppend(buffer, value.data, value.size_bytes); } -static inline ArrowErrorCode ArrowBufferAppendBufferView( - struct ArrowBuffer* buffer, struct ArrowBufferView value) { +static inline ArrowErrorCode ArrowBufferAppendBufferView(struct ArrowBuffer* buffer, + struct ArrowBufferView value) { return ArrowBufferAppend(buffer, value.data.data, value.size_bytes); } -static inline ArrowErrorCode ArrowBufferAppendFill( - struct ArrowBuffer* buffer, uint8_t value, int64_t size_bytes) { +static inline ArrowErrorCode ArrowBufferAppendFill(struct ArrowBuffer* buffer, + uint8_t value, int64_t size_bytes) { NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, size_bytes)); memset(buffer->data + buffer->size_bytes, value, size_bytes); @@ -2453,24 +2344,20 @@ static inline ArrowErrorCode ArrowBufferAppendFill( } static const uint8_t _ArrowkBitmask[] = {1, 2, 4, 8, 16, 32, 64, 128}; -static const uint8_t _ArrowkFlippedBitmask[] = { - 254, 253, 251, 247, 239, 223, 191, 127}; +static const uint8_t _ArrowkFlippedBitmask[] = {254, 253, 251, 247, 239, 223, 191, 127}; static const uint8_t _ArrowkPrecedingBitmask[] = {0, 1, 3, 7, 15, 31, 63, 127}; -static const uint8_t _ArrowkTrailingBitmask[] = { - 255, 254, 252, 248, 240, 224, 192, 128}; +static const uint8_t _ArrowkTrailingBitmask[] = {255, 254, 252, 248, 240, 224, 192, 128}; static const uint8_t _ArrowkBytePopcount[] = { - 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, - 2, 3, 3, 4, 3, 4, 4, 5, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, - 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, - 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, - 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, - 4, 5, 5, 6, 5, 6, 6, 7, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, - 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, - 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, - 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, - 4, 5, 5, 6, 5, 6, 6, 7, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, - 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8}; + 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, + 4, 4, 5, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, + 4, 5, 4, 5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, + 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, + 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, + 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, + 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, + 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 3, 4, 4, 5, 4, 5, 5, 6, + 4, 5, 5, 6, 5, 6, 6, 7, 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8}; static inline int64_t _ArrowRoundUpToMultipleOf8(int64_t value) { return (value + 7) & ~((int64_t)7); @@ -2507,25 +2394,25 @@ static inline void _ArrowBitsUnpackInt32(const uint8_t word, int32_t* out) { } static inline void _ArrowBitmapPackInt8(const int8_t* values, uint8_t* out) { - *out = (uint8_t)(values[0] | ((values[1] + 0x1) & 0x2) | - ((values[2] + 0x3) & 0x4) | ((values[3] + 0x7) & 0x8) | - ((values[4] + 0xf) & 0x10) | ((values[5] + 0x1f) & 0x20) | - ((values[6] + 0x3f) & 0x40) | ((values[7] + 0x7f) & 0x80)); + *out = (uint8_t)(values[0] | ((values[1] + 0x1) & 0x2) | ((values[2] + 0x3) & 0x4) | + ((values[3] + 0x7) & 0x8) | ((values[4] + 0xf) & 0x10) | + ((values[5] + 0x1f) & 0x20) | ((values[6] + 0x3f) & 0x40) | + ((values[7] + 0x7f) & 0x80)); } static inline void _ArrowBitmapPackInt32(const int32_t* values, uint8_t* out) { - *out = (uint8_t)(values[0] | ((values[1] + 0x1) & 0x2) | - ((values[2] + 0x3) & 0x4) | ((values[3] + 0x7) & 0x8) | - ((values[4] + 0xf) & 0x10) | ((values[5] + 0x1f) & 0x20) | - ((values[6] + 0x3f) & 0x40) | ((values[7] + 0x7f) & 0x80)); + *out = (uint8_t)(values[0] | ((values[1] + 0x1) & 0x2) | ((values[2] + 0x3) & 0x4) | + ((values[3] + 0x7) & 0x8) | ((values[4] + 0xf) & 0x10) | + ((values[5] + 0x1f) & 0x20) | ((values[6] + 0x3f) & 0x40) | + ((values[7] + 0x7f) & 0x80)); } static inline int8_t ArrowBitGet(const uint8_t* bits, int64_t i) { return (bits[i >> 3] >> (i & 0x07)) & 1; } -static inline void ArrowBitsUnpackInt8( - const uint8_t* bits, int64_t start_offset, int64_t length, int8_t* out) { +static inline void ArrowBitsUnpackInt8(const uint8_t* bits, int64_t start_offset, + int64_t length, int8_t* out) { if (length == 0) { return; } @@ -2563,8 +2450,8 @@ static inline void ArrowBitsUnpackInt8( } } -static inline void ArrowBitsUnpackInt32( - const uint8_t* bits, int64_t start_offset, int64_t length, int32_t* out) { +static inline void ArrowBitsUnpackInt32(const uint8_t* bits, int64_t start_offset, + int64_t length, int32_t* out) { if (length == 0) { return; } @@ -2611,12 +2498,12 @@ static inline void ArrowBitClear(uint8_t* bits, int64_t i) { } static inline void ArrowBitSetTo(uint8_t* bits, int64_t i, uint8_t bit_is_set) { - bits[i / 8] ^= ((uint8_t)(-((uint8_t)(bit_is_set != 0)) ^ bits[i / 8])) & - _ArrowkBitmask[i % 8]; + bits[i / 8] ^= + ((uint8_t)(-((uint8_t)(bit_is_set != 0)) ^ bits[i / 8])) & _ArrowkBitmask[i % 8]; } -static inline void ArrowBitsSetTo( - uint8_t* bits, int64_t start_offset, int64_t length, uint8_t bits_are_set) { +static inline void ArrowBitsSetTo(uint8_t* bits, int64_t start_offset, int64_t length, + uint8_t bits_are_set) { const int64_t i_begin = start_offset; const int64_t i_end = start_offset + length; const uint8_t fill_byte = (uint8_t)(-bits_are_set); @@ -2630,8 +2517,7 @@ static inline void ArrowBitsSetTo( if (bytes_end == bytes_begin + 1) { // set bits within a single byte const uint8_t only_byte_mask = - i_end % 8 == 0 ? first_byte_mask : - (uint8_t)(first_byte_mask | last_byte_mask); + i_end % 8 == 0 ? first_byte_mask : (uint8_t)(first_byte_mask | last_byte_mask); bits[bytes_begin] &= only_byte_mask; bits[bytes_begin] |= (uint8_t)(fill_byte & ~only_byte_mask); return; @@ -2643,10 +2529,7 @@ static inline void ArrowBitsSetTo( if (bytes_end - bytes_begin > 2) { // set/clear whole bytes - memset( - bits + bytes_begin + 1, - fill_byte, - (size_t)(bytes_end - bytes_begin - 2)); + memset(bits + bytes_begin + 1, fill_byte, (size_t)(bytes_end - bytes_begin - 2)); } if (i_end % 8 == 0) { @@ -2658,8 +2541,8 @@ static inline void ArrowBitsSetTo( bits[bytes_end - 1] |= (uint8_t)(fill_byte & ~last_byte_mask); } -static inline int64_t ArrowBitCountSet( - const uint8_t* bits, int64_t start_offset, int64_t length) { +static inline int64_t ArrowBitCountSet(const uint8_t* bits, int64_t start_offset, + int64_t length) { if (length == 0) { return 0; } @@ -2677,16 +2560,14 @@ static inline int64_t ArrowBitCountSet( const uint8_t last_byte_mask = _ArrowkTrailingBitmask[i_begin % 8]; const uint8_t only_byte_mask = - i_end % 8 == 0 ? last_byte_mask : - (uint8_t)(first_byte_mask & last_byte_mask); + i_end % 8 == 0 ? last_byte_mask : (uint8_t)(first_byte_mask & last_byte_mask); const uint8_t byte_masked = bits[bytes_begin] & only_byte_mask; return _ArrowkBytePopcount[byte_masked]; } const uint8_t first_byte_mask = _ArrowkPrecedingBitmask[i_begin % 8]; - const uint8_t last_byte_mask = - i_end % 8 == 0 ? 0 : _ArrowkTrailingBitmask[i_end % 8]; + const uint8_t last_byte_mask = i_end % 8 == 0 ? 0 : _ArrowkTrailingBitmask[i_end % 8]; int64_t count = 0; // first byte @@ -2708,29 +2589,29 @@ static inline void ArrowBitmapInit(struct ArrowBitmap* bitmap) { bitmap->size_bits = 0; } -static inline void ArrowBitmapMove( - struct ArrowBitmap* src, struct ArrowBitmap* dst) { +static inline void ArrowBitmapMove(struct ArrowBitmap* src, struct ArrowBitmap* dst) { ArrowBufferMove(&src->buffer, &dst->buffer); dst->size_bits = src->size_bits; src->size_bits = 0; } -static inline ArrowErrorCode ArrowBitmapReserve( - struct ArrowBitmap* bitmap, int64_t additional_size_bits) { +static inline ArrowErrorCode ArrowBitmapReserve(struct ArrowBitmap* bitmap, + int64_t additional_size_bits) { int64_t min_capacity_bits = bitmap->size_bits + additional_size_bits; if (min_capacity_bits <= (bitmap->buffer.capacity_bytes * 8)) { return NANOARROW_OK; } - NANOARROW_RETURN_NOT_OK(ArrowBufferReserve( - &bitmap->buffer, _ArrowBytesForBits(additional_size_bits))); + NANOARROW_RETURN_NOT_OK( + ArrowBufferReserve(&bitmap->buffer, _ArrowBytesForBits(additional_size_bits))); bitmap->buffer.data[bitmap->buffer.capacity_bytes - 1] = 0; return NANOARROW_OK; } -static inline ArrowErrorCode ArrowBitmapResize( - struct ArrowBitmap* bitmap, int64_t new_capacity_bits, char shrink_to_fit) { +static inline ArrowErrorCode ArrowBitmapResize(struct ArrowBitmap* bitmap, + int64_t new_capacity_bits, + char shrink_to_fit) { if (new_capacity_bits < 0) { return EINVAL; } @@ -2746,23 +2627,23 @@ static inline ArrowErrorCode ArrowBitmapResize( return NANOARROW_OK; } -static inline ArrowErrorCode ArrowBitmapAppend( - struct ArrowBitmap* bitmap, uint8_t bits_are_set, int64_t length) { +static inline ArrowErrorCode ArrowBitmapAppend(struct ArrowBitmap* bitmap, + uint8_t bits_are_set, int64_t length) { NANOARROW_RETURN_NOT_OK(ArrowBitmapReserve(bitmap, length)); ArrowBitmapAppendUnsafe(bitmap, bits_are_set, length); return NANOARROW_OK; } -static inline void ArrowBitmapAppendUnsafe( - struct ArrowBitmap* bitmap, uint8_t bits_are_set, int64_t length) { +static inline void ArrowBitmapAppendUnsafe(struct ArrowBitmap* bitmap, + uint8_t bits_are_set, int64_t length) { ArrowBitsSetTo(bitmap->buffer.data, bitmap->size_bits, length, bits_are_set); bitmap->size_bits += length; bitmap->buffer.size_bytes = _ArrowBytesForBits(bitmap->size_bits); } -static inline void ArrowBitmapAppendInt8Unsafe( - struct ArrowBitmap* bitmap, const int8_t* values, int64_t n_values) { +static inline void ArrowBitmapAppendInt8Unsafe(struct ArrowBitmap* bitmap, + const int8_t* values, int64_t n_values) { if (n_values == 0) { return; } @@ -2774,8 +2655,7 @@ static inline void ArrowBitmapAppendInt8Unsafe( // First byte if ((out_i_cursor % 8) != 0) { - int64_t n_partial_bits = - _ArrowRoundUpToMultipleOf8(out_i_cursor) - out_i_cursor; + int64_t n_partial_bits = _ArrowRoundUpToMultipleOf8(out_i_cursor) - out_i_cursor; for (int i = 0; i < n_partial_bits; i++) { ArrowBitSetTo(bitmap->buffer.data, out_i_cursor++, values[i]); } @@ -2809,8 +2689,8 @@ static inline void ArrowBitmapAppendInt8Unsafe( bitmap->buffer.size_bytes = out_cursor - bitmap->buffer.data; } -static inline void ArrowBitmapAppendInt32Unsafe( - struct ArrowBitmap* bitmap, const int32_t* values, int64_t n_values) { +static inline void ArrowBitmapAppendInt32Unsafe(struct ArrowBitmap* bitmap, + const int32_t* values, int64_t n_values) { if (n_values == 0) { return; } @@ -2822,8 +2702,7 @@ static inline void ArrowBitmapAppendInt32Unsafe( // First byte if ((out_i_cursor % 8) != 0) { - int64_t n_partial_bits = - _ArrowRoundUpToMultipleOf8(out_i_cursor) - out_i_cursor; + int64_t n_partial_bits = _ArrowRoundUpToMultipleOf8(out_i_cursor) - out_i_cursor; for (int i = 0; i < n_partial_bits; i++) { ArrowBitSetTo(bitmap->buffer.data, out_i_cursor++, (uint8_t)values[i]); } @@ -2848,8 +2727,7 @@ static inline void ArrowBitmapAppendInt32Unsafe( // Zero out the last byte *out_cursor = 0x00; for (int i = 0; i < n_remaining; i++) { - ArrowBitSetTo( - bitmap->buffer.data, out_i_cursor++, (uint8_t)values_cursor[i]); + ArrowBitSetTo(bitmap->buffer.data, out_i_cursor++, (uint8_t)values_cursor[i]); } out_cursor++; } @@ -2894,19 +2772,20 @@ static inline void ArrowBitmapReset(struct ArrowBitmap* bitmap) { #include #include + + + #ifdef __cplusplus extern "C" { #endif -static inline struct ArrowBitmap* ArrowArrayValidityBitmap( - struct ArrowArray* array) { +static inline struct ArrowBitmap* ArrowArrayValidityBitmap(struct ArrowArray* array) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; return &private_data->bitmap; } -static inline struct ArrowBuffer* ArrowArrayBuffer( - struct ArrowArray* array, int64_t i) { +static inline struct ArrowBuffer* ArrowArrayBuffer(struct ArrowArray* array, int64_t i) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; switch (i) { @@ -2920,20 +2799,19 @@ static inline struct ArrowBuffer* ArrowArrayBuffer( // We don't currently support the case of unions where type_id != child_index; // however, these functions are used to keep track of where that assumption // is made. -static inline int8_t _ArrowArrayUnionChildIndex( - struct ArrowArray* array, int8_t type_id) { +static inline int8_t _ArrowArrayUnionChildIndex(struct ArrowArray* array, + int8_t type_id) { NANOARROW_UNUSED(array); return type_id; } -static inline int8_t _ArrowArrayUnionTypeId( - struct ArrowArray* array, int8_t child_index) { +static inline int8_t _ArrowArrayUnionTypeId(struct ArrowArray* array, + int8_t child_index) { NANOARROW_UNUSED(array); return child_index; } -static inline int32_t _ArrowParseUnionTypeIds( - const char* type_ids, int8_t* out) { +static inline int32_t _ArrowParseUnionTypeIds(const char* type_ids, int8_t* out) { if (*type_ids == '\0') { return 0; } @@ -2966,8 +2844,9 @@ static inline int32_t _ArrowParseUnionTypeIds( return -1; } -static inline int8_t _ArrowParsedUnionTypeIdsWillEqualChildIndices( - const int8_t* type_ids, int64_t n_type_ids, int64_t n_children) { +static inline int8_t _ArrowParsedUnionTypeIdsWillEqualChildIndices(const int8_t* type_ids, + int64_t n_type_ids, + int64_t n_children) { if (n_type_ids != n_children) { return 0; } @@ -2981,16 +2860,14 @@ static inline int8_t _ArrowParsedUnionTypeIdsWillEqualChildIndices( return 1; } -static inline int8_t _ArrowUnionTypeIdsWillEqualChildIndices( - const char* type_id_str, int64_t n_children) { +static inline int8_t _ArrowUnionTypeIdsWillEqualChildIndices(const char* type_id_str, + int64_t n_children) { int8_t type_ids[128]; int32_t n_type_ids = _ArrowParseUnionTypeIds(type_id_str, type_ids); - return _ArrowParsedUnionTypeIdsWillEqualChildIndices( - type_ids, n_type_ids, n_children); + return _ArrowParsedUnionTypeIdsWillEqualChildIndices(type_ids, n_type_ids, n_children); } -static inline ArrowErrorCode ArrowArrayStartAppending( - struct ArrowArray* array) { +static inline ArrowErrorCode ArrowArrayStartAppending(struct ArrowArray* array) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -3014,17 +2891,12 @@ static inline ArrowErrorCode ArrowArrayStartAppending( // Initialize any data offset buffer with a single zero for (int i = 0; i < NANOARROW_MAX_FIXED_BUFFERS; i++) { - if (private_data->layout.buffer_type[i] == - NANOARROW_BUFFER_TYPE_DATA_OFFSET && + if (private_data->layout.buffer_type[i] == NANOARROW_BUFFER_TYPE_DATA_OFFSET && private_data->layout.element_size_bits[i] == 64) { - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendInt64(ArrowArrayBuffer(array, i), 0)); - } else if ( - private_data->layout.buffer_type[i] == - NANOARROW_BUFFER_TYPE_DATA_OFFSET && - private_data->layout.element_size_bits[i] == 32) { - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendInt32(ArrowArrayBuffer(array, i), 0)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt64(ArrowArrayBuffer(array, i), 0)); + } else if (private_data->layout.buffer_type[i] == NANOARROW_BUFFER_TYPE_DATA_OFFSET && + private_data->layout.element_size_bits[i] == 32) { + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(ArrowArrayBuffer(array, i), 0)); } } @@ -3057,15 +2929,15 @@ static inline ArrowErrorCode ArrowArrayShrinkToFit(struct ArrowArray* array) { return NANOARROW_OK; } -static inline ArrowErrorCode _ArrowArrayAppendBits( - struct ArrowArray* array, int64_t buffer_i, uint8_t value, int64_t n) { +static inline ArrowErrorCode _ArrowArrayAppendBits(struct ArrowArray* array, + int64_t buffer_i, uint8_t value, + int64_t n) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; struct ArrowBuffer* buffer = ArrowArrayBuffer(array, buffer_i); int64_t bytes_required = - _ArrowRoundUpToMultipleOf8( - private_data->layout.element_size_bits[buffer_i] * - (array->length + 1)) / + _ArrowRoundUpToMultipleOf8(private_data->layout.element_size_bits[buffer_i] * + (array->length + 1)) / 8; if (bytes_required > buffer->size_bytes) { NANOARROW_RETURN_NOT_OK( @@ -3076,8 +2948,8 @@ static inline ArrowErrorCode _ArrowArrayAppendBits( return NANOARROW_OK; } -static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal( - struct ArrowArray* array, int64_t n, uint8_t is_valid) { +static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal(struct ArrowArray* array, + int64_t n, uint8_t is_valid) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -3102,11 +2974,10 @@ static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal( ArrowBufferAppendFill(ArrowArrayBuffer(array, 0), type_id, n)); for (int64_t i = 0; i < n; i++) { NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32( - ArrowArrayBuffer(array, 1), - (int32_t)array->children[0]->length - 1)); + ArrowArrayBuffer(array, 1), (int32_t)array->children[0]->length - 1)); } - // For the purposes of array->null_count, union elements are never - // considered "null" even if some children contain nulls. + // For the purposes of array->null_count, union elements are never considered "null" + // even if some children contain nulls. array->length += n; return NANOARROW_OK; } @@ -3122,8 +2993,8 @@ static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal( NANOARROW_RETURN_NOT_OK( ArrowBufferAppendFill(ArrowArrayBuffer(array, 0), type_id, n)); - // For the purposes of array->null_count, union elements are never - // considered "null" even if some children contain nulls. + // For the purposes of array->null_count, union elements are never considered "null" + // even if some children contain nulls. array->length += n; return NANOARROW_OK; } @@ -3142,11 +3013,10 @@ static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal( break; } - // Append n is_valid bits to the validity bitmap. If we haven't allocated a - // bitmap yet and we need to append nulls, do it now. + // Append n is_valid bits to the validity bitmap. If we haven't allocated a bitmap yet + // and we need to append nulls, do it now. if (!is_valid && private_data->bitmap.buffer.data == NULL) { - NANOARROW_RETURN_NOT_OK( - ArrowBitmapReserve(&private_data->bitmap, array->length + n)); + NANOARROW_RETURN_NOT_OK(ArrowBitmapReserve(&private_data->bitmap, array->length + n)); ArrowBitmapAppendUnsafe(&private_data->bitmap, 1, array->length); ArrowBitmapAppendUnsafe(&private_data->bitmap, is_valid, n); } else if (private_data->bitmap.buffer.data != NULL) { @@ -3167,15 +3037,12 @@ static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal( case NANOARROW_BUFFER_TYPE_VALIDITY: continue; case NANOARROW_BUFFER_TYPE_DATA_OFFSET: - // Append the current value at the end of the offset buffer for each - // element + // Append the current value at the end of the offset buffer for each element NANOARROW_RETURN_NOT_OK(ArrowBufferReserve(buffer, size_bytes * n)); for (int64_t j = 0; j < n; j++) { - ArrowBufferAppendUnsafe( - buffer, - buffer->data + size_bytes * (array->length + j), - size_bytes); + ArrowBufferAppendUnsafe(buffer, buffer->data + size_bytes * (array->length + j), + size_bytes); } // Skip the data buffer @@ -3184,8 +3051,7 @@ static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal( case NANOARROW_BUFFER_TYPE_DATA: // Zero out the next bit of memory if (private_data->layout.element_size_bits[i] % 8 == 0) { - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendFill(buffer, 0, size_bytes * n)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendFill(buffer, 0, size_bytes * n)); } else { NANOARROW_RETURN_NOT_OK(_ArrowArrayAppendBits(array, i, 0, n)); } @@ -3203,18 +3069,16 @@ static inline ArrowErrorCode _ArrowArrayAppendEmptyInternal( return NANOARROW_OK; } -static inline ArrowErrorCode ArrowArrayAppendNull( - struct ArrowArray* array, int64_t n) { +static inline ArrowErrorCode ArrowArrayAppendNull(struct ArrowArray* array, int64_t n) { return _ArrowArrayAppendEmptyInternal(array, n, 0); } -static inline ArrowErrorCode ArrowArrayAppendEmpty( - struct ArrowArray* array, int64_t n) { +static inline ArrowErrorCode ArrowArrayAppendEmpty(struct ArrowArray* array, int64_t n) { return _ArrowArrayAppendEmptyInternal(array, n, 1); } -static inline ArrowErrorCode ArrowArrayAppendInt( - struct ArrowArray* array, int64_t value) { +static inline ArrowErrorCode ArrowArrayAppendInt(struct ArrowArray* array, + int64_t value) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -3222,23 +3086,19 @@ static inline ArrowErrorCode ArrowArrayAppendInt( switch (private_data->storage_type) { case NANOARROW_TYPE_INT64: - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppend(data_buffer, &value, sizeof(int64_t))); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(data_buffer, &value, sizeof(int64_t))); break; case NANOARROW_TYPE_INT32: _NANOARROW_CHECK_RANGE(value, INT32_MIN, INT32_MAX); - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendInt32(data_buffer, (int32_t)value)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(data_buffer, (int32_t)value)); break; case NANOARROW_TYPE_INT16: _NANOARROW_CHECK_RANGE(value, INT16_MIN, INT16_MAX); - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendInt16(data_buffer, (int16_t)value)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt16(data_buffer, (int16_t)value)); break; case NANOARROW_TYPE_INT8: _NANOARROW_CHECK_RANGE(value, INT8_MIN, INT8_MAX); - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendInt8(data_buffer, (int8_t)value)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt8(data_buffer, (int8_t)value)); break; case NANOARROW_TYPE_UINT64: case NANOARROW_TYPE_UINT32: @@ -3247,12 +3107,10 @@ static inline ArrowErrorCode ArrowArrayAppendInt( _NANOARROW_CHECK_RANGE(value, 0, INT64_MAX); return ArrowArrayAppendUInt(array, value); case NANOARROW_TYPE_DOUBLE: - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendDouble(data_buffer, (double)value)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendDouble(data_buffer, (double)value)); break; case NANOARROW_TYPE_FLOAT: - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendFloat(data_buffer, (float)value)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendFloat(data_buffer, (float)value)); break; case NANOARROW_TYPE_BOOL: NANOARROW_RETURN_NOT_OK(_ArrowArrayAppendBits(array, 1, value != 0, 1)); @@ -3262,16 +3120,15 @@ static inline ArrowErrorCode ArrowArrayAppendInt( } if (private_data->bitmap.buffer.data != NULL) { - NANOARROW_RETURN_NOT_OK( - ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); + NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); } array->length++; return NANOARROW_OK; } -static inline ArrowErrorCode ArrowArrayAppendUInt( - struct ArrowArray* array, uint64_t value) { +static inline ArrowErrorCode ArrowArrayAppendUInt(struct ArrowArray* array, + uint64_t value) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -3279,23 +3136,19 @@ static inline ArrowErrorCode ArrowArrayAppendUInt( switch (private_data->storage_type) { case NANOARROW_TYPE_UINT64: - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppend(data_buffer, &value, sizeof(uint64_t))); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(data_buffer, &value, sizeof(uint64_t))); break; case NANOARROW_TYPE_UINT32: _NANOARROW_CHECK_UPPER_LIMIT(value, UINT32_MAX); - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendUInt32(data_buffer, (uint32_t)value)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendUInt32(data_buffer, (uint32_t)value)); break; case NANOARROW_TYPE_UINT16: _NANOARROW_CHECK_UPPER_LIMIT(value, UINT16_MAX); - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendUInt16(data_buffer, (uint16_t)value)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendUInt16(data_buffer, (uint16_t)value)); break; case NANOARROW_TYPE_UINT8: _NANOARROW_CHECK_UPPER_LIMIT(value, UINT8_MAX); - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendUInt8(data_buffer, (uint8_t)value)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendUInt8(data_buffer, (uint8_t)value)); break; case NANOARROW_TYPE_INT64: case NANOARROW_TYPE_INT32: @@ -3304,12 +3157,10 @@ static inline ArrowErrorCode ArrowArrayAppendUInt( _NANOARROW_CHECK_UPPER_LIMIT(value, INT64_MAX); return ArrowArrayAppendInt(array, value); case NANOARROW_TYPE_DOUBLE: - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendDouble(data_buffer, (double)value)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendDouble(data_buffer, (double)value)); break; case NANOARROW_TYPE_FLOAT: - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendFloat(data_buffer, (float)value)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendFloat(data_buffer, (float)value)); break; case NANOARROW_TYPE_BOOL: NANOARROW_RETURN_NOT_OK(_ArrowArrayAppendBits(array, 1, value != 0, 1)); @@ -3319,16 +3170,15 @@ static inline ArrowErrorCode ArrowArrayAppendUInt( } if (private_data->bitmap.buffer.data != NULL) { - NANOARROW_RETURN_NOT_OK( - ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); + NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); } array->length++; return NANOARROW_OK; } -static inline ArrowErrorCode ArrowArrayAppendDouble( - struct ArrowArray* array, double value) { +static inline ArrowErrorCode ArrowArrayAppendDouble(struct ArrowArray* array, + double value) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -3336,35 +3186,31 @@ static inline ArrowErrorCode ArrowArrayAppendDouble( switch (private_data->storage_type) { case NANOARROW_TYPE_DOUBLE: - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppend(data_buffer, &value, sizeof(double))); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(data_buffer, &value, sizeof(double))); break; case NANOARROW_TYPE_FLOAT: - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendFloat(data_buffer, (float)value)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendFloat(data_buffer, (float)value)); break; default: return EINVAL; } if (private_data->bitmap.buffer.data != NULL) { - NANOARROW_RETURN_NOT_OK( - ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); + NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); } array->length++; return NANOARROW_OK; } -static inline ArrowErrorCode ArrowArrayAppendBytes( - struct ArrowArray* array, struct ArrowBufferView value) { +static inline ArrowErrorCode ArrowArrayAppendBytes(struct ArrowArray* array, + struct ArrowBufferView value) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; struct ArrowBuffer* offset_buffer = ArrowArrayBuffer(array, 1); struct ArrowBuffer* data_buffer = ArrowArrayBuffer( - array, - 1 + (private_data->storage_type != NANOARROW_TYPE_FIXED_SIZE_BINARY)); + array, 1 + (private_data->storage_type != NANOARROW_TYPE_FIXED_SIZE_BINARY)); int32_t offset; int64_t large_offset; int64_t fixed_size_bytes = private_data->layout.element_size_bits[1] / 8; @@ -3378,8 +3224,7 @@ static inline ArrowErrorCode ArrowArrayAppendBytes( } offset += (int32_t)value.size_bytes; - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppend(offset_buffer, &offset, sizeof(int32_t))); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppend(offset_buffer, &offset, sizeof(int32_t))); NANOARROW_RETURN_NOT_OK( ArrowBufferAppend(data_buffer, value.data.data, value.size_bytes)); break; @@ -3407,16 +3252,15 @@ static inline ArrowErrorCode ArrowArrayAppendBytes( } if (private_data->bitmap.buffer.data != NULL) { - NANOARROW_RETURN_NOT_OK( - ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); + NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); } array->length++; return NANOARROW_OK; } -static inline ArrowErrorCode ArrowArrayAppendString( - struct ArrowArray* array, struct ArrowStringView value) { +static inline ArrowErrorCode ArrowArrayAppendString(struct ArrowArray* array, + struct ArrowStringView value) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -3435,8 +3279,8 @@ static inline ArrowErrorCode ArrowArrayAppendString( } } -static inline ArrowErrorCode ArrowArrayAppendInterval( - struct ArrowArray* array, const struct ArrowInterval* value) { +static inline ArrowErrorCode ArrowArrayAppendInterval(struct ArrowArray* array, + const struct ArrowInterval* value) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -3448,8 +3292,7 @@ static inline ArrowErrorCode ArrowArrayAppendInterval( return EINVAL; } - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendInt32(data_buffer, value->months)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(data_buffer, value->months)); break; } case NANOARROW_TYPE_INTERVAL_DAY_TIME: { @@ -3466,8 +3309,7 @@ static inline ArrowErrorCode ArrowArrayAppendInterval( return EINVAL; } - NANOARROW_RETURN_NOT_OK( - ArrowBufferAppendInt32(data_buffer, value->months)); + NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(data_buffer, value->months)); NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32(data_buffer, value->days)); NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt64(data_buffer, value->ns)); break; @@ -3477,16 +3319,15 @@ static inline ArrowErrorCode ArrowArrayAppendInterval( } if (private_data->bitmap.buffer.data != NULL) { - NANOARROW_RETURN_NOT_OK( - ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); + NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); } array->length++; return NANOARROW_OK; } -static inline ArrowErrorCode ArrowArrayAppendDecimal( - struct ArrowArray* array, const struct ArrowDecimal* value) { +static inline ArrowErrorCode ArrowArrayAppendDecimal(struct ArrowArray* array, + const struct ArrowDecimal* value) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; struct ArrowBuffer* data_buffer = ArrowArrayBuffer(array, 1); @@ -3513,8 +3354,7 @@ static inline ArrowErrorCode ArrowArrayAppendDecimal( } if (private_data->bitmap.buffer.data != NULL) { - NANOARROW_RETURN_NOT_OK( - ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); + NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); } array->length++; @@ -3534,8 +3374,8 @@ static inline ArrowErrorCode ArrowArrayFinishElement(struct ArrowArray* array) { if (child_length > INT32_MAX) { return EOVERFLOW; } - NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32( - ArrowArrayBuffer(array, 1), (int32_t)child_length)); + NANOARROW_RETURN_NOT_OK( + ArrowBufferAppendInt32(ArrowArrayBuffer(array, 1), (int32_t)child_length)); break; case NANOARROW_TYPE_LARGE_LIST: child_length = array->children[0]->length; @@ -3562,16 +3402,15 @@ static inline ArrowErrorCode ArrowArrayFinishElement(struct ArrowArray* array) { } if (private_data->bitmap.buffer.data != NULL) { - NANOARROW_RETURN_NOT_OK( - ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); + NANOARROW_RETURN_NOT_OK(ArrowBitmapAppend(ArrowArrayValidityBitmap(array), 1, 1)); } array->length++; return NANOARROW_OK; } -static inline ArrowErrorCode ArrowArrayFinishUnionElement( - struct ArrowArray* array, int8_t type_id) { +static inline ArrowErrorCode ArrowArrayFinishUnionElement(struct ArrowArray* array, + int8_t type_id) { struct ArrowArrayPrivateData* private_data = (struct ArrowArrayPrivateData*)array->private_data; @@ -3583,19 +3422,15 @@ static inline ArrowErrorCode ArrowArrayFinishUnionElement( switch (private_data->storage_type) { case NANOARROW_TYPE_DENSE_UNION: // Append the target child length to the union offsets buffer - _NANOARROW_CHECK_RANGE( - array->children[child_index]->length, 0, INT32_MAX); + _NANOARROW_CHECK_RANGE(array->children[child_index]->length, 0, INT32_MAX); NANOARROW_RETURN_NOT_OK(ArrowBufferAppendInt32( - ArrowArrayBuffer(array, 1), - (int32_t)array->children[child_index]->length - 1)); + ArrowArrayBuffer(array, 1), (int32_t)array->children[child_index]->length - 1)); break; case NANOARROW_TYPE_SPARSE_UNION: - // Append one empty to any non-target column that isn't already the right - // length or abort if appending a null will result in a column with - // invalid length + // Append one empty to any non-target column that isn't already the right length + // or abort if appending a null will result in a column with invalid length for (int64_t i = 0; i < array->n_children; i++) { - if (i == child_index || - array->children[i]->length == (array->length + 1)) { + if (i == child_index || array->children[i]->length == (array->length + 1)) { continue; } @@ -3618,14 +3453,14 @@ static inline ArrowErrorCode ArrowArrayFinishUnionElement( return NANOARROW_OK; } -static inline void ArrowArrayViewMove( - struct ArrowArrayView* src, struct ArrowArrayView* dst) { +static inline void ArrowArrayViewMove(struct ArrowArrayView* src, + struct ArrowArrayView* dst) { memcpy(dst, src, sizeof(struct ArrowArrayView)); ArrowArrayViewInitFromType(src, NANOARROW_TYPE_UNINITIALIZED); } -static inline int8_t ArrowArrayViewIsNull( - const struct ArrowArrayView* array_view, int64_t i) { +static inline int8_t ArrowArrayViewIsNull(const struct ArrowArrayView* array_view, + int64_t i) { const uint8_t* validity_buffer = array_view->buffer_views[0].data.as_uint8; i += array_view->offset; switch (array_view->storage_type) { @@ -3640,8 +3475,8 @@ static inline int8_t ArrowArrayViewIsNull( } } -static inline int8_t ArrowArrayViewUnionTypeId( - const struct ArrowArrayView* array_view, int64_t i) { +static inline int8_t ArrowArrayViewUnionTypeId(const struct ArrowArrayView* array_view, + int64_t i) { switch (array_view->storage_type) { case NANOARROW_TYPE_DENSE_UNION: case NANOARROW_TYPE_SPARSE_UNION: @@ -3685,8 +3520,8 @@ static inline int64_t ArrowArrayViewListChildOffset( } } -static inline int64_t ArrowArrayViewGetIntUnsafe( - const struct ArrowArrayView* array_view, int64_t i) { +static inline int64_t ArrowArrayViewGetIntUnsafe(const struct ArrowArrayView* array_view, + int64_t i) { const struct ArrowBufferView* data_view = &array_view->buffer_views[1]; i += array_view->offset; switch (array_view->storage_type) { @@ -3805,8 +3640,7 @@ static inline struct ArrowStringView ArrowArrayViewGetStringUnsafe( break; case NANOARROW_TYPE_FIXED_SIZE_BINARY: view.size_bytes = array_view->layout.element_size_bits[1] / 8; - view.data = - array_view->buffer_views[1].data.as_char + (i * view.size_bytes); + view.data = array_view->buffer_views[1].data.as_char + (i * view.size_bytes); break; default: view.data = NULL; @@ -3852,9 +3686,7 @@ static inline struct ArrowBufferView ArrowArrayViewGetBytesUnsafe( } static inline void ArrowArrayViewGetIntervalUnsafe( - const struct ArrowArrayView* array_view, - int64_t i, - struct ArrowInterval* out) { + const struct ArrowArrayView* array_view, int64_t i, struct ArrowInterval* out) { const uint8_t* data_view = array_view->buffer_views[1].data.as_uint8; switch (array_view->storage_type) { case NANOARROW_TYPE_INTERVAL_MONTHS: { @@ -3880,10 +3712,8 @@ static inline void ArrowArrayViewGetIntervalUnsafe( } } -static inline void ArrowArrayViewGetDecimalUnsafe( - const struct ArrowArrayView* array_view, - int64_t i, - struct ArrowDecimal* out) { +static inline void ArrowArrayViewGetDecimalUnsafe(const struct ArrowArrayView* array_view, + int64_t i, struct ArrowDecimal* out) { i += array_view->offset; const uint8_t* data_view = array_view->buffer_views[1].data.as_uint8; switch (array_view->storage_type) { From b16b3897accd5e39e059d609391ce584d5aebf01 Mon Sep 17 00:00:00 2001 From: Vivian Nguyen Date: Tue, 9 Apr 2024 14:12:56 -0500 Subject: [PATCH 06/10] Restore TimestampRange --- libtiledbsoma/src/soma/soma_collection.cc | 2 +- libtiledbsoma/src/soma/soma_collection.h | 4 ++-- libtiledbsoma/src/soma/soma_dataframe.cc | 4 ++-- libtiledbsoma/src/soma/soma_dataframe.h | 6 +++--- libtiledbsoma/src/soma/soma_dense_ndarray.cc | 4 ++-- libtiledbsoma/src/soma/soma_dense_ndarray.h | 6 +++--- libtiledbsoma/src/soma/soma_experiment.cc | 2 +- libtiledbsoma/src/soma/soma_experiment.h | 4 ++-- libtiledbsoma/src/soma/soma_measurement.cc | 2 +- libtiledbsoma/src/soma/soma_measurement.h | 4 ++-- libtiledbsoma/src/soma/soma_object.cc | 2 +- libtiledbsoma/src/soma/soma_object.h | 2 +- libtiledbsoma/src/soma/soma_sparse_ndarray.cc | 4 ++-- libtiledbsoma/src/soma/soma_sparse_ndarray.h | 6 +++--- 14 files changed, 26 insertions(+), 26 deletions(-) diff --git a/libtiledbsoma/src/soma/soma_collection.cc b/libtiledbsoma/src/soma/soma_collection.cc index dff721027f..f4b25de7b0 100644 --- a/libtiledbsoma/src/soma/soma_collection.cc +++ b/libtiledbsoma/src/soma/soma_collection.cc @@ -52,7 +52,7 @@ std::unique_ptr SOMACollection::open( std::string_view uri, OpenMode mode, std::shared_ptr ctx, - std::optional> timestamp) { + std::optional timestamp) { return std::make_unique(mode, uri, ctx, timestamp); } diff --git a/libtiledbsoma/src/soma/soma_collection.h b/libtiledbsoma/src/soma/soma_collection.h index f022752ef5..e8e6ff6e2a 100644 --- a/libtiledbsoma/src/soma/soma_collection.h +++ b/libtiledbsoma/src/soma/soma_collection.h @@ -80,7 +80,7 @@ class SOMACollection : public SOMAGroup { std::string_view uri, OpenMode mode, std::shared_ptr ctx, - std::optional> timestamp = std::nullopt); + std::optional timestamp = std::nullopt); //=================================================================== //= public non-static @@ -99,7 +99,7 @@ class SOMACollection : public SOMAGroup { OpenMode mode, std::string_view uri, std::shared_ptr ctx, - std::optional> timestamp) + std::optional timestamp) : SOMAGroup( mode, uri, diff --git a/libtiledbsoma/src/soma/soma_dataframe.cc b/libtiledbsoma/src/soma/soma_dataframe.cc index 6058b81968..8577428d01 100644 --- a/libtiledbsoma/src/soma/soma_dataframe.cc +++ b/libtiledbsoma/src/soma/soma_dataframe.cc @@ -45,7 +45,7 @@ void SOMADataFrame::create( ColumnIndexInfo index_columns, std::shared_ptr ctx, std::optional platform_config, - std::optional> timestamp) { + std::optional timestamp) { auto tiledb_schema = ArrowAdapter::tiledb_schema_from_arrow_schema( ctx->tiledb_ctx(), schema, index_columns, platform_config); SOMAArray::create(ctx, uri, tiledb_schema, "SOMADataFrame", timestamp); @@ -57,7 +57,7 @@ std::unique_ptr SOMADataFrame::open( std::shared_ptr ctx, std::vector column_names, ResultOrder result_order, - std::optional> timestamp) { + std::optional timestamp) { return std::make_unique( mode, uri, ctx, column_names, result_order, timestamp); } diff --git a/libtiledbsoma/src/soma/soma_dataframe.h b/libtiledbsoma/src/soma/soma_dataframe.h index 6f163d9f2c..22b0f11c59 100644 --- a/libtiledbsoma/src/soma/soma_dataframe.h +++ b/libtiledbsoma/src/soma/soma_dataframe.h @@ -62,7 +62,7 @@ class SOMADataFrame : public SOMAArray { ColumnIndexInfo index_columns, std::shared_ptr ctx, std::optional platform_config = std::nullopt, - std::optional> timestamp = std::nullopt); + std::optional timestamp = std::nullopt); /** * @brief Open and return a SOMADataFrame object at the given URI. @@ -86,7 +86,7 @@ class SOMADataFrame : public SOMAArray { std::shared_ptr ctx, std::vector column_names = {}, ResultOrder result_order = ResultOrder::automatic, - std::optional> timestamp = std::nullopt); + std::optional timestamp = std::nullopt); /** * @brief Check if the SOMADataFrame exists at the URI. @@ -116,7 +116,7 @@ class SOMADataFrame : public SOMAArray { std::shared_ptr ctx, std::vector column_names, ResultOrder result_order, - std::optional> timestamp = std::nullopt) + std::optional timestamp = std::nullopt) : SOMAArray( mode, uri, diff --git a/libtiledbsoma/src/soma/soma_dense_ndarray.cc b/libtiledbsoma/src/soma/soma_dense_ndarray.cc index 1bcc1c4efd..fa093941ec 100644 --- a/libtiledbsoma/src/soma/soma_dense_ndarray.cc +++ b/libtiledbsoma/src/soma/soma_dense_ndarray.cc @@ -42,7 +42,7 @@ void SOMADenseNDArray::create( std::string_view uri, ArraySchema schema, std::shared_ptr ctx, - std::optional> timestamp) { + std::optional timestamp) { SOMAArray::create(ctx, uri, schema, "SOMADenseNDArray", timestamp); } @@ -52,7 +52,7 @@ std::unique_ptr SOMADenseNDArray::open( std::shared_ptr ctx, std::vector column_names, ResultOrder result_order, - std::optional> timestamp) { + std::optional timestamp) { return std::make_unique( mode, uri, ctx, column_names, result_order, timestamp); } diff --git a/libtiledbsoma/src/soma/soma_dense_ndarray.h b/libtiledbsoma/src/soma/soma_dense_ndarray.h index a55ba630d3..0403a0a171 100644 --- a/libtiledbsoma/src/soma/soma_dense_ndarray.h +++ b/libtiledbsoma/src/soma/soma_dense_ndarray.h @@ -61,7 +61,7 @@ class SOMADenseNDArray : public SOMAArray { std::string_view uri, ArraySchema schema, std::shared_ptr ctx, - std::optional> timestamp = std::nullopt); + std::optional timestamp = std::nullopt); /** * @brief Open and return a SOMADenseNDArray object at the given URI. @@ -85,7 +85,7 @@ class SOMADenseNDArray : public SOMAArray { std::shared_ptr ctx, std::vector column_names = {}, ResultOrder result_order = ResultOrder::automatic, - std::optional> timestamp = std::nullopt); + std::optional timestamp = std::nullopt); /** * @brief Check if the SOMADenseNDArray exists at the URI. @@ -114,7 +114,7 @@ class SOMADenseNDArray : public SOMAArray { std::shared_ptr ctx, std::vector column_names, ResultOrder result_order, - std::optional> timestamp) + std::optional timestamp) : SOMAArray( mode, uri, diff --git a/libtiledbsoma/src/soma/soma_experiment.cc b/libtiledbsoma/src/soma/soma_experiment.cc index 2c42aea68a..b6cab600a3 100644 --- a/libtiledbsoma/src/soma/soma_experiment.cc +++ b/libtiledbsoma/src/soma/soma_experiment.cc @@ -72,7 +72,7 @@ std::unique_ptr SOMAExperiment::open( std::string_view uri, OpenMode mode, std::shared_ptr ctx, - std::optional> timestamp) { + std::optional timestamp) { return std::make_unique(mode, uri, ctx, timestamp); } } // namespace tiledbsoma diff --git a/libtiledbsoma/src/soma/soma_experiment.h b/libtiledbsoma/src/soma/soma_experiment.h index 14b918a269..fdbe914b84 100644 --- a/libtiledbsoma/src/soma/soma_experiment.h +++ b/libtiledbsoma/src/soma/soma_experiment.h @@ -76,7 +76,7 @@ class SOMAExperiment : public SOMACollection { std::string_view uri, OpenMode mode, std::shared_ptr ctx, - std::optional> timestamp = std::nullopt); + std::optional timestamp = std::nullopt); //=================================================================== //= public non-static @@ -86,7 +86,7 @@ class SOMAExperiment : public SOMACollection { OpenMode mode, std::string_view uri, std::shared_ptr ctx, - std::optional> timestamp = std::nullopt) + std::optional timestamp = std::nullopt) : SOMACollection(mode, uri, ctx, timestamp) { } diff --git a/libtiledbsoma/src/soma/soma_measurement.cc b/libtiledbsoma/src/soma/soma_measurement.cc index d6605e7383..dc19e78bc9 100644 --- a/libtiledbsoma/src/soma/soma_measurement.cc +++ b/libtiledbsoma/src/soma/soma_measurement.cc @@ -79,7 +79,7 @@ std::unique_ptr SOMAMeasurement::open( std::string_view uri, OpenMode mode, std::shared_ptr ctx, - std::optional> timestamp) { + std::optional timestamp) { return std::make_unique(mode, uri, ctx, timestamp); } } // namespace tiledbsoma diff --git a/libtiledbsoma/src/soma/soma_measurement.h b/libtiledbsoma/src/soma/soma_measurement.h index 591c057751..2500a1321b 100644 --- a/libtiledbsoma/src/soma/soma_measurement.h +++ b/libtiledbsoma/src/soma/soma_measurement.h @@ -77,7 +77,7 @@ class SOMAMeasurement : public SOMACollection { std::string_view uri, OpenMode mode, std::shared_ptr ctx, - std::optional> timestamp = std::nullopt); + std::optional timestamp = std::nullopt); //=================================================================== //= public non-static @@ -86,7 +86,7 @@ class SOMAMeasurement : public SOMACollection { OpenMode mode, std::string_view uri, std::shared_ptr ctx, - std::optional> timestamp = std::nullopt) + std::optional timestamp = std::nullopt) : SOMACollection(mode, uri, ctx, timestamp) { } diff --git a/libtiledbsoma/src/soma/soma_object.cc b/libtiledbsoma/src/soma/soma_object.cc index 4c2828b355..7d51b40d69 100644 --- a/libtiledbsoma/src/soma/soma_object.cc +++ b/libtiledbsoma/src/soma/soma_object.cc @@ -18,7 +18,7 @@ std::unique_ptr SOMAObject::open( std::string_view uri, OpenMode mode, std::shared_ptr ctx, - std::optional> timestamp, + std::optional timestamp, std::optional soma_type) { if (soma_type == std::nullopt) { auto tiledb_type = Object::object(*ctx->tiledb_ctx(), std::string(uri)) diff --git a/libtiledbsoma/src/soma/soma_object.h b/libtiledbsoma/src/soma/soma_object.h index 048796fc84..b2067696ef 100644 --- a/libtiledbsoma/src/soma/soma_object.h +++ b/libtiledbsoma/src/soma/soma_object.h @@ -56,7 +56,7 @@ class SOMAObject { std::string_view uri, OpenMode mode, std::shared_ptr ctx, - std::optional> timestamp = std::nullopt, + std::optional timestamp = std::nullopt, std::optional soma_type = std::nullopt); /** diff --git a/libtiledbsoma/src/soma/soma_sparse_ndarray.cc b/libtiledbsoma/src/soma/soma_sparse_ndarray.cc index f7b0cd4ecd..074e9536c6 100644 --- a/libtiledbsoma/src/soma/soma_sparse_ndarray.cc +++ b/libtiledbsoma/src/soma/soma_sparse_ndarray.cc @@ -43,7 +43,7 @@ void SOMASparseNDArray::create( std::string_view uri, ArraySchema schema, std::shared_ptr ctx, - std::optional> timestamp) { + std::optional timestamp) { SOMAArray::create(ctx, uri, schema, "SOMASparseNDArray", timestamp); } @@ -53,7 +53,7 @@ std::unique_ptr SOMASparseNDArray::open( std::shared_ptr ctx, std::vector column_names, ResultOrder result_order, - std::optional> timestamp) { + std::optional timestamp) { return std::make_unique( mode, uri, ctx, column_names, result_order, timestamp); } diff --git a/libtiledbsoma/src/soma/soma_sparse_ndarray.h b/libtiledbsoma/src/soma/soma_sparse_ndarray.h index 4bd9cddee7..1cc4ed8904 100644 --- a/libtiledbsoma/src/soma/soma_sparse_ndarray.h +++ b/libtiledbsoma/src/soma/soma_sparse_ndarray.h @@ -61,7 +61,7 @@ class SOMASparseNDArray : public SOMAArray { std::string_view uri, ArraySchema schema, std::shared_ptr ctx, - std::optional> timestamp = std::nullopt); + std::optional timestamp = std::nullopt); /** * @brief Open and return a SOMASparseNDArray object at the given URI. @@ -85,7 +85,7 @@ class SOMASparseNDArray : public SOMAArray { std::shared_ptr ctx, std::vector column_names = {}, ResultOrder result_order = ResultOrder::automatic, - std::optional> timestamp = std::nullopt); + std::optional timestamp = std::nullopt); /** * @brief Check if the SOMASparseNDArray exists at the URI. @@ -114,7 +114,7 @@ class SOMASparseNDArray : public SOMAArray { std::shared_ptr ctx, std::vector column_names, ResultOrder result_order, - std::optional> timestamp) + std::optional timestamp) : SOMAArray( mode, uri, From 11670cc485aba6d738f4467b50cde8fc70a53d55 Mon Sep 17 00:00:00 2001 From: Vivian Nguyen Date: Thu, 11 Apr 2024 20:57:43 -0500 Subject: [PATCH 07/10] Use unique_ptr --- libtiledbsoma/src/soma/soma_collection.cc | 18 +++++----- libtiledbsoma/src/soma/soma_collection.h | 6 ++-- libtiledbsoma/src/soma/soma_dataframe.cc | 6 ++-- libtiledbsoma/src/soma/soma_dataframe.h | 4 +-- libtiledbsoma/src/soma/soma_dense_ndarray.cc | 2 +- libtiledbsoma/src/soma/soma_dense_ndarray.h | 2 +- libtiledbsoma/src/soma/soma_experiment.cc | 4 +-- libtiledbsoma/src/soma/soma_experiment.h | 2 +- libtiledbsoma/src/soma/soma_measurement.cc | 4 +-- libtiledbsoma/src/soma/soma_measurement.h | 2 +- libtiledbsoma/src/soma/soma_sparse_ndarray.cc | 2 +- libtiledbsoma/src/soma/soma_sparse_ndarray.h | 2 +- libtiledbsoma/src/utils/arrow_adapter.cc | 2 +- libtiledbsoma/src/utils/arrow_adapter.h | 2 +- libtiledbsoma/test/common.cc | 6 ++-- libtiledbsoma/test/common.h | 2 +- libtiledbsoma/test/unit_column_buffer.cc | 2 +- libtiledbsoma/test/unit_managed_query.cc | 2 +- libtiledbsoma/test/unit_soma_array.cc | 5 +-- libtiledbsoma/test/unit_soma_collection.cc | 35 ++++++++++++++++--- libtiledbsoma/test/unit_soma_dataframe.cc | 9 +++-- libtiledbsoma/test/unit_soma_group.cc | 2 +- 22 files changed, 77 insertions(+), 44 deletions(-) diff --git a/libtiledbsoma/src/soma/soma_collection.cc b/libtiledbsoma/src/soma/soma_collection.cc index f4b25de7b0..2c6541b642 100644 --- a/libtiledbsoma/src/soma/soma_collection.cc +++ b/libtiledbsoma/src/soma/soma_collection.cc @@ -110,10 +110,11 @@ std::shared_ptr SOMACollection::add_new_experiment( std::string_view uri, URIType uri_type, std::shared_ptr ctx, - std::shared_ptr schema, + std::unique_ptr schema, ColumnIndexInfo index_columns, std::optional platform_config) { - SOMAExperiment::create(uri, schema, index_columns, ctx, platform_config); + SOMAExperiment::create( + uri, std::move(schema), index_columns, ctx, platform_config); std::shared_ptr member = SOMAExperiment::open( uri, OpenMode::read, ctx); this->set(std::string(uri), uri_type, std::string(key)); @@ -126,9 +127,9 @@ std::shared_ptr SOMACollection::add_new_measurement( std::string_view uri, URIType uri_type, std::shared_ptr ctx, - std::shared_ptr schema, + std::unique_ptr schema, ColumnIndexInfo index_columns) { - SOMAMeasurement::create(uri, schema, index_columns, ctx); + SOMAMeasurement::create(uri, std::move(schema), index_columns, ctx); std::shared_ptr member = SOMAMeasurement::open( uri, OpenMode::read, ctx); this->set(std::string(uri), uri_type, std::string(key)); @@ -141,10 +142,11 @@ std::shared_ptr SOMACollection::add_new_dataframe( std::string_view uri, URIType uri_type, std::shared_ptr ctx, - std::shared_ptr schema, + std::unique_ptr schema, ColumnIndexInfo index_columns, std::optional platform_config) { - SOMADataFrame::create(uri, schema, index_columns, ctx, platform_config); + SOMADataFrame::create( + uri, std::move(schema), index_columns, ctx, platform_config); std::shared_ptr member = SOMADataFrame::open( uri, OpenMode::read, ctx); this->set(std::string(uri), uri_type, std::string(key)); @@ -158,7 +160,7 @@ std::shared_ptr SOMACollection::add_new_dense_ndarray( URIType uri_type, std::shared_ptr ctx, ArraySchema schema) { - SOMADenseNDArray::create(uri, schema, ctx); + SOMADenseNDArray::create(uri, std::move(schema), ctx); std::shared_ptr member = SOMADenseNDArray::open( uri, OpenMode::read, ctx); this->set(std::string(uri), uri_type, std::string(key)); @@ -172,7 +174,7 @@ std::shared_ptr SOMACollection::add_new_sparse_ndarray( URIType uri_type, std::shared_ptr ctx, ArraySchema schema) { - SOMASparseNDArray::create(uri, schema, ctx); + SOMASparseNDArray::create(uri, std::move(schema), ctx); std::shared_ptr member = SOMASparseNDArray::open( uri, OpenMode::read, ctx); this->set(std::string(uri), uri_type, std::string(key)); diff --git a/libtiledbsoma/src/soma/soma_collection.h b/libtiledbsoma/src/soma/soma_collection.h index e8e6ff6e2a..47564b67f3 100644 --- a/libtiledbsoma/src/soma/soma_collection.h +++ b/libtiledbsoma/src/soma/soma_collection.h @@ -157,7 +157,7 @@ class SOMACollection : public SOMAGroup { std::string_view uri, URIType uri_type, std::shared_ptr ctx, - std::shared_ptr schema, + std::unique_ptr schema, ColumnIndexInfo index_columns, std::optional platform_config = std::nullopt); @@ -174,7 +174,7 @@ class SOMACollection : public SOMAGroup { std::string_view uri, URIType uri_type, std::shared_ptr ctx, - std::shared_ptr schema, + std::unique_ptr schema, ColumnIndexInfo index_columns); /** @@ -190,7 +190,7 @@ class SOMACollection : public SOMAGroup { std::string_view uri, URIType uri_type, std::shared_ptr ctx, - std::shared_ptr schema, + std::unique_ptr schema, ColumnIndexInfo index_columns, std::optional platform_config = std::nullopt); diff --git a/libtiledbsoma/src/soma/soma_dataframe.cc b/libtiledbsoma/src/soma/soma_dataframe.cc index 8577428d01..56de8b193b 100644 --- a/libtiledbsoma/src/soma/soma_dataframe.cc +++ b/libtiledbsoma/src/soma/soma_dataframe.cc @@ -41,13 +41,13 @@ using namespace tiledb; void SOMADataFrame::create( std::string_view uri, - std::shared_ptr schema, + std::unique_ptr schema, ColumnIndexInfo index_columns, std::shared_ptr ctx, std::optional platform_config, std::optional timestamp) { auto tiledb_schema = ArrowAdapter::tiledb_schema_from_arrow_schema( - ctx->tiledb_ctx(), schema, index_columns, platform_config); + ctx->tiledb_ctx(), std::move(schema), index_columns, platform_config); SOMAArray::create(ctx, uri, tiledb_schema, "SOMADataFrame", timestamp); } @@ -76,7 +76,7 @@ bool SOMADataFrame::exists(std::string_view uri) { //= public non-static //=================================================================== -std::shared_ptr SOMADataFrame::schema() const { +std::unique_ptr SOMADataFrame::schema() const { return this->arrow_schema(); } diff --git a/libtiledbsoma/src/soma/soma_dataframe.h b/libtiledbsoma/src/soma/soma_dataframe.h index 22b0f11c59..f2e8a2aa63 100644 --- a/libtiledbsoma/src/soma/soma_dataframe.h +++ b/libtiledbsoma/src/soma/soma_dataframe.h @@ -58,7 +58,7 @@ class SOMADataFrame : public SOMAArray { */ static void create( std::string_view uri, - std::shared_ptr schema, + std::unique_ptr schema, ColumnIndexInfo index_columns, std::shared_ptr ctx, std::optional platform_config = std::nullopt, @@ -144,7 +144,7 @@ class SOMADataFrame : public SOMAArray { * * @return std::shared_ptr */ - std::shared_ptr schema() const; + std::unique_ptr schema() const; /** * Return the index (dimension) column names. diff --git a/libtiledbsoma/src/soma/soma_dense_ndarray.cc b/libtiledbsoma/src/soma/soma_dense_ndarray.cc index fa093941ec..ca53fbfbee 100644 --- a/libtiledbsoma/src/soma/soma_dense_ndarray.cc +++ b/libtiledbsoma/src/soma/soma_dense_ndarray.cc @@ -71,7 +71,7 @@ bool SOMADenseNDArray::exists(std::string_view uri) { //= public non-static //=================================================================== -std::shared_ptr SOMADenseNDArray::schema() const { +std::unique_ptr SOMADenseNDArray::schema() const { return this->arrow_schema(); } diff --git a/libtiledbsoma/src/soma/soma_dense_ndarray.h b/libtiledbsoma/src/soma/soma_dense_ndarray.h index 0403a0a171..18b8432cdd 100644 --- a/libtiledbsoma/src/soma/soma_dense_ndarray.h +++ b/libtiledbsoma/src/soma/soma_dense_ndarray.h @@ -151,7 +151,7 @@ class SOMADenseNDArray : public SOMAArray { * * @return std::unique_ptr */ - std::shared_ptr schema() const; + std::unique_ptr schema() const; }; } // namespace tiledbsoma diff --git a/libtiledbsoma/src/soma/soma_experiment.cc b/libtiledbsoma/src/soma/soma_experiment.cc index b6cab600a3..370c79419b 100644 --- a/libtiledbsoma/src/soma/soma_experiment.cc +++ b/libtiledbsoma/src/soma/soma_experiment.cc @@ -43,7 +43,7 @@ using namespace tiledb; void SOMAExperiment::create( std::string_view uri, - std::shared_ptr schema, + std::unique_ptr schema, ColumnIndexInfo index_columns, std::shared_ptr ctx, std::optional platform_config, @@ -53,7 +53,7 @@ void SOMAExperiment::create( SOMAGroup::create(ctx, exp_uri, "SOMAExperiment", timestamp); SOMADataFrame::create( exp_uri + "/obs", - schema, + std::move(schema), index_columns, ctx, platform_config, diff --git a/libtiledbsoma/src/soma/soma_experiment.h b/libtiledbsoma/src/soma/soma_experiment.h index fdbe914b84..43e75eac0c 100644 --- a/libtiledbsoma/src/soma/soma_experiment.h +++ b/libtiledbsoma/src/soma/soma_experiment.h @@ -56,7 +56,7 @@ class SOMAExperiment : public SOMACollection { */ static void create( std::string_view uri, - std::shared_ptr schema, + std::unique_ptr schema, ColumnIndexInfo index_columns, std::shared_ptr ctx, std::optional platform_config = std::nullopt, diff --git a/libtiledbsoma/src/soma/soma_measurement.cc b/libtiledbsoma/src/soma/soma_measurement.cc index dc19e78bc9..dc043bc1fe 100644 --- a/libtiledbsoma/src/soma/soma_measurement.cc +++ b/libtiledbsoma/src/soma/soma_measurement.cc @@ -43,7 +43,7 @@ using namespace tiledb; void SOMAMeasurement::create( std::string_view uri, - std::shared_ptr schema, + std::unique_ptr schema, ColumnIndexInfo index_columns, std::shared_ptr ctx, std::optional platform_config, @@ -53,7 +53,7 @@ void SOMAMeasurement::create( SOMAGroup::create(ctx, exp_uri, "SOMAMeasurement", timestamp); SOMADataFrame::create( exp_uri + "/var", - schema, + std::move(schema), index_columns, ctx, platform_config, diff --git a/libtiledbsoma/src/soma/soma_measurement.h b/libtiledbsoma/src/soma/soma_measurement.h index 2500a1321b..0d60702723 100644 --- a/libtiledbsoma/src/soma/soma_measurement.h +++ b/libtiledbsoma/src/soma/soma_measurement.h @@ -57,7 +57,7 @@ class SOMAMeasurement : public SOMACollection { */ static void create( std::string_view uri, - std::shared_ptr schema, + std::unique_ptr schema, ColumnIndexInfo index_columns, std::shared_ptr ctx, std::optional platform_config = std::nullopt, diff --git a/libtiledbsoma/src/soma/soma_sparse_ndarray.cc b/libtiledbsoma/src/soma/soma_sparse_ndarray.cc index 074e9536c6..73358dff5a 100644 --- a/libtiledbsoma/src/soma/soma_sparse_ndarray.cc +++ b/libtiledbsoma/src/soma/soma_sparse_ndarray.cc @@ -72,7 +72,7 @@ bool SOMASparseNDArray::exists(std::string_view uri) { //= public non-static //=================================================================== -std::shared_ptr SOMASparseNDArray::schema() const { +std::unique_ptr SOMASparseNDArray::schema() const { return this->arrow_schema(); } } // namespace tiledbsoma diff --git a/libtiledbsoma/src/soma/soma_sparse_ndarray.h b/libtiledbsoma/src/soma/soma_sparse_ndarray.h index 1cc4ed8904..e212dee78b 100644 --- a/libtiledbsoma/src/soma/soma_sparse_ndarray.h +++ b/libtiledbsoma/src/soma/soma_sparse_ndarray.h @@ -151,7 +151,7 @@ class SOMASparseNDArray : public SOMAArray { * * @return std::shared_ptr */ - std::shared_ptr schema() const; + std::unique_ptr schema() const; }; } // namespace tiledbsoma diff --git a/libtiledbsoma/src/utils/arrow_adapter.cc b/libtiledbsoma/src/utils/arrow_adapter.cc index 146cf99679..e4f5d2dc17 100644 --- a/libtiledbsoma/src/utils/arrow_adapter.cc +++ b/libtiledbsoma/src/utils/arrow_adapter.cc @@ -223,7 +223,7 @@ std::unique_ptr ArrowAdapter::arrow_schema_from_tiledb_array( ArraySchema ArrowAdapter::tiledb_schema_from_arrow_schema( std::shared_ptr ctx, - std::shared_ptr arrow_schema, + std::unique_ptr arrow_schema, ColumnIndexInfo index_column_info, std::optional platform_config) { auto [index_column_names, domains, extents] = index_column_info; diff --git a/libtiledbsoma/src/utils/arrow_adapter.h b/libtiledbsoma/src/utils/arrow_adapter.h index 5ac12bca38..367c34a8cd 100644 --- a/libtiledbsoma/src/utils/arrow_adapter.h +++ b/libtiledbsoma/src/utils/arrow_adapter.h @@ -88,7 +88,7 @@ class ArrowAdapter { */ static ArraySchema tiledb_schema_from_arrow_schema( std::shared_ptr ctx, - std::shared_ptr arrow_schema, + std::unique_ptr arrow_schema, ColumnIndexInfo index_column_info, std::optional platform_config); diff --git a/libtiledbsoma/test/common.cc b/libtiledbsoma/test/common.cc index f497908dd6..f21c3c4afc 100644 --- a/libtiledbsoma/test/common.cc +++ b/libtiledbsoma/test/common.cc @@ -51,9 +51,9 @@ ArraySchema create_schema(Context& ctx, bool allow_duplicates) { return schema; } -std::pair, ColumnIndexInfo> create_arrow_schema() { +std::pair, ColumnIndexInfo> create_arrow_schema() { // Create ArrowSchema - auto arrow_schema = std::make_shared(); + auto arrow_schema = std::make_unique(); arrow_schema->format = "+s"; arrow_schema->n_children = 2; arrow_schema->dictionary = nullptr; @@ -128,6 +128,6 @@ std::pair, ColumnIndexInfo> create_arrow_schema() { ColumnIndexInfo index_columns_info = std::tuple( index_column_names, domains, tiles); - return std::pair(arrow_schema, index_columns_info); + return std::pair(std::move(arrow_schema), index_columns_info); } } // namespace helper \ No newline at end of file diff --git a/libtiledbsoma/test/common.h b/libtiledbsoma/test/common.h index 16ce7e4bbd..d157cdce38 100644 --- a/libtiledbsoma/test/common.h +++ b/libtiledbsoma/test/common.h @@ -61,6 +61,6 @@ static const std::string src_path = TILEDBSOMA_SOURCE_ROOT; namespace helper { ArraySchema create_schema(Context& ctx, bool allow_duplicates = false); -std::pair, ColumnIndexInfo> create_arrow_schema(); +std::pair, ColumnIndexInfo> create_arrow_schema(); } // namespace helper #endif \ No newline at end of file diff --git a/libtiledbsoma/test/unit_column_buffer.cc b/libtiledbsoma/test/unit_column_buffer.cc index e9eb6ba4fc..79b04308dc 100644 --- a/libtiledbsoma/test/unit_column_buffer.cc +++ b/libtiledbsoma/test/unit_column_buffer.cc @@ -74,7 +74,7 @@ static std::shared_ptr create_array( attr.set_cell_val_num(TILEDB_VAR_NUM); schema.add_attribute(attr); - Array::create(uri, schema); + Array::create(uri, std::move(schema)); return std::make_shared(ctx, uri, TILEDB_READ); } diff --git a/libtiledbsoma/test/unit_managed_query.cc b/libtiledbsoma/test/unit_managed_query.cc index 6797c2d5eb..ee708533c5 100644 --- a/libtiledbsoma/test/unit_managed_query.cc +++ b/libtiledbsoma/test/unit_managed_query.cc @@ -81,7 +81,7 @@ auto create_array(const std::string& uri, Context& ctx) { schema.check(); // Create array and open for writing - Array::create(uri, schema); + Array::create(uri, std::move(schema)); Array array(ctx, uri, TILEDB_WRITE); std::vector d0 = { diff --git a/libtiledbsoma/test/unit_soma_array.cc b/libtiledbsoma/test/unit_soma_array.cc index 07c4b7abf5..f27a71680f 100644 --- a/libtiledbsoma/test/unit_soma_array.cc +++ b/libtiledbsoma/test/unit_soma_array.cc @@ -86,7 +86,8 @@ std::tuple create_array( schema.check(); // Create array - SOMAArray::create(ctx, uri, schema, "NONE", TimestampRange(0, 2)); + SOMAArray::create( + ctx, uri, std::move(schema), "NONE", TimestampRange(0, 2)); uint64_t nnz = num_fragments * num_cells_per_fragment; @@ -462,7 +463,7 @@ TEST_CASE("SOMAArray: Enumeration") { *ctx->tiledb_ctx(), attr, "rbg"); schema.add_attribute(attr); - Array::create(uri, schema); + Array::create(uri, std::move(schema)); auto soma_array = SOMAArray::open(OpenMode::read, uri, ctx); auto attr_to_enum = soma_array->get_attr_to_enum_mapping(); diff --git a/libtiledbsoma/test/unit_soma_collection.cc b/libtiledbsoma/test/unit_soma_collection.cc index 914e263c70..b6551b57f0 100644 --- a/libtiledbsoma/test/unit_soma_collection.cc +++ b/libtiledbsoma/test/unit_soma_collection.cc @@ -113,7 +113,12 @@ TEST_CASE("SOMACollection: add SOMADataFrame") { auto soma_collection = SOMACollection::open(base_uri, OpenMode::write, ctx); auto soma_dataframe = soma_collection->add_new_dataframe( - "dataframe", sub_uri, URIType::absolute, ctx, schema, index_columns); + "dataframe", + sub_uri, + URIType::absolute, + ctx, + std::move(schema), + index_columns); REQUIRE(soma_collection->member_to_uri_mapping() == expected_map); REQUIRE(soma_dataframe->uri() == sub_uri); REQUIRE(soma_dataframe->ctx() == ctx); @@ -165,7 +170,12 @@ TEST_CASE("SOMACollection: add SOMAExperiment") { auto soma_collection = SOMACollection::open(base_uri, OpenMode::write, ctx); auto soma_experiment = soma_collection->add_new_experiment( - "experiment", sub_uri, URIType::absolute, ctx, schema, index_columns); + "experiment", + sub_uri, + URIType::absolute, + ctx, + std::move(schema), + index_columns); REQUIRE(soma_collection->member_to_uri_mapping() == expected_map); REQUIRE(soma_experiment->uri() == sub_uri); REQUIRE(soma_experiment->ctx() == ctx); @@ -190,7 +200,12 @@ TEST_CASE("SOMACollection: add SOMAMeasurement") { auto soma_collection = SOMACollection::open(base_uri, OpenMode::write, ctx); auto soma_measurement = soma_collection->add_new_measurement( - "measurement", sub_uri, URIType::absolute, ctx, schema, index_columns); + "measurement", + sub_uri, + URIType::absolute, + ctx, + std::move(schema), + index_columns); REQUIRE(soma_collection->member_to_uri_mapping() == expected_map); REQUIRE(soma_measurement->uri() == sub_uri); REQUIRE(soma_measurement->ctx() == ctx); @@ -262,7 +277,12 @@ TEST_CASE("SOMAExperiment: metadata") { std::string uri = "mem://unit-test-experiment"; auto [schema, index_columns] = helper::create_arrow_schema(); SOMAExperiment::create( - uri, schema, index_columns, ctx, std::nullopt, TimestampRange(0, 2)); + uri, + std::move(schema), + index_columns, + ctx, + std::nullopt, + TimestampRange(0, 2)); auto soma_experiment = SOMAExperiment::open( uri, OpenMode::write, ctx, std::pair(1, 1)); @@ -316,7 +336,12 @@ TEST_CASE("SOMAMeasurement: metadata") { std::string uri = "mem://unit-test-measurement"; auto [schema, index_columns] = helper::create_arrow_schema(); SOMAMeasurement::create( - uri, schema, index_columns, ctx, std::nullopt, TimestampRange(0, 2)); + uri, + std::move(schema), + index_columns, + ctx, + std::nullopt, + TimestampRange(0, 2)); auto soma_measurement = SOMAMeasurement::open( uri, OpenMode::write, ctx, std::pair(1, 1)); diff --git a/libtiledbsoma/test/unit_soma_dataframe.cc b/libtiledbsoma/test/unit_soma_dataframe.cc index a5eb783aca..bd21e0c7b9 100644 --- a/libtiledbsoma/test/unit_soma_dataframe.cc +++ b/libtiledbsoma/test/unit_soma_dataframe.cc @@ -37,7 +37,7 @@ TEST_CASE("SOMADataFrame: basic") { std::string uri = "mem://unit-test-dataframe-basic"; auto [schema, index_columns] = helper::create_arrow_schema(); - SOMADataFrame::create(uri, schema, index_columns, ctx); + SOMADataFrame::create(uri, std::move(schema), index_columns, ctx); auto soma_dataframe = SOMADataFrame::open(uri, OpenMode::read, ctx); REQUIRE(soma_dataframe->uri() == uri); @@ -85,7 +85,12 @@ TEST_CASE("SOMADataFrame: metadata") { std::string uri = "mem://unit-test-collection"; auto [schema, index_columns] = helper::create_arrow_schema(); SOMADataFrame::create( - uri, schema, index_columns, ctx, std::nullopt, TimestampRange(0, 2)); + uri, + std::move(schema), + index_columns, + ctx, + std::nullopt, + TimestampRange(0, 2)); auto soma_dataframe = SOMADataFrame::open( uri, diff --git a/libtiledbsoma/test/unit_soma_group.cc b/libtiledbsoma/test/unit_soma_group.cc index c04030037b..8e2454313a 100644 --- a/libtiledbsoma/test/unit_soma_group.cc +++ b/libtiledbsoma/test/unit_soma_group.cc @@ -91,7 +91,7 @@ std::tuple create_array( schema.check(); // Create array - Array::create(uri, schema); + Array::create(uri, std::move(schema)); } // Open array for writing From 1e70f975b3dfe14c73f7c3c68d1cc7eff6b92d79 Mon Sep 17 00:00:00 2001 From: Vivian Nguyen Date: Fri, 12 Apr 2024 08:17:41 -0500 Subject: [PATCH 08/10] Set flags to 0 --- libtiledbsoma/test/common.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/libtiledbsoma/test/common.cc b/libtiledbsoma/test/common.cc index f21c3c4afc..7826564708 100644 --- a/libtiledbsoma/test/common.cc +++ b/libtiledbsoma/test/common.cc @@ -73,6 +73,7 @@ std::pair, ColumnIndexInfo> create_arrow_schema() { attr->format = "l"; attr->name = "a0"; attr->n_children = 0; + attr->flags = 0; attr->dictionary = nullptr; attr->release = &ArrowAdapter::release_schema; From 2ab5c8ea3324514d808185aecd74fb88de971849 Mon Sep 17 00:00:00 2001 From: Vivian Nguyen Date: Wed, 17 Apr 2024 12:05:41 -0500 Subject: [PATCH 09/10] Correct documentation --- libtiledbsoma/src/soma/soma_array.h | 7 +++---- libtiledbsoma/src/soma/soma_dataframe.h | 17 ++++++++++------- libtiledbsoma/src/soma/soma_dense_ndarray.h | 20 ++++++++++---------- libtiledbsoma/src/soma/soma_sparse_ndarray.h | 18 +++++++++--------- 4 files changed, 32 insertions(+), 30 deletions(-) diff --git a/libtiledbsoma/src/soma/soma_array.h b/libtiledbsoma/src/soma/soma_array.h index 7d0e379731..65049be95b 100644 --- a/libtiledbsoma/src/soma/soma_array.h +++ b/libtiledbsoma/src/soma/soma_array.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2022-2023 TileDB, Inc. + * @copyright Copyright (c) 2022-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -102,8 +102,7 @@ class SOMAArray : public SOMAObject { * * @param mode read or write * @param uri URI of the array - * @param name Name of the array - * @param platform_config Config parameter dictionary + * @param ctx SOMAContext * @param column_names Columns to read * @param batch_size Read batch size * @param result_order Read result order: automatic (default), rowmajor, @@ -152,8 +151,8 @@ class SOMAArray : public SOMAObject { * * @param mode read or write * @param uri URI of the array + * @param ctx SOMAContext * @param name name of the array - * @param platform_config Config parameter dictionary * @param column_names Columns to read * @param batch_size Batch size * @param result_order Result order diff --git a/libtiledbsoma/src/soma/soma_dataframe.h b/libtiledbsoma/src/soma/soma_dataframe.h index f2e8a2aa63..fe95efdb66 100644 --- a/libtiledbsoma/src/soma/soma_dataframe.h +++ b/libtiledbsoma/src/soma/soma_dataframe.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2023 TileDB, Inc. + * @copyright Copyright (c) 2023-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -53,8 +53,12 @@ class SOMADataFrame : public SOMAArray { * @brief Create a SOMADataFrame object at the given URI. * * @param uri URI to create the SOMADataFrame - * @param schema TileDB ArraySchema + * @param schema Arrow schema + * @param index_columns The index column names with associated domains + * and tile extents per dimension + * @param ctx SOMAContext * @param platform_config Optional config parameter dictionary + * @param timestamp Optional the timestamp range to write SOMA metadata info */ static void create( std::string_view uri, @@ -67,18 +71,17 @@ class SOMADataFrame : public SOMAArray { /** * @brief Open and return a SOMADataFrame object at the given URI. * - * @param mode read or write * @param uri URI to create the SOMADataFrame + * @param mode read or write + * @param ctx SOMAContext * @param column_names A list of column names to use as user-defined index * columns (e.g., ``['cell_type', 'tissue_type']``). All named columns must * exist in the schema, and at least one index column name is required. - * @param platform_config Platform-specific options used to create this - * DataFrame * @param result_order Read result order: automatic (default), rowmajor, or * colmajor * @param timestamp If specified, overrides the default timestamp used to * open this object. If unset, uses the timestamp provided by the context. - * @return std::shared_ptr SOMADataFrame + * @return std::unique_ptr SOMADataFrame */ static std::unique_ptr open( std::string_view uri, @@ -142,7 +145,7 @@ class SOMADataFrame : public SOMAArray { /** * Return the data schema, in the form of a ArrowSchema. * - * @return std::shared_ptr + * @return std::unique_ptr */ std::unique_ptr schema() const; diff --git a/libtiledbsoma/src/soma/soma_dense_ndarray.h b/libtiledbsoma/src/soma/soma_dense_ndarray.h index 18b8432cdd..a1997019a0 100644 --- a/libtiledbsoma/src/soma/soma_dense_ndarray.h +++ b/libtiledbsoma/src/soma/soma_dense_ndarray.h @@ -5,7 +5,7 @@ * * The MIT License * - * @copyright Copyright (c) 2023 TileDB, Inc. + * @copyright Copyright (c) 2023-2024 TileDB, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -53,9 +53,9 @@ class SOMADenseNDArray : public SOMAArray { * @brief Create a SOMADenseNDArray object at the given URI. * * @param uri URI to create the SOMADenseNDArray - * @param schema TileDB ArraySchema - * @param platform_config Optional config parameter dictionary - * @return std::shared_ptr opened in read mode + * @param schema Arrow schema + * @param ctx SOMAContext + * @param timestamp Optional the timestamp range to write SOMA metadata info */ static void create( std::string_view uri, @@ -66,18 +66,17 @@ class SOMADenseNDArray : public SOMAArray { /** * @brief Open and return a SOMADenseNDArray object at the given URI. * - * @param mode read or write * @param uri URI to create the SOMADenseNDArray + * @param mode read or write + * @param ctx SOMAContext * @param column_names A list of column names to use as user-defined index * columns (e.g., ``['cell_type', 'tissue_type']``). All named columns must * exist in the schema, and at least one index column name is required. - * @param platform_config Platform-specific options used to create this - * SOMADenseNDArray - * @param timestamp If specified, overrides the default timestamp used to - * open this object. If unset, uses the timestamp provided by the context. * @param result_order Read result order: automatic (default), rowmajor, or * colmajor - * @return std::shared_ptr SOMADenseNDArray + * @param timestamp If specified, overrides the default timestamp used to + * open this object. If unset, uses the timestamp provided by the context. + * @return std::unique_ptr SOMADenseNDArray */ static std::unique_ptr open( std::string_view uri, @@ -104,6 +103,7 @@ class SOMADenseNDArray : public SOMAArray { * @param mode read or write * @param uri URI of the array * @param ctx TileDB context + * @param column_names Columns to read * @param result_order Read result order: automatic (default), rowmajor, or * colmajor * @param timestamp Timestamp diff --git a/libtiledbsoma/src/soma/soma_sparse_ndarray.h b/libtiledbsoma/src/soma/soma_sparse_ndarray.h index e212dee78b..db3b251a53 100644 --- a/libtiledbsoma/src/soma/soma_sparse_ndarray.h +++ b/libtiledbsoma/src/soma/soma_sparse_ndarray.h @@ -53,9 +53,9 @@ class SOMASparseNDArray : public SOMAArray { * @brief Create a SOMASparseNDArray object at the given URI. * * @param uri URI to create the SOMASparseNDArray - * @param schema TileDB ArraySchema - * @param platform_config Optional config parameter dictionary - * @return std::shared_ptr opened in read mode + * @param schema Arrow schema + * @param ctx SOMAContext + * @param timestamp Optional the timestamp range to write SOMA metadata info */ static void create( std::string_view uri, @@ -66,18 +66,17 @@ class SOMASparseNDArray : public SOMAArray { /** * @brief Open and return a SOMASparseNDArray object at the given URI. * - * @param mode read or write * @param uri URI to create the SOMASparseNDArray + * @param mode read or write + * @param ctx SOMAContext * @param column_names A list of column names to use as user-defined index * columns (e.g., ``['cell_type', 'tissue_type']``). All named columns must * exist in the schema, and at least one index column name is required. - * @param platform_config Platform-specific options used to create this - * SOMASparseNDArray * @param result_order Read result order: automatic (default), rowmajor, or * colmajor * @param timestamp If specified, overrides the default timestamp used to * open this object. If unset, uses the timestamp provided by the context. - * @return std::shared_ptr SOMASparseNDArray + * @return std::unique_ptr SOMASparseNDArray */ static std::unique_ptr open( std::string_view uri, @@ -104,6 +103,7 @@ class SOMASparseNDArray : public SOMAArray { * @param mode read or write * @param uri URI of the array * @param ctx TileDB context + * @param column_names Columns to read * @param result_order Read result order: automatic (default), rowmajor, or * colmajor * @param timestamp Timestamp @@ -138,7 +138,7 @@ class SOMASparseNDArray : public SOMAArray { using SOMAArray::open; /** - * Return whether the NDArray is sparse. + * Return whether the SOMASparseNDArray is sparse. * * @return true */ @@ -149,7 +149,7 @@ class SOMASparseNDArray : public SOMAArray { /** * Return the data schema, in the form of an ArrowSchema. * - * @return std::shared_ptr + * @return std::unique_ptr */ std::unique_ptr schema() const; }; From ae8dac4e6c7fd27035195ad2fe45b91aebaa4cb7 Mon Sep 17 00:00:00 2001 From: Vivian Nguyen Date: Wed, 17 Apr 2024 12:25:05 -0500 Subject: [PATCH 10/10] clang-formatting --- libtiledbsoma/src/soma/soma_dataframe.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libtiledbsoma/src/soma/soma_dataframe.h b/libtiledbsoma/src/soma/soma_dataframe.h index fe95efdb66..504779913f 100644 --- a/libtiledbsoma/src/soma/soma_dataframe.h +++ b/libtiledbsoma/src/soma/soma_dataframe.h @@ -54,7 +54,7 @@ class SOMADataFrame : public SOMAArray { * * @param uri URI to create the SOMADataFrame * @param schema Arrow schema - * @param index_columns The index column names with associated domains + * @param index_columns The index column names with associated domains * and tile extents per dimension * @param ctx SOMAContext * @param platform_config Optional config parameter dictionary