otherarch/ggml_v3.h

#pragma once

//
// GGML Tensor Library
//
// This documentation is still a work in progress.
// If you wish some specific topics to be covered, feel free to drop a comment:
//
//   https://github.com/ggerganov/whisper.cpp/issues/40
//
// ## Overview
//
// This library implements:
//
//  - a set of tensor operations
//  - automatic differentiation
//  - basic optimization algorithms
//
// The aim of this library is to provide a minimalistic approach for various machine learning tasks. This includes,
// but is not limited to, the following:
//
//  - linear regression
//  - support vector machines
//  - neural networks
//
// The library allows the user to define a certain function using the available tensor operations. This function
// definition is represented internally via a computation graph. Each tensor operation in the function definition
// corresponds to a node in the graph. Having the computation graph defined, the user can choose to compute the
// function's value and/or its gradient with respect to the input variables. Optionally, the function can be optimized
// using one of the available optimization algorithms.
//
// For example, here we define the function: f(x) = a*x^2 + b
//
//   {
//       struct ggml_v3_init_params params = {
//           .mem_size   = 16*1024*1024,
//           .mem_buffer = NULL,
//       };
//
//       // memory allocation happens here
//       struct ggml_v3_context * ctx = ggml_v3_init(params);
//
//       struct ggml_v3_tensor * x = ggml_v3_new_tensor_1d(ctx, GGML_V3_TYPE_F32, 1);
//
//       ggml_v3_set_param(ctx, x); // x is an input variable
//
//       struct ggml_v3_tensor * a  = ggml_v3_new_tensor_1d(ctx, GGML_V3_TYPE_F32, 1);
//       struct ggml_v3_tensor * b  = ggml_v3_new_tensor_1d(ctx, GGML_V3_TYPE_F32, 1);
//       struct ggml_v3_tensor * x2 = ggml_v3_mul(ctx, x, x);
//       struct ggml_v3_tensor * f  = ggml_v3_add(ctx, ggml_v3_mul(ctx, a, x2), b);
//
//       ...
//   }
//
// Notice that the function definition above does not involve any actual computation. The computation is performed only
// when the user explicitly requests it. For example, to compute the function's value at x = 2.0:
//
//   {
//       ...
//
//       struct ggml_v3_cgraph * gf = ggml_v3_new_graph(ctx);
//       ggml_v3_build_forward_expand(gf, f);
//
//       // set the input variable and parameter values
//       ggml_v3_set_f32(x, 2.0f);
//       ggml_v3_set_f32(a, 3.0f);
//       ggml_v3_set_f32(b, 4.0f);
//
//       ggml_v3_graph_compute_with_ctx(ctx, &gf, n_threads);
//
//       printf("f = %f\n", ggml_v3_get_f32_1d(f, 0));
//
//       ...
//   }
//
// The actual computation is performed in the ggml_v3_graph_compute() function.
//
// The ggml_v3_new_tensor_...() functions create new tensors. They are allocated in the memory buffer provided to the
// ggml_v3_init() function. You have to be careful not to exceed the memory buffer size. Therefore, you have to know
// in advance how much memory you need for your computation. Alternatively, you can allocate a large enough memory
// and after defining the computation graph, call the ggml_v3_used_mem() function to find out how much memory was
// actually needed.
//
// The ggml_v3_set_param() function marks a tensor as an input variable. This is used by the automatic
// differentiation and optimization algorithms.
//
// The described approach allows to define the function graph once and then compute its forward or backward graphs
// multiple times. All computations will use the same memory buffer allocated in the ggml_v3_init() function. This way
// the user can avoid the memory allocation overhead at runtime.
//
// The library supports multi-dimensional tensors - up to 4 dimensions. The FP16 and FP32 data types are first class
// citizens, but in theory the library can be extended to support FP8 and integer data types.
//
// Each tensor operation produces a new tensor. Initially the library was envisioned to support only the use of unary
// and binary operations. Most of the available operations fall into one of these two categories. With time, it became
// clear that the library needs to support more complex operations. The way to support these operations is not clear
// yet, but a few examples are demonstrated in the following operations:
//
//   - ggml_v3_permute()
//   - ggml_v3_conv_1d_1s()
//   - ggml_v3_conv_1d_2s()
//
// For each tensor operator, the library implements a forward and backward computation function. The forward function
// computes the output tensor value given the input tensor values. The backward function computes the adjoint of the
// input tensors given the adjoint of the output tensor. For a detailed explanation of what this means, take a
// calculus class, or watch the following video:
//
//   What is Automatic Differentiation?
//   https://www.youtube.com/watch?v=wG_nF1awSSY
//
//
// ## Tensor data (struct ggml_v3_tensor)
//
// The tensors are stored in memory via the ggml_v3_tensor struct. The structure provides information about the size of
// the tensor, the data type, and the memory buffer where the tensor data is stored. Additionally, it contains
// pointers to the "source" tensors - i.e. the tensors that were used to compute the current tensor. For example:
//
//   {
//       struct ggml_v3_tensor * c = ggml_v3_add(ctx, a, b);
//
//       assert(c->src[0] == a);
//       assert(c->src[1] == b);
//   }
//
// The multi-dimensional tensors are stored in row-major order. The ggml_v3_tensor struct contains fields for the
// number of elements in each dimension ("ne") as well as the number of bytes ("nb", a.k.a. stride). This allows
// to store tensors that are not contiguous in memory, which is useful for operations such as transposition and
// permutation. All tensor operations have to take the stride into account and not assume that the tensor is
// contiguous in memory.
//
// The data of the tensor is accessed via the "data" pointer. For example:
//
//   {
//       const int nx = 2;
//       const int ny = 3;
//
//       struct ggml_v3_tensor * a = ggml_v3_new_tensor_2d(ctx, GGML_V3_TYPE_F32, nx, ny);
//
//       for (int y = 0; y < ny; y++) {
//           for (int x = 0; x < nx; x++) {
//               *(float *) ((char *) a->data + y*a->nb[1] + x*a->nb[0]) = x + y;
//           }
//       }
//
//       ...
//   }
//
// Alternatively, there are helper functions, such as ggml_v3_get_f32_1d() and ggml_v3_set_f32_1d() that can be used.
//
// ## The matrix multiplication operator (ggml_v3_mul_mat)
//
// TODO
//
//
// ## Multi-threading
//
// TODO
//
//
// ## Overview of ggml.c
//
// TODO
//
//
// ## SIMD optimizations
//
// TODO
//
//
// ## Debugging ggml
//
// TODO
//
//

#ifdef GGML_V3_SHARED
#    if defined(_WIN32) && !defined(__MINGW32__)
#        ifdef GGML_V3_BUILD
#            define GGML_V3_API __declspec(dllexport)
#        else
#            define GGML_V3_API __declspec(dllimport)
#        endif
#    else
#        define GGML_V3_API __attribute__ ((visibility ("default")))
#    endif
#else
#    define GGML_V3_API
#endif

// TODO: support for clang
#    define GGML_V3_DEPRECATED(func, hint) func


#ifndef __GNUC__
#    define GGML_V3_ATTRIBUTE_FORMAT(...)
#elif defined(__MINGW32__)
#    define GGML_V3_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__)))
#else
#    define GGML_V3_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__)))
#endif

#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>

#define GGML_V3_FILE_MAGIC   0x67676d6c // "ggml"
#define GGML_V3_FILE_VERSION 1

#define GGML_V3_QNT_VERSION        2    // bump this on quantization format changes
#define GGML_V3_QNT_VERSION_FACTOR 1000 // do not change this

#define GGML_V3_MAX_DIMS           4
#define GGML_V3_MAX_PARAMS         2048
#define GGML_V3_MAX_CONTEXTS       64
#define GGML_V3_MAX_SRC            10
#ifndef GGML_V3_MAX_NAME
#define GGML_V3_MAX_NAME           64
#endif
#define GGML_V3_MAX_OP_PARAMS      64
#define GGML_V3_DEFAULT_N_THREADS  4
#define GGML_V3_DEFAULT_GRAPH_SIZE 2048
#if UINTPTR_MAX == 0xFFFFFFFF
    #define GGML_V3_MEM_ALIGN 4
#else
    #define GGML_V3_MEM_ALIGN 16
#endif

#define GGML_V3_EXIT_SUCCESS 0
#define GGML_V3_EXIT_ABORTED 1

#define GGUF_V3_MAGIC "GGUF"

#define GGUF_V3_VERSION 3

#define GGUF_V3_DEFAULT_ALIGNMENT 32

#define GGML_V3_UNUSED(x) (void)(x)

#define GGML_V3_PAD(x, n) (((x) + (n) - 1) & ~((n) - 1))

#define GGML_V3_ASSERT_CONTINUE(x) \
    do { \
        if (!(x)) { \
            fprintf(stderr, "GGML_V3_ASSERT_CONTINUE: %s:%d: %s\n", __FILE__, __LINE__, #x); \
        } \
    } while (0)

#define GGML_V3_ASSERT(x) \
    do { \
        if (!(x)) { \
            fflush(stdout); \
            fprintf(stderr, "GGML_V3_ASSERT: %s:%d: %s\n", __FILE__, __LINE__, #x); \
            ggml_v3_print_backtrace(); \
            abort(); \
        } \
    } while (0)

#ifndef NDEBUG
#define GGML_V3_UNREACHABLE() GGML_V3_ASSERT(!"statement should not be reached")
#elif defined(__GNUC__)
#define GGML_V3_UNREACHABLE() __builtin_unreachable()
#elif defined(_MSC_VER)
#define GGML_V3_UNREACHABLE() __assume(0)
#else
#define GGML_V3_UNREACHABLE() ((void) 0)
#endif

// used to copy the number of elements and stride in bytes of tensors into local variables.
// main purpose is to reduce code duplication and improve readability.
//
// example:
//
//    GGML_V3_TENSOR_LOCALS(int64_t, ne1, src1, ne);
//    GGML_V3_TENSOR_LOCALS(size_t,  nb1, src1, nb);
//
#define GGML_V3_TENSOR_LOCALS_1(type, prefix, pointer, array) \
    const type prefix##0 = (pointer)->array[0]; \
    GGML_V3_UNUSED(prefix##0);
#define GGML_V3_TENSOR_LOCALS_2(type, prefix, pointer, array) \
    GGML_V3_TENSOR_LOCALS_1    (type, prefix, pointer, array) \
    const type prefix##1 = (pointer)->array[1]; \
    GGML_V3_UNUSED(prefix##1);
#define GGML_V3_TENSOR_LOCALS_3(type, prefix, pointer, array) \
    GGML_V3_TENSOR_LOCALS_2    (type, prefix, pointer, array) \
    const type prefix##2 = (pointer)->array[2]; \
    GGML_V3_UNUSED(prefix##2);
#define GGML_V3_TENSOR_LOCALS(type, prefix, pointer, array) \
    GGML_V3_TENSOR_LOCALS_3  (type, prefix, pointer, array) \
    const type prefix##3 = (pointer)->array[3]; \
    GGML_V3_UNUSED(prefix##3);

#define GGML_V3_TENSOR_UNARY_OP_LOCALS \
    GGML_V3_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
    GGML_V3_TENSOR_LOCALS(size_t,  nb0, src0, nb) \
    GGML_V3_TENSOR_LOCALS(int64_t, ne,  dst,  ne) \
    GGML_V3_TENSOR_LOCALS(size_t,  nb,  dst,  nb)

#define GGML_V3_TENSOR_BINARY_OP_LOCALS \
    GGML_V3_TENSOR_LOCALS(int64_t, ne0, src0, ne) \
    GGML_V3_TENSOR_LOCALS(size_t,  nb0, src0, nb) \
    GGML_V3_TENSOR_LOCALS(int64_t, ne1, src1, ne) \
    GGML_V3_TENSOR_LOCALS(size_t,  nb1, src1, nb) \
    GGML_V3_TENSOR_LOCALS(int64_t, ne,  dst,  ne) \
    GGML_V3_TENSOR_LOCALS(size_t,  nb,  dst,  nb)

#ifdef  __cplusplus
extern "C" {
#endif

#if defined(__ARM_NEON) && defined(__CUDACC__)
    typedef half ggml_v3_fp16_t;
#elif defined(__ARM_NEON) && !defined(_MSC_VER)
    typedef __fp16 ggml_v3_fp16_t;
#else
    typedef uint16_t ggml_v3_fp16_t;
#endif

    // convert FP16 <-> FP32
    GGML_V3_API float       ggml_v3_fp16_to_fp32(ggml_v3_fp16_t x);
    GGML_V3_API ggml_v3_fp16_t ggml_v3_fp32_to_fp16(float x);

    GGML_V3_API void ggml_v3_fp16_to_fp32_row(const ggml_v3_fp16_t * x, float * y, int n);
    GGML_V3_API void ggml_v3_fp32_to_fp16_row(const float * x, ggml_v3_fp16_t * y, int n);

    struct ggml_v3_object;
    struct ggml_v3_context;

    enum ggml_v3_type {
        GGML_V3_TYPE_F32  = 0,
        GGML_V3_TYPE_F16  = 1,
        GGML_V3_TYPE_Q4_0 = 2,
        GGML_V3_TYPE_Q4_1 = 3,
        // GGML_V3_TYPE_Q4_2 = 4, support has been removed
        // GGML_V3_TYPE_Q4_3 (5) support has been removed
        GGML_V3_TYPE_Q5_0 = 6,
        GGML_V3_TYPE_Q5_1 = 7,
        GGML_V3_TYPE_Q8_0 = 8,
        GGML_V3_TYPE_Q8_1 = 9,
        // k-quantizations
        GGML_V3_TYPE_Q2_K = 10,
        GGML_V3_TYPE_Q3_K = 11,
        GGML_V3_TYPE_Q4_K = 12,
        GGML_V3_TYPE_Q5_K = 13,
        GGML_V3_TYPE_Q6_K = 14,
        GGML_V3_TYPE_Q8_K = 15,
        GGML_V3_TYPE_IQ2_XXS = 16,
        GGML_V3_TYPE_IQ2_XS  = 17,
        GGML_V3_TYPE_I8,
        GGML_V3_TYPE_I16,
        GGML_V3_TYPE_I32,
        GGML_V3_TYPE_COUNT,
    };

    // precision
    enum ggml_v3_prec {
        GGML_V3_PREC_DEFAULT,
        GGML_V3_PREC_F32,
    };

    enum ggml_v3_backend_type {
        GGML_V3_BACKEND_CPU = 0,
        GGML_V3_BACKEND_GPU = 10,
        GGML_V3_BACKEND_GPU_SPLIT = 20,
    };

    // model file types
    enum ggml_v3_ftype {
        GGML_V3_FTYPE_UNKNOWN     = -1,
        GGML_V3_FTYPE_ALL_F32     = 0,
        GGML_V3_FTYPE_MOSTLY_F16  = 1,  // except 1d tensors
        GGML_V3_FTYPE_MOSTLY_Q4_0 = 2,  // except 1d tensors
        GGML_V3_FTYPE_MOSTLY_Q4_1 = 3,  // except 1d tensors
        GGML_V3_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16
        GGML_V3_FTYPE_MOSTLY_Q8_0 = 7,  // except 1d tensors
        GGML_V3_FTYPE_MOSTLY_Q5_0 = 8,  // except 1d tensors
        GGML_V3_FTYPE_MOSTLY_Q5_1 = 9,  // except 1d tensors
        GGML_V3_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors
        GGML_V3_FTYPE_MOSTLY_Q3_K = 11, // except 1d tensors
        GGML_V3_FTYPE_MOSTLY_Q4_K = 12, // except 1d tensors
        GGML_V3_FTYPE_MOSTLY_Q5_K = 13, // except 1d tensors
        GGML_V3_FTYPE_MOSTLY_Q6_K = 14, // except 1d tensors
        GGML_V3_FTYPE_MOSTLY_IQ2_XXS = 15, // except 1d tensors
        GGML_V3_FTYPE_MOSTLY_IQ2_XS  = 16, // except 1d tensors
    };

    // available tensor operations:
    enum ggml_v3_op {
        GGML_V3_OP_NONE = 0,

        GGML_V3_OP_DUP,
        GGML_V3_OP_ADD,
        GGML_V3_OP_ADD1,
        GGML_V3_OP_ACC,
        GGML_V3_OP_SUB,
        GGML_V3_OP_MUL,
        GGML_V3_OP_DIV,
        GGML_V3_OP_SQR,
        GGML_V3_OP_SQRT,
        GGML_V3_OP_LOG,
        GGML_V3_OP_SUM,
        GGML_V3_OP_SUM_ROWS,
        GGML_V3_OP_MEAN,
        GGML_V3_OP_ARGMAX,
        GGML_V3_OP_REPEAT,
        GGML_V3_OP_REPEAT_BACK,
        GGML_V3_OP_CONCAT,
        GGML_V3_OP_SILU_BACK,
        GGML_V3_OP_NORM, // normalize
        GGML_V3_OP_RMS_NORM,
        GGML_V3_OP_RMS_NORM_BACK,
        GGML_V3_OP_GROUP_NORM,

        GGML_V3_OP_MUL_MAT,
        GGML_V3_OP_MUL_MAT_ID,
        GGML_V3_OP_OUT_PROD,

        GGML_V3_OP_SCALE,
        GGML_V3_OP_SET,
        GGML_V3_OP_CPY,
        GGML_V3_OP_CONT,
        GGML_V3_OP_RESHAPE,
        GGML_V3_OP_VIEW,
        GGML_V3_OP_PERMUTE,
        GGML_V3_OP_TRANSPOSE,
        GGML_V3_OP_GET_ROWS,
        GGML_V3_OP_GET_ROWS_BACK,
        GGML_V3_OP_DIAG,
        GGML_V3_OP_DIAG_MASK_INF,
        GGML_V3_OP_DIAG_MASK_ZERO,
        GGML_V3_OP_SOFT_MAX,
        GGML_V3_OP_SOFT_MAX_BACK,
        GGML_V3_OP_ROPE,
        GGML_V3_OP_ROPE_BACK,
        GGML_V3_OP_ALIBI,
        GGML_V3_OP_CLAMP,
        GGML_V3_OP_CONV_TRANSPOSE_1D,
        GGML_V3_OP_IM2COL,
        GGML_V3_OP_CONV_TRANSPOSE_2D,
        GGML_V3_OP_POOL_1D,
        GGML_V3_OP_POOL_2D,
        GGML_V3_OP_UPSCALE, // nearest interpolate
        GGML_V3_OP_PAD,
        GGML_V3_OP_ARGSORT,
        GGML_V3_OP_LEAKY_RELU,

        GGML_V3_OP_FLASH_ATTN,
        GGML_V3_OP_FLASH_FF,
        GGML_V3_OP_FLASH_ATTN_BACK,
        GGML_V3_OP_WIN_PART,
        GGML_V3_OP_WIN_UNPART,
        GGML_V3_OP_GET_REL_POS,
        GGML_V3_OP_ADD_REL_POS,

        GGML_V3_OP_UNARY,

        GGML_V3_OP_MAP_UNARY,
        GGML_V3_OP_MAP_BINARY,

        GGML_V3_OP_MAP_CUSTOM1_F32,
        GGML_V3_OP_MAP_CUSTOM2_F32,
        GGML_V3_OP_MAP_CUSTOM3_F32,

        GGML_V3_OP_MAP_CUSTOM1,
        GGML_V3_OP_MAP_CUSTOM2,
        GGML_V3_OP_MAP_CUSTOM3,

        GGML_V3_OP_CROSS_ENTROPY_LOSS,
        GGML_V3_OP_CROSS_ENTROPY_LOSS_BACK,

        GGML_V3_OP_COUNT,
    };

    enum ggml_v3_unary_op {
        GGML_V3_UNARY_OP_ABS,
        GGML_V3_UNARY_OP_SGN,
        GGML_V3_UNARY_OP_NEG,
        GGML_V3_UNARY_OP_STEP,
        GGML_V3_UNARY_OP_TANH,
        GGML_V3_UNARY_OP_ELU,
        GGML_V3_UNARY_OP_RELU,
        GGML_V3_UNARY_OP_GELU,
        GGML_V3_UNARY_OP_GELU_QUICK,
        GGML_V3_UNARY_OP_SILU,

        GGML_V3_UNARY_OP_COUNT,
    };

    enum ggml_v3_object_type {
        GGML_V3_OBJECT_TENSOR,
        GGML_V3_OBJECT_GRAPH,
        GGML_V3_OBJECT_WORK_BUFFER
    };

    enum ggml_v3_log_level {
        GGML_V3_LOG_LEVEL_ERROR = 2,
        GGML_V3_LOG_LEVEL_WARN = 3,
        GGML_V3_LOG_LEVEL_INFO = 4,
        GGML_V3_LOG_LEVEL_DEBUG = 5
    };

    // ggml object
    struct ggml_v3_object {
        size_t offs;
        size_t size;

        struct ggml_v3_object * next;

        enum ggml_v3_object_type type;

        char padding[4];
    };

    static const size_t GGML_V3_OBJECT_SIZE = sizeof(struct ggml_v3_object);

    // n-dimensional tensor
    struct ggml_v3_tensor {
        enum ggml_v3_type         type;
        enum ggml_v3_backend_type backend;

        struct ggml_v3_backend_buffer * buffer;

        int64_t ne[GGML_V3_MAX_DIMS]; // number of elements
        size_t  nb[GGML_V3_MAX_DIMS]; // stride in bytes:
                                   // nb[0] = ggml_v3_type_size(type)
                                   // nb[1] = nb[0]   * (ne[0] / ggml_v3_blck_size(type)) + padding
                                   // nb[i] = nb[i-1] * ne[i-1]

        // compute data
        enum ggml_v3_op op;

        // op params - allocated as int32_t for alignment
        int32_t op_params[GGML_V3_MAX_OP_PARAMS / sizeof(int32_t)];

        bool is_param;

        struct ggml_v3_tensor * grad;
        struct ggml_v3_tensor * src[GGML_V3_MAX_SRC];

        // performance
        int     perf_runs;
        int64_t perf_cycles;
        int64_t perf_time_us;

        struct ggml_v3_tensor * view_src;
        size_t               view_offs;

        void * data;

        char name[GGML_V3_MAX_NAME];

        void * extra; // extra things e.g. for ggml-cuda.cu

        char padding[8];
    };

    static const size_t GGML_V3_TENSOR_SIZE = sizeof(struct ggml_v3_tensor);

    // the compute plan that needs to be prepared for ggml_v3_graph_compute()
    // since https://github.com/ggerganov/ggml/issues/287
    struct ggml_v3_cplan {
        size_t    work_size; // size of work buffer, calculated by `ggml_v3_graph_plan()`
        uint8_t * work_data; // work buffer, to be allocated by caller before calling to `ggml_v3_graph_compute()`

        int n_threads;

        // abort ggml_v3_graph_compute when true
        bool (*abort_callback)(void * data);
        void * abort_callback_data;
    };

    enum ggml_v3_cgraph_eval_order {
        GGML_V3_CGRAPH_EVAL_ORDER_LEFT_TO_RIGHT = 0,
        GGML_V3_CGRAPH_EVAL_ORDER_RIGHT_TO_LEFT,
        GGML_V3_CGRAPH_EVAL_ORDER_COUNT
    };

    struct ggml_v3_hash_set {
        size_t size;
        struct ggml_v3_tensor ** keys;
    };

    // computation graph
    struct ggml_v3_cgraph {
        int size;
        int n_nodes;
        int n_leafs;

        struct ggml_v3_tensor ** nodes;
        struct ggml_v3_tensor ** grads;
        struct ggml_v3_tensor ** leafs;

        struct ggml_v3_hash_set visited_hash_table;

        enum ggml_v3_cgraph_eval_order order;

        // performance
        int     perf_runs;
        int64_t perf_cycles;
        int64_t perf_time_us;
    };

    // scratch buffer
    struct ggml_v3_scratch {
        size_t offs;
        size_t size;
        void * data;
    };

    struct ggml_v3_init_params {
        // memory pool
        size_t mem_size;   // bytes
        void * mem_buffer; // if NULL, memory will be allocated internally
        bool   no_alloc;   // don't allocate memory for the tensor data
    };


    // compute types

    // NOTE: the INIT or FINALIZE pass is not scheduled unless explicitly enabled.
    // This behavior was changed since https://github.com/ggerganov/llama.cpp/pull/1995.
    enum ggml_v3_task_type {
        GGML_V3_TASK_INIT = 0,
        GGML_V3_TASK_COMPUTE,
        GGML_V3_TASK_FINALIZE,
    };

    struct ggml_v3_compute_params {
        enum ggml_v3_task_type type;

        // ith = thread index, nth = number of threads
        int ith, nth;

        // work buffer for all threads
        size_t wsize;
        void * wdata;
    };

    // misc

    GGML_V3_API void    ggml_v3_time_init(void); // call this once at the beginning of the program
    GGML_V3_API int64_t ggml_v3_time_ms(void);
    GGML_V3_API int64_t ggml_v3_time_us(void);
    GGML_V3_API int64_t ggml_v3_cycles(void);
    GGML_V3_API int64_t ggml_v3_cycles_per_ms(void);

    GGML_V3_API void    ggml_v3_print_backtrace(void);

    GGML_V3_API void    ggml_v3_numa_init(void); // call once for better performance on NUMA systems
    GGML_V3_API bool    ggml_v3_is_numa(void); // true if init detected that system has >1 NUMA node

    GGML_V3_API void    ggml_v3_print_object (const struct ggml_v3_object * obj);
    GGML_V3_API void    ggml_v3_print_objects(const struct ggml_v3_context * ctx);

    GGML_V3_API int64_t ggml_v3_nelements   (const struct ggml_v3_tensor * tensor);
    GGML_V3_API int64_t ggml_v3_nrows       (const struct ggml_v3_tensor * tensor);
    GGML_V3_API size_t  ggml_v3_nbytes      (const struct ggml_v3_tensor * tensor);
    GGML_V3_API size_t  ggml_v3_nbytes_pad  (const struct ggml_v3_tensor * tensor); // same as ggml_v3_nbytes() but padded to GGML_V3_MEM_ALIGN

    GGML_V3_API int    ggml_v3_blck_size(enum ggml_v3_type type);
    GGML_V3_API size_t ggml_v3_type_size(enum ggml_v3_type type);             // size in bytes for all elements in a block
    GGML_V3_API size_t ggml_v3_row_size (enum ggml_v3_type type, int64_t ne); // size in bytes for all elements in a row

    GGML_V3_DEPRECATED(
    GGML_V3_API double ggml_v3_type_sizef(enum ggml_v3_type type), // ggml_v3_type_size()/ggml_v3_blck_size() as float
    "use ggml_v3_row_size() instead");

    GGML_V3_API const char * ggml_v3_type_name(enum ggml_v3_type type);
    GGML_V3_API const char * ggml_v3_op_name  (enum ggml_v3_op   op);
    GGML_V3_API const char * ggml_v3_op_symbol(enum ggml_v3_op   op);

    GGML_V3_API const char * ggml_v3_unary_op_name(enum ggml_v3_unary_op op);
    GGML_V3_API const char * ggml_v3_op_desc(const struct ggml_v3_tensor * t); // unary or op name

    GGML_V3_API size_t  ggml_v3_element_size(const struct ggml_v3_tensor * tensor);

    GGML_V3_API bool    ggml_v3_is_quantized(enum ggml_v3_type type);

    // TODO: temporary until model loading of ggml examples is refactored
    GGML_V3_API enum ggml_v3_type ggml_v3_ftype_to_ggml_v3_type(enum ggml_v3_ftype ftype);

    GGML_V3_API bool ggml_v3_is_transposed(const struct ggml_v3_tensor * tensor);
    GGML_V3_API bool ggml_v3_is_contiguous(const struct ggml_v3_tensor * tensor);
    GGML_V3_API bool ggml_v3_is_permuted  (const struct ggml_v3_tensor * tensor);
    GGML_V3_API bool ggml_v3_is_scalar    (const struct ggml_v3_tensor * tensor);
    GGML_V3_API bool ggml_v3_is_vector    (const struct ggml_v3_tensor * tensor);
    GGML_V3_API bool ggml_v3_is_matrix    (const struct ggml_v3_tensor * tensor);
    GGML_V3_API bool ggml_v3_is_3d        (const struct ggml_v3_tensor * tensor);
    GGML_V3_API int  ggml_v3_n_dims       (const struct ggml_v3_tensor * tensor); // returns 1 for scalars

    GGML_V3_API bool ggml_v3_are_same_shape(const struct ggml_v3_tensor * t0, const struct ggml_v3_tensor * t1);

    // use this to compute the memory overhead of a tensor
    GGML_V3_API size_t ggml_v3_tensor_overhead(void);

    // main

    GGML_V3_API struct ggml_v3_context * ggml_v3_init(struct ggml_v3_init_params params);
    GGML_V3_API void                  ggml_v3_free(struct ggml_v3_context * ctx);

    GGML_V3_API size_t  ggml_v3_used_mem(const struct ggml_v3_context * ctx);

    GGML_V3_API size_t  ggml_v3_set_scratch (struct ggml_v3_context * ctx, struct ggml_v3_scratch scratch);
    GGML_V3_API bool    ggml_v3_get_no_alloc(struct ggml_v3_context * ctx);
    GGML_V3_API void    ggml_v3_set_no_alloc(struct ggml_v3_context * ctx, bool no_alloc);

    GGML_V3_API void *  ggml_v3_get_mem_buffer     (const struct ggml_v3_context * ctx);
    GGML_V3_API size_t  ggml_v3_get_mem_size       (const struct ggml_v3_context * ctx);
    GGML_V3_API size_t  ggml_v3_get_max_tensor_size(const struct ggml_v3_context * ctx);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_new_tensor(
            struct ggml_v3_context * ctx,
            enum   ggml_v3_type type,
            int    n_dims,
            const int64_t *ne);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_new_tensor_1d(
            struct ggml_v3_context * ctx,
            enum   ggml_v3_type type,
            int64_t ne0);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_new_tensor_2d(
            struct ggml_v3_context * ctx,
            enum   ggml_v3_type type,
            int64_t ne0,
            int64_t ne1);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_new_tensor_3d(
            struct ggml_v3_context * ctx,
            enum   ggml_v3_type type,
            int64_t ne0,
            int64_t ne1,
            int64_t ne2);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_new_tensor_4d(
            struct ggml_v3_context * ctx,
            enum   ggml_v3_type type,
            int64_t ne0,
            int64_t ne1,
            int64_t ne2,
            int64_t ne3);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_new_i32(struct ggml_v3_context * ctx, int32_t value);
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_new_f32(struct ggml_v3_context * ctx, float value);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_dup_tensor (struct ggml_v3_context * ctx, const struct ggml_v3_tensor * src);
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_view_tensor(struct ggml_v3_context * ctx, struct ggml_v3_tensor * src);

    // Context tensor enumeration and lookup
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_get_first_tensor(const struct ggml_v3_context * ctx);
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_get_next_tensor (const struct ggml_v3_context * ctx, struct ggml_v3_tensor * tensor);
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_get_tensor(struct ggml_v3_context * ctx, const char * name);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_set_zero(struct ggml_v3_tensor * tensor);
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_set_i32 (struct ggml_v3_tensor * tensor, int32_t value);
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_set_f32 (struct ggml_v3_tensor * tensor, float value);

    // Converts a flat index into coordinates
    GGML_V3_API void    ggml_v3_unravel_index(const struct ggml_v3_tensor * tensor, int64_t i, int64_t * i0, int64_t * i1, int64_t * i2, int64_t * i3);

    GGML_V3_API int32_t ggml_v3_get_i32_1d(const struct ggml_v3_tensor * tensor, int i);
    GGML_V3_API void    ggml_v3_set_i32_1d(const struct ggml_v3_tensor * tensor, int i, int32_t value);

    GGML_V3_API int32_t ggml_v3_get_i32_nd(const struct ggml_v3_tensor * tensor, int i0, int i1, int i2, int i3);
    GGML_V3_API void    ggml_v3_set_i32_nd(const struct ggml_v3_tensor * tensor, int i0, int i1, int i2, int i3, int32_t value);

    GGML_V3_API float   ggml_v3_get_f32_1d(const struct ggml_v3_tensor * tensor, int i);
    GGML_V3_API void    ggml_v3_set_f32_1d(const struct ggml_v3_tensor * tensor, int i, float value);

    GGML_V3_API float   ggml_v3_get_f32_nd(const struct ggml_v3_tensor * tensor, int i0, int i1, int i2, int i3);
    GGML_V3_API void    ggml_v3_set_f32_nd(const struct ggml_v3_tensor * tensor, int i0, int i1, int i2, int i3, float value);

    GGML_V3_API void *  ggml_v3_get_data    (const struct ggml_v3_tensor * tensor);
    GGML_V3_API float * ggml_v3_get_data_f32(const struct ggml_v3_tensor * tensor);

    GGML_V3_API enum ggml_v3_unary_op ggml_v3_get_unary_op(const struct ggml_v3_tensor * tensor);

    GGML_V3_API const char *         ggml_v3_get_name   (const struct ggml_v3_tensor * tensor);
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_set_name   (      struct ggml_v3_tensor * tensor, const char * name);
    GGML_V3_ATTRIBUTE_FORMAT(2, 3)
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_format_name(      struct ggml_v3_tensor * tensor, const char * fmt, ...);

    //
    // operations on tensors with backpropagation
    //

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_dup(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    // in-place, returns view(a)
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_dup_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_add(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_add_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_add_cast(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b,
            enum   ggml_v3_type      type);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_add1(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_add1_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b);

    // dst = a
    // view(dst, nb1, nb2, nb3, offset) += b
    // return dst
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_acc(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b,
            size_t                nb1,
            size_t                nb2,
            size_t                nb3,
            size_t                offset);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_acc_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b,
            size_t                nb1,
            size_t                nb2,
            size_t                nb3,
            size_t                offset);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_sub(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_sub_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_mul(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_mul_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_div(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_div_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_sqr(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_sqr_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_sqrt(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_sqrt_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_log(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_log_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    // return scalar
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_sum(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    // sums along rows, with input shape [a,b,c,d] return shape [1,b,c,d]
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_sum_rows(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    // mean along rows
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_mean(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    // argmax along rows
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_argmax(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    // if a is the same shape as b, and a is not parameter, return a
    // otherwise, return a new tensor: repeat(a) to fit in b
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_repeat(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b);

    // sums repetitions in a into shape of b
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_repeat_back(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b);

    // concat a and b on dim 2
    // used in stable-diffusion
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_concat(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_abs(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_abs_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_sgn(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_sgn_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_neg(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_neg_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_step(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_step_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_tanh(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_tanh_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_elu(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_elu_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_relu(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_leaky_relu(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a, float negative_slope, bool inplace);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_relu_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_gelu(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_gelu_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_gelu_quick(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_gelu_quick_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_silu(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_silu_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    // a - x
    // b - dy
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_silu_back(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b);

    // normalize along rows
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_norm(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            float                 eps);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_norm_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            float                 eps);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_rms_norm(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            float                 eps);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_rms_norm_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            float                 eps);

    // group normalize along ne0*ne1*n_groups
    // used in stable-diffusion
    // TODO: eps is hardcoded to 1e-6 for now
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_group_norm(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            int                   n_groups);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_group_norm_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            int                   n_groups);

    // a - x
    // b - dy
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_rms_norm_back(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b,
            float                 eps);

    // A: k columns, n rows => [ne03, ne02, n, k]
    // B: k columns, m rows  (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k]
    // result is n columns, m rows => [ne03 * x, ne02 * y, m, n]
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_mul_mat(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b);

    // change the precision of a matrix multiplication
    // set to GGML_V3_PREC_F32 for higher precision (useful for phi-2)
    GGML_V3_API void ggml_v3_mul_mat_set_prec(
            struct ggml_v3_tensor * a,
            enum ggml_v3_prec       prec);

    // indirect matrix multiplication
    //  ggml_v3_mul_mat_id(ctx, as, ids, id, b) ~= ggml_v3_mul_mat(as[ids[id]], b)
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_mul_mat_id(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * const as[],
            int                   n_as,
            struct ggml_v3_tensor  * ids,
            int                   id,
            struct ggml_v3_tensor  * b);

    // A: m columns, n rows,
    // B: p columns, n rows,
    // result is m columns, p rows
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_out_prod(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b);

    //
    // operations on tensors without backpropagation
    //

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_scale(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            float                 s);

    // in-place, returns view(a)
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_scale_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            float                 s);

    // b -> view(a,offset,nb1,nb2,3), return modified a
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_set(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b,
            size_t                nb1,
            size_t                nb2,
            size_t                nb3,
            size_t                offset);

    // b -> view(a,offset,nb1,nb2,3), return view(a)
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_set_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b,
            size_t                nb1,
            size_t                nb2,
            size_t                nb3,
            size_t                offset);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_set_1d(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b,
            size_t                offset);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_set_1d_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b,
            size_t                offset);

    // b -> view(a,offset,nb1,nb2,3), return modified a
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_set_2d(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b,
            size_t                nb1,
            size_t                offset);

    // b -> view(a,offset,nb1,nb2,3), return view(a)
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_set_2d_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b,
            size_t                nb1,
            size_t                offset);

    // a -> b, return view(b)
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_cpy(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b);

    // make contiguous
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_cont(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    // make contiguous, with new shape
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_cont_1d(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            int64_t               ne0);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_cont_2d(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            int64_t               ne0,
            int64_t               ne1);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_cont_3d(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            int64_t               ne0,
            int64_t               ne1,
            int64_t               ne2);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_cont_4d(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            int64_t               ne0,
            int64_t               ne1,
            int64_t               ne2,
            int64_t               ne3);

    // return view(a), b specifies the new shape
    // TODO: when we start computing gradient, make a copy instead of view
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_reshape(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b);

    // return view(a)
    // TODO: when we start computing gradient, make a copy instead of view
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_reshape_1d(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            int64_t               ne0);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_reshape_2d(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            int64_t               ne0,
            int64_t               ne1);

    // return view(a)
    // TODO: when we start computing gradient, make a copy instead of view
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_reshape_3d(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            int64_t               ne0,
            int64_t               ne1,
            int64_t               ne2);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_reshape_4d(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            int64_t               ne0,
            int64_t               ne1,
            int64_t               ne2,
            int64_t               ne3);

    // offset in bytes
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_view_1d(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            int64_t               ne0,
            size_t                offset);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_view_2d(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            int64_t               ne0,
            int64_t               ne1,
            size_t                nb1, // row stride in bytes
            size_t                offset);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_view_3d(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            int64_t               ne0,
            int64_t               ne1,
            int64_t               ne2,
            size_t                nb1, // row   stride in bytes
            size_t                nb2, // slice stride in bytes
            size_t                offset);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_view_4d(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            int64_t               ne0,
            int64_t               ne1,
            int64_t               ne2,
            int64_t               ne3,
            size_t                nb1, // row   stride in bytes
            size_t                nb2, // slice stride in bytes
            size_t                nb3,
            size_t                offset);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_permute(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            int                   axis0,
            int                   axis1,
            int                   axis2,
            int                   axis3);

    // alias for ggml_v3_permute(ctx, a, 1, 0, 2, 3)
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_transpose(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    // supports 3D: a->ne[2] == b->ne[1]
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_get_rows(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_get_rows_back(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b,
            struct ggml_v3_tensor  * c);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_diag(
        struct ggml_v3_context     * ctx,
        struct ggml_v3_tensor      * a);

    // set elements above the diagonal to -INF
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_diag_mask_inf(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            int                   n_past);

    // in-place, returns view(a)
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_diag_mask_inf_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            int                   n_past);

    // set elements above the diagonal to 0
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_diag_mask_zero(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            int                   n_past);

    // in-place, returns view(a)
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_diag_mask_zero_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            int                   n_past);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_soft_max(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    // in-place, returns view(a)
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_soft_max_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a);

    // fused soft_max(a*scale + mask)
    // mask is optional
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_soft_max_ext(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * mask,
            float                 scale);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_soft_max_back(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b);

    // in-place, returns view(a)
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_soft_max_back_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b);

    // rotary position embedding
    // if mode & 1 == 1, skip n_past elements (DEPRECATED)
    // if mode & 2 == 1, GPT-NeoX style
    // if mode & 4 == 1, ChatGLM style
    //
    // b is an int32 vector with size a->ne[2], it contains the positions
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_rope(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b,
            int                   n_dims,
            int                   mode,
            int                   n_ctx);

    // in-place, returns view(a)
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_rope_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b,
            int                   n_dims,
            int                   mode,
            int                   n_ctx);

    // custom RoPE
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_rope_custom(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b,
            int                   n_dims,
            int                   mode,
            int                   n_ctx,
            int                   n_orig_ctx,
            float                 freq_base,
            float                 freq_scale,
            float                 ext_factor,
            float                 attn_factor,
            float                 beta_fast,
            float                 beta_slow);

    // in-place, returns view(a)
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_rope_custom_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b,
            int                   n_dims,
            int                   mode,
            int                   n_ctx,
            int                   n_orig_ctx,
            float                 freq_base,
            float                 freq_scale,
            float                 ext_factor,
            float                 attn_factor,
            float                 beta_fast,
            float                 beta_slow);

    // compute correction dims for YaRN RoPE scaling
    void ggml_v3_rope_yarn_corr_dims(
        int n_dims, int n_orig_ctx, float freq_base, float beta_fast, float beta_slow, float dims[2]);

    // xPos RoPE, in-place, returns view(a)
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_rope_xpos_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b,
            int                   n_dims,
            float                 base,
            bool                  down);

    // rotary position embedding backward, i.e compute dx from dy
    // a - dy
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_rope_back(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b,
            int                   n_dims,
            int                   mode,
            int                   n_ctx,
            int                   n_orig_ctx,
            float                 freq_base,
            float                 freq_scale,
            float                 ext_factor,
            float                 attn_factor,
            float                 beta_fast,
            float                 beta_slow,
            float                 xpos_base,
            bool                  xpos_down);

    // alibi position embedding
    // in-place, returns view(a)
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_alibi(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            int                   n_past,
            int                   n_head,
            float                 bias_max);

    // clamp
    // in-place, returns view(a)
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_clamp(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            float                 min,
            float                 max);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_im2col(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b,
            int                  s0,
            int                  s1,
            int                  p0,
            int                  p1,
            int                  d0,
            int                  d1,
            bool                 is_2D);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_conv_1d(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b,
            int                   s0,  // stride
            int                   p0,  // padding
            int                   d0); // dilation

    // conv_1d with padding = half
    // alias for ggml_v3_conv_1d(a, b, s, a->ne[0]/2, d)
    GGML_V3_API struct ggml_v3_tensor* ggml_v3_conv_1d_ph(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b,
            int                   s,
            int                   d);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_conv_transpose_1d(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b,
            int                   s0,
            int                   p0,
            int                   d0);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_conv_2d(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b,
            int                   s0,
            int                   s1,
            int                   p0,
            int                   p1,
            int                   d0,
            int                   d1);


    // kernel size is a->ne[0] x a->ne[1]
    // stride is equal to kernel size
    // padding is zero
    // example:
    // a:     16   16    3  768
    // b:   1024 1024    3    1
    // res:   64   64  768    1
    // used in sam
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_conv_2d_sk_p0(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b);

    // kernel size is a->ne[0] x a->ne[1]
    // stride is 1
    // padding is half
    // example:
    // a:      3    3    256  256
    // b:     64   64    256    1
    // res:   64   64    256    1
    // used in sam
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_conv_2d_s1_ph(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_conv_transpose_2d_p0(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b,
            int                   stride);

    enum ggml_v3_op_pool {
        GGML_V3_OP_POOL_MAX,
        GGML_V3_OP_POOL_AVG,
        GGML_V3_OP_POOL_COUNT,
    };

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_pool_1d(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            enum ggml_v3_op_pool     op,
            int                   k0, // kernel size
            int                   s0, // stride
            int                   p0); // padding

    // the result will have 2*p0 padding for the first dimension
    // and 2*p1 padding for the second dimension
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_pool_2d(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            enum ggml_v3_op_pool     op,
            int                   k0,
            int                   k1,
            int                   s0,
            int                   s1,
            float                 p0,
            float                 p1);

    // nearest interpolate
    // used in stable-diffusion
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_upscale(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            int                   scale_factor);

    // pad each dimension with zeros: [x, ..., x] -> [x, ..., x, 0, ..., 0]
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_pad(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            int                  p0,
            int                  p1,
            int                  p2,
            int                  p3);

    // sort rows
    enum ggml_v3_sort_order {
        GGML_V3_SORT_ASC,
        GGML_V3_SORT_DESC,
    };

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_argsort(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            enum ggml_v3_sort_order  order);

    // top k elements per row
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_top_k(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            int                   k);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_flash_attn(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * q,
            struct ggml_v3_tensor  * k,
            struct ggml_v3_tensor  * v,
            bool                  masked);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_flash_attn_back(
           struct ggml_v3_context * ctx,
           struct ggml_v3_tensor  * q,
           struct ggml_v3_tensor  * k,
           struct ggml_v3_tensor  * v,
           struct ggml_v3_tensor  * d,
           bool                  masked);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_flash_ff(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * b0,
            struct ggml_v3_tensor  * b1,
            struct ggml_v3_tensor  * c0,
            struct ggml_v3_tensor  * c1);

    // partition into non-overlapping windows with padding if needed
    // example:
    // a:   768   64   64    1
    // w:    14
    // res: 768   14   14    25
    // used in sam
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_win_part(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            int                   w);

    // reverse of ggml_v3_win_part
    // used in sam
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_win_unpart(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            int                   w0,
            int                   h0,
            int                   w);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_unary(
            struct ggml_v3_context * ctx,
             struct ggml_v3_tensor * a,
             enum ggml_v3_unary_op op);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_unary_inplace(
        struct ggml_v3_context * ctx,
        struct ggml_v3_tensor  * a,
        enum ggml_v3_unary_op op);

    // used in sam
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_get_rel_pos(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            int                   qh,
            int                   kh);

    // used in sam
    GGML_V3_API struct ggml_v3_tensor * ggml_v3_add_rel_pos(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * pw,
            struct ggml_v3_tensor  * ph);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_add_rel_pos_inplace(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * a,
            struct ggml_v3_tensor  * pw,
            struct ggml_v3_tensor  * ph);

    // custom operators

    typedef void (*ggml_v3_unary_op_f32_t) (const int, float *, const float *);
    typedef void (*ggml_v3_binary_op_f32_t)(const int, float *, const float *, const float *);

    typedef void (*ggml_v3_custom1_op_f32_t)(struct ggml_v3_tensor *, const struct ggml_v3_tensor *);
    typedef void (*ggml_v3_custom2_op_f32_t)(struct ggml_v3_tensor *, const struct ggml_v3_tensor *, const struct ggml_v3_tensor *);
    typedef void (*ggml_v3_custom3_op_f32_t)(struct ggml_v3_tensor *, const struct ggml_v3_tensor *, const struct ggml_v3_tensor *, const struct ggml_v3_tensor *);

    GGML_V3_DEPRECATED(GGML_V3_API struct ggml_v3_tensor * ggml_v3_map_unary_f32(
            struct ggml_v3_context        * ctx,
            struct ggml_v3_tensor         * a,
                   ggml_v3_unary_op_f32_t   fun),
        "use ggml_v3_map_custom1 instead");

    GGML_V3_DEPRECATED(GGML_V3_API struct ggml_v3_tensor * ggml_v3_map_unary_inplace_f32(
            struct ggml_v3_context        * ctx,
            struct ggml_v3_tensor         * a,
                   ggml_v3_unary_op_f32_t   fun),
        "use ggml_v3_map_custom1_inplace instead");

    GGML_V3_DEPRECATED(GGML_V3_API struct ggml_v3_tensor * ggml_v3_map_binary_f32(
            struct ggml_v3_context         * ctx,
            struct ggml_v3_tensor          * a,
            struct ggml_v3_tensor          * b,
                   ggml_v3_binary_op_f32_t   fun),
        "use ggml_v3_map_custom2 instead");

    GGML_V3_DEPRECATED(GGML_V3_API struct ggml_v3_tensor * ggml_v3_map_binary_inplace_f32(
            struct ggml_v3_context         * ctx,
            struct ggml_v3_tensor          * a,
            struct ggml_v3_tensor          * b,
                   ggml_v3_binary_op_f32_t   fun),
        "use ggml_v3_map_custom2_inplace instead");

    GGML_V3_DEPRECATED(GGML_V3_API struct ggml_v3_tensor * ggml_v3_map_custom1_f32(
            struct ggml_v3_context          * ctx,
            struct ggml_v3_tensor           * a,
                   ggml_v3_custom1_op_f32_t   fun),
        "use ggml_v3_map_custom1 instead");

    GGML_V3_DEPRECATED(GGML_V3_API struct ggml_v3_tensor * ggml_v3_map_custom1_inplace_f32(
            struct ggml_v3_context          * ctx,
            struct ggml_v3_tensor           * a,
                   ggml_v3_custom1_op_f32_t   fun),
        "use ggml_v3_map_custom1_inplace instead");

    GGML_V3_DEPRECATED(GGML_V3_API struct ggml_v3_tensor * ggml_v3_map_custom2_f32(
            struct ggml_v3_context          * ctx,
            struct ggml_v3_tensor           * a,
            struct ggml_v3_tensor           * b,
                   ggml_v3_custom2_op_f32_t   fun),
        "use ggml_v3_map_custom2 instead");

    GGML_V3_DEPRECATED(GGML_V3_API struct ggml_v3_tensor * ggml_v3_map_custom2_inplace_f32(
            struct ggml_v3_context          * ctx,
            struct ggml_v3_tensor           * a,
            struct ggml_v3_tensor           * b,
                   ggml_v3_custom2_op_f32_t   fun),
        "use ggml_v3_map_custom2_inplace instead");

    GGML_V3_DEPRECATED(GGML_V3_API struct ggml_v3_tensor * ggml_v3_map_custom3_f32(
            struct ggml_v3_context          * ctx,
            struct ggml_v3_tensor           * a,
            struct ggml_v3_tensor           * b,
            struct ggml_v3_tensor           * c,
                   ggml_v3_custom3_op_f32_t   fun),
        "use ggml_v3_map_custom3 instead");

    GGML_V3_DEPRECATED(GGML_V3_API struct ggml_v3_tensor * ggml_v3_map_custom3_inplace_f32(
            struct ggml_v3_context          * ctx,
            struct ggml_v3_tensor           * a,
            struct ggml_v3_tensor           * b,
            struct ggml_v3_tensor           * c,
                   ggml_v3_custom3_op_f32_t   fun),
        "use ggml_v3_map_custom3_inplace instead");

    // custom operators v2

    typedef void (*ggml_v3_custom1_op_t)(struct ggml_v3_tensor * dst , const struct ggml_v3_tensor * a, int ith, int nth, void * userdata);
    typedef void (*ggml_v3_custom2_op_t)(struct ggml_v3_tensor * dst , const struct ggml_v3_tensor * a, const struct ggml_v3_tensor * b, int ith, int nth, void * userdata);
    typedef void (*ggml_v3_custom3_op_t)(struct ggml_v3_tensor * dst , const struct ggml_v3_tensor * a, const struct ggml_v3_tensor * b, const struct ggml_v3_tensor * c, int ith, int nth, void * userdata);

    #define GGML_V3_N_TASKS_MAX -1

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_map_custom1(
            struct ggml_v3_context   * ctx,
            struct ggml_v3_tensor    * a,
            ggml_v3_custom1_op_t       fun,
            int                     n_tasks,
            void                  * userdata);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_map_custom1_inplace(
            struct ggml_v3_context   * ctx,
            struct ggml_v3_tensor    * a,
            ggml_v3_custom1_op_t       fun,
            int                     n_tasks,
            void                  * userdata);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_map_custom2(
            struct ggml_v3_context   * ctx,
            struct ggml_v3_tensor    * a,
            struct ggml_v3_tensor    * b,
            ggml_v3_custom2_op_t       fun,
            int                     n_tasks,
            void                  * userdata);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_map_custom2_inplace(
            struct ggml_v3_context   * ctx,
            struct ggml_v3_tensor    * a,
            struct ggml_v3_tensor    * b,
            ggml_v3_custom2_op_t       fun,
            int                     n_tasks,
            void                  * userdata);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_map_custom3(
            struct ggml_v3_context   * ctx,
            struct ggml_v3_tensor    * a,
            struct ggml_v3_tensor    * b,
            struct ggml_v3_tensor    * c,
            ggml_v3_custom3_op_t       fun,
            int                     n_tasks,
            void                  * userdata);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_map_custom3_inplace(
            struct ggml_v3_context   * ctx,
            struct ggml_v3_tensor    * a,
            struct ggml_v3_tensor    * b,
            struct ggml_v3_tensor    * c,
            ggml_v3_custom3_op_t       fun,
            int                     n_tasks,
            void                  * userdata);

    // loss function

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_cross_entropy_loss(
            struct ggml_v3_context         * ctx,
            struct ggml_v3_tensor          * a,
            struct ggml_v3_tensor          * b);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_cross_entropy_loss_back(
            struct ggml_v3_context         * ctx,
            struct ggml_v3_tensor          * a,
            struct ggml_v3_tensor          * b,
            struct ggml_v3_tensor          * c);

    //
    // automatic differentiation
    //

    GGML_V3_API void ggml_v3_set_param(
            struct ggml_v3_context * ctx,
            struct ggml_v3_tensor  * tensor);


    GGML_V3_API void ggml_v3_build_forward_expand (struct ggml_v3_cgraph * cgraph, struct ggml_v3_tensor * tensor);
    GGML_V3_API void ggml_v3_build_backward_expand(struct ggml_v3_context * ctx, struct ggml_v3_cgraph * gf, struct ggml_v3_cgraph * gb, bool keep);

    // graph allocation in a context
    GGML_V3_API struct ggml_v3_cgraph * ggml_v3_new_graph         (struct ggml_v3_context * ctx); // size = GGML_V3_DEFAULT_GRAPH_SIZE, grads = false
    GGML_V3_API struct ggml_v3_cgraph * ggml_v3_new_graph_custom  (struct ggml_v3_context * ctx, size_t size, bool grads);
    GGML_V3_API struct ggml_v3_cgraph * ggml_v3_graph_dup         (struct ggml_v3_context * ctx, struct ggml_v3_cgraph * cgraph);
    GGML_V3_API struct ggml_v3_cgraph   ggml_v3_graph_view        (struct ggml_v3_cgraph * cgraph, int i0, int i1);
    GGML_V3_API void                 ggml_v3_graph_cpy         (struct ggml_v3_cgraph * src, struct ggml_v3_cgraph * dst);
    GGML_V3_API void                 ggml_v3_graph_reset       (struct ggml_v3_cgraph * cgraph);  // zero grads
    GGML_V3_API void                 ggml_v3_graph_clear       (struct ggml_v3_cgraph * cgraph);

    GGML_V3_API size_t ggml_v3_graph_overhead(void);
    GGML_V3_API size_t ggml_v3_graph_overhead_custom(size_t size, bool grads);

    // ggml_v3_graph_plan() has to be called before ggml_v3_graph_compute()
    // when plan.work_size > 0, caller must allocate memory for plan.work_data
    GGML_V3_API struct ggml_v3_cplan ggml_v3_graph_plan   (struct ggml_v3_cgraph * cgraph, int n_threads /*= GGML_V3_DEFAULT_N_THREADS*/);
    GGML_V3_API int               ggml_v3_graph_compute(struct ggml_v3_cgraph * cgraph, struct ggml_v3_cplan * cplan);

    // same as ggml_v3_graph_compute() but the work data is allocated as a part of the context
    // note: the drawback of this API is that you must have ensured that the context has enough memory for the work data
    GGML_V3_API void ggml_v3_graph_compute_with_ctx(struct ggml_v3_context * ctx, struct ggml_v3_cgraph * cgraph, int n_threads);

    GGML_V3_API struct ggml_v3_tensor * ggml_v3_graph_get_tensor(struct ggml_v3_cgraph * cgraph, const char * name);

    GGML_V3_API void                 ggml_v3_graph_export(const struct ggml_v3_cgraph * cgraph, const char * fname);
    GGML_V3_API struct ggml_v3_cgraph * ggml_v3_graph_import(const char * fname, struct ggml_v3_context ** ctx_data, struct ggml_v3_context ** ctx_eval);

    // print info and performance information for the graph
    GGML_V3_API void ggml_v3_graph_print(const struct ggml_v3_cgraph * cgraph);

    // dump the graph into a file using the dot format
    GGML_V3_API void ggml_v3_graph_dump_dot(const struct ggml_v3_cgraph * gb, const struct ggml_v3_cgraph * gf, const char * filename);

    // build gradient checkpointing backward graph gb for gf using provided checkpoints
    // gb_tmp will contain original backward graph with rewritten backward process nodes,
    // but without the second forward pass nodes.
    GGML_V3_API void ggml_v3_build_backward_gradient_checkpointing(
            struct ggml_v3_context   * ctx,
            struct ggml_v3_cgraph    * gf,
            struct ggml_v3_cgraph    * gb,
            struct ggml_v3_cgraph    * gb_tmp,
            struct ggml_v3_tensor  * * checkpoints,
            int                     n_checkpoints);
    //
    // optimization
    //

    // optimization methods
    enum ggml_v3_opt_type {
        GGML_V3_OPT_ADAM,
        GGML_V3_OPT_LBFGS,
    };

    // linesearch methods
    enum ggml_v3_linesearch {
        GGML_V3_LINESEARCH_DEFAULT = 1,

        GGML_V3_LINESEARCH_BACKTRACKING_ARMIJO       = 0,
        GGML_V3_LINESEARCH_BACKTRACKING_WOLFE        = 1,
        GGML_V3_LINESEARCH_BACKTRACKING_STRONG_WOLFE = 2,
    };

    // optimization return values
    enum ggml_v3_opt_result {
        GGML_V3_OPT_OK = 0,
        GGML_V3_OPT_DID_NOT_CONVERGE,
        GGML_V3_OPT_NO_CONTEXT,
        GGML_V3_OPT_INVALID_WOLFE,
        GGML_V3_OPT_FAIL,
        GGML_V3_OPT_CANCEL,

        GGML_V3_LINESEARCH_FAIL = -128,
        GGML_V3_LINESEARCH_MINIMUM_STEP,
        GGML_V3_LINESEARCH_MAXIMUM_STEP,
        GGML_V3_LINESEARCH_MAXIMUM_ITERATIONS,
        GGML_V3_LINESEARCH_INVALID_PARAMETERS,
    };

    typedef void (*ggml_v3_opt_callback)(void * data, int accum_step, float * sched, bool * cancel);
    typedef void (*ggml_v3_log_callback)(enum ggml_v3_log_level level, const char * text, void * user_data);

    // optimization parameters
    //
    //   see ggml.c (ggml_v3_opt_default_params) for default values
    //
    struct ggml_v3_opt_params {
        enum ggml_v3_opt_type type;

        size_t graph_size;

        int n_threads;

        // delta-based convergence test
        //
        //   if past == 0 - disabled
        //   if past > 0:
        //     stop if |f(x) - f(x_past)| < delta * max(1, |f(x)|)
        //
        int past;
        float delta;

        // maximum number of iterations without improvement
        //
        //   if 0 - disabled
        //   if > 0:
        //     assume convergence if no cost improvement in this number of iterations
        //
        int max_no_improvement;

        bool print_forward_graph;
        bool print_backward_graph;

        int n_gradient_accumulation;

        // ADAM parameters
        struct {
            int n_iter;

            float sched; // schedule multiplier (fixed, decay or warmup)
            float decay; // weight decay for AdamW, use 0.0f to disable
            int   decay_min_ndim; // minimum number of tensor dimension to apply weight decay
            float alpha; // learning rate
            float beta1;
            float beta2;
            float eps;   // epsilon for numerical stability
            float eps_f; // epsilon for convergence test
            float eps_g; // epsilon for convergence test
            float gclip; // gradient clipping
        } adam;

        // LBFGS parameters
        struct {
            int m; // number of corrections to approximate the inv. Hessian
            int n_iter;
            int max_linesearch;

            float eps;      // convergence tolerance
            float ftol;     // line search tolerance
            float wolfe;
            float min_step;
            float max_step;

            enum ggml_v3_linesearch linesearch;
        } lbfgs;
    };

    struct ggml_v3_opt_context {
        struct ggml_v3_context * ctx;
        struct ggml_v3_opt_params params;

        int iter;
        int64_t nx; // number of parameter elements

        bool just_initialized;

        float loss_before;
        float loss_after;

        struct {
            struct ggml_v3_tensor * g;  // current gradient
            struct ggml_v3_tensor * m;  // first moment
            struct ggml_v3_tensor * v;  // second moment
            struct ggml_v3_tensor * pf; // past function values
            float fx_best;
            float fx_prev;
            int n_no_improvement;
        } adam;

        struct {
            struct ggml_v3_tensor * x;    // current parameters
            struct ggml_v3_tensor * xp;   // previous parameters
            struct ggml_v3_tensor * g;    // current gradient
            struct ggml_v3_tensor * gp;   // previous gradient
            struct ggml_v3_tensor * d;    // search direction
            struct ggml_v3_tensor * pf;   // past function values
            struct ggml_v3_tensor * lmal; // the L-BFGS memory alpha
            struct ggml_v3_tensor * lmys; // the L-BFGS memory ys
            struct ggml_v3_tensor * lms;  // the L-BFGS memory s
            struct ggml_v3_tensor * lmy;  // the L-BFGS memory y
            float fx_best;
            float step;
            int j;
            int k;
            int end;
            int n_no_improvement;
        } lbfgs;
    };

    GGML_V3_API struct ggml_v3_opt_params ggml_v3_opt_default_params(enum ggml_v3_opt_type type);

    // optimize the function defined by the tensor f
    GGML_V3_API enum ggml_v3_opt_result ggml_v3_opt(
            struct ggml_v3_context * ctx,
            struct ggml_v3_opt_params params,
            struct ggml_v3_tensor * f);

    // initialize optimizer context
    GGML_V3_API void ggml_v3_opt_init(
            struct ggml_v3_context     * ctx,
            struct ggml_v3_opt_context * opt,
            struct ggml_v3_opt_params    params,
            int64_t                   nx);

    // continue optimizing the function defined by the tensor f
    GGML_V3_API enum ggml_v3_opt_result ggml_v3_opt_resume(
            struct ggml_v3_context * ctx,
            struct ggml_v3_opt_context * opt,
            struct ggml_v3_tensor * f);

    // continue optimizing the function defined by the tensor f
    GGML_V3_API enum ggml_v3_opt_result ggml_v3_opt_resume_g(
            struct ggml_v3_context * ctx,
            struct ggml_v3_opt_context * opt,
            struct ggml_v3_tensor * f,
            struct ggml_v3_cgraph * gf,
            struct ggml_v3_cgraph * gb,
            ggml_v3_opt_callback callback,
            void * callback_data);

    //
    // quantization
    //

    // TODO: these would probably get removed in favor of the more general ggml_v3_quantize_chunk
    GGML_V3_API size_t ggml_v3_quantize_q4_0(const float * src, void * dst, int n, int k, int64_t * hist);
    GGML_V3_API size_t ggml_v3_quantize_q4_1(const float * src, void * dst, int n, int k, int64_t * hist);
    GGML_V3_API size_t ggml_v3_quantize_q5_0(const float * src, void * dst, int n, int k, int64_t * hist);
    GGML_V3_API size_t ggml_v3_quantize_q5_1(const float * src, void * dst, int n, int k, int64_t * hist);
    GGML_V3_API size_t ggml_v3_quantize_q8_0(const float * src, void * dst, int n, int k, int64_t * hist);

    GGML_V3_API size_t ggml_v3_quantize_q2_K(const float * src, void * dst, int n, int k, int64_t * hist);
    GGML_V3_API size_t ggml_v3_quantize_q3_K(const float * src, void * dst, int n, int k, int64_t * hist);
    GGML_V3_API size_t ggml_v3_quantize_q4_K(const float * src, void * dst, int n, int k, int64_t * hist);
    GGML_V3_API size_t ggml_v3_quantize_q5_K(const float * src, void * dst, int n, int k, int64_t * hist);
    GGML_V3_API size_t ggml_v3_quantize_q6_K(const float * src, void * dst, int n, int k, int64_t * hist);
    GGML_V3_API size_t ggml_v3_quantize_iq2_xxs(const float * src, void * dst, int n, int k, int64_t * hist);
    GGML_V3_API size_t ggml_v3_quantize_iq2_xs (const float * src, void * dst, int n, int k, int64_t * hist);

    GGML_V3_API size_t ggml_v3_quantize_chunk(enum ggml_v3_type type, const float * src, void * dst, int start, int n, int64_t * hist);

    //
    // Importance matrix
    //
    typedef void(*ggml_v3_collect_imatrix_t)(const struct ggml_v3_tensor * src0, const struct ggml_v3_tensor * src1);
    GGML_V3_API void ggml_v3_set_imatrix_collection(ggml_v3_collect_imatrix_t imatrix_collect);

    //
    // gguf
    //

    enum gguf_v3_type {
        GGUF_V3_TYPE_UINT8   = 0,
        GGUF_V3_TYPE_INT8    = 1,
        GGUF_V3_TYPE_UINT16  = 2,
        GGUF_V3_TYPE_INT16   = 3,
        GGUF_V3_TYPE_UINT32  = 4,
        GGUF_V3_TYPE_INT32   = 5,
        GGUF_V3_TYPE_FLOAT32 = 6,
        GGUF_V3_TYPE_BOOL    = 7,
        GGUF_V3_TYPE_STRING  = 8,
        GGUF_V3_TYPE_ARRAY   = 9,
        GGUF_V3_TYPE_UINT64  = 10,
        GGUF_V3_TYPE_INT64   = 11,
        GGUF_V3_TYPE_FLOAT64 = 12,
        GGUF_V3_TYPE_COUNT,       // marks the end of the enum
    };

    struct gguf_v3_context;

    struct gguf_v3_init_params {
        bool no_alloc;

        // if not NULL, create a ggml_v3_context and allocate the tensor data in it
        struct ggml_v3_context ** ctx;
    };

    GGML_V3_API struct gguf_v3_context * gguf_v3_init_empty(void);
    GGML_V3_API struct gguf_v3_context * gguf_v3_init_from_file(const char * fname, struct gguf_v3_init_params params);
    //GGML_V3_API struct gguf_v3_context * gguf_v3_init_from_buffer(..);

    GGML_V3_API void gguf_v3_free(struct gguf_v3_context * ctx);

    GGML_V3_API const char * gguf_v3_type_name(enum gguf_v3_type type);

    GGML_V3_API int    gguf_v3_get_version    (const struct gguf_v3_context * ctx);
    GGML_V3_API size_t gguf_v3_get_alignment  (const struct gguf_v3_context * ctx);
    GGML_V3_API size_t gguf_v3_get_data_offset(const struct gguf_v3_context * ctx);
    GGML_V3_API void * gguf_v3_get_data       (const struct gguf_v3_context * ctx);

    GGML_V3_API int          gguf_v3_get_n_kv(const struct gguf_v3_context * ctx);
    GGML_V3_API int          gguf_v3_find_key(const struct gguf_v3_context * ctx, const char * key);
    GGML_V3_API const char * gguf_v3_get_key (const struct gguf_v3_context * ctx, int key_id);

    GGML_V3_API enum gguf_v3_type gguf_v3_get_kv_type (const struct gguf_v3_context * ctx, int key_id);
    GGML_V3_API enum gguf_v3_type gguf_v3_get_arr_type(const struct gguf_v3_context * ctx, int key_id);

    // will abort if the wrong type is used for the key
    GGML_V3_API uint8_t      gguf_v3_get_val_u8  (const struct gguf_v3_context * ctx, int key_id);
    GGML_V3_API int8_t       gguf_v3_get_val_i8  (const struct gguf_v3_context * ctx, int key_id);
    GGML_V3_API uint16_t     gguf_v3_get_val_u16 (const struct gguf_v3_context * ctx, int key_id);
    GGML_V3_API int16_t      gguf_v3_get_val_i16 (const struct gguf_v3_context * ctx, int key_id);
    GGML_V3_API uint32_t     gguf_v3_get_val_u32 (const struct gguf_v3_context * ctx, int key_id);
    GGML_V3_API int32_t      gguf_v3_get_val_i32 (const struct gguf_v3_context * ctx, int key_id);
    GGML_V3_API float        gguf_v3_get_val_f32 (const struct gguf_v3_context * ctx, int key_id);
    GGML_V3_API uint64_t     gguf_v3_get_val_u64 (const struct gguf_v3_context * ctx, int key_id);
    GGML_V3_API int64_t      gguf_v3_get_val_i64 (const struct gguf_v3_context * ctx, int key_id);
    GGML_V3_API double       gguf_v3_get_val_f64 (const struct gguf_v3_context * ctx, int key_id);
    GGML_V3_API bool         gguf_v3_get_val_bool(const struct gguf_v3_context * ctx, int key_id);
    GGML_V3_API const char * gguf_v3_get_val_str (const struct gguf_v3_context * ctx, int key_id);
    GGML_V3_API const void * gguf_v3_get_val_data(const struct gguf_v3_context * ctx, int key_id);
    GGML_V3_API int          gguf_v3_get_arr_n   (const struct gguf_v3_context * ctx, int key_id);
    GGML_V3_API const void * gguf_v3_get_arr_data(const struct gguf_v3_context * ctx, int key_id);
    GGML_V3_API const char * gguf_v3_get_arr_str (const struct gguf_v3_context * ctx, int key_id, int i);

    GGML_V3_API int            gguf_v3_get_n_tensors    (const struct gguf_v3_context * ctx);
    GGML_V3_API int            gguf_v3_find_tensor      (const struct gguf_v3_context * ctx, const char * name);
    GGML_V3_API size_t         gguf_v3_get_tensor_offset(const struct gguf_v3_context * ctx, int i);
    GGML_V3_API char *         gguf_v3_get_tensor_name  (const struct gguf_v3_context * ctx, int i);
    GGML_V3_API enum ggml_v3_type gguf_v3_get_tensor_type  (const struct gguf_v3_context * ctx, int i);

    // overrides existing values or adds a new one
    GGML_V3_API void gguf_v3_set_val_u8  (struct gguf_v3_context * ctx, const char * key, uint8_t  val);
    GGML_V3_API void gguf_v3_set_val_i8  (struct gguf_v3_context * ctx, const char * key, int8_t   val);
    GGML_V3_API void gguf_v3_set_val_u16 (struct gguf_v3_context * ctx, const char * key, uint16_t val);
    GGML_V3_API void gguf_v3_set_val_i16 (struct gguf_v3_context * ctx, const char * key, int16_t  val);
    GGML_V3_API void gguf_v3_set_val_u32 (struct gguf_v3_context * ctx, const char * key, uint32_t val);
    GGML_V3_API void gguf_v3_set_val_i32 (struct gguf_v3_context * ctx, const char * key, int32_t  val);
    GGML_V3_API void gguf_v3_set_val_f32 (struct gguf_v3_context * ctx, const char * key, float    val);
    GGML_V3_API void gguf_v3_set_val_u64 (struct gguf_v3_context * ctx, const char * key, uint64_t val);
    GGML_V3_API void gguf_v3_set_val_i64 (struct gguf_v3_context * ctx, const char * key, int64_t  val);
    GGML_V3_API void gguf_v3_set_val_f64 (struct gguf_v3_context * ctx, const char * key, double   val);
    GGML_V3_API void gguf_v3_set_val_bool(struct gguf_v3_context * ctx, const char * key, bool     val);
    GGML_V3_API void gguf_v3_set_val_str (struct gguf_v3_context * ctx, const char * key, const char * val);
    GGML_V3_API void gguf_v3_set_arr_data(struct gguf_v3_context * ctx, const char * key, enum gguf_v3_type type, const void * data, int n);
    GGML_V3_API void gguf_v3_set_arr_str (struct gguf_v3_context * ctx, const char * key, const char ** data, int n);

    // set or add KV pairs from another context
    GGML_V3_API void gguf_v3_set_kv(struct gguf_v3_context * ctx, struct gguf_v3_context * src);

    // manage tensor info
    GGML_V3_API void gguf_v3_add_tensor(struct gguf_v3_context * ctx, const struct ggml_v3_tensor * tensor);
    GGML_V3_API void gguf_v3_set_tensor_type(struct gguf_v3_context * ctx, const char * name, enum ggml_v3_type type);
    GGML_V3_API void gguf_v3_set_tensor_data(struct gguf_v3_context * ctx, const char * name, const void * data, size_t size);

    // writing gguf files can be done in 2 ways:
    //
    // - write the entire gguf_v3_context to a binary file in a single pass:
    //
    //   gguf_v3_write_to_file(ctx, fname);
    //
    // - first prepare a file with a placeholder for the meta data, write the tensor data, then write the meta data:
    //
    //   FILE * f = fopen(fname, "wb");
    //   fseek(f, gguf_v3_get_meta_size(ctx), SEEK_SET);
    //   fwrite(f, ...);
    //   void * data = gguf_v3_meta_get_meta_data(ctx);
    //   fseek(f, 0, SEEK_SET);
    //   fwrite(f, data, gguf_v3_get_meta_size(ctx));
    //   free(data);
    //   fclose(f);
    //

    // write the entire context to a binary file
    GGML_V3_API void gguf_v3_write_to_file(const struct gguf_v3_context * ctx, const char * fname, bool only_meta);

    // get the size in bytes of the meta data (header, kv pairs, tensor info) including padding
    GGML_V3_API size_t gguf_v3_get_meta_size(const struct gguf_v3_context * ctx);
    GGML_V3_API void   gguf_v3_get_meta_data(const struct gguf_v3_context * ctx, void * data);

    //
    // system info
    //

    GGML_V3_API int ggml_v3_cpu_has_avx        (void);
    GGML_V3_API int ggml_v3_cpu_has_avx_vnni   (void);
    GGML_V3_API int ggml_v3_cpu_has_avx2       (void);
    GGML_V3_API int ggml_v3_cpu_has_avx512     (void);
    GGML_V3_API int ggml_v3_cpu_has_avx512_vbmi(void);
    GGML_V3_API int ggml_v3_cpu_has_avx512_vnni(void);
    GGML_V3_API int ggml_v3_cpu_has_fma        (void);
    GGML_V3_API int ggml_v3_cpu_has_neon       (void);
    GGML_V3_API int ggml_v3_cpu_has_arm_fma    (void);
    GGML_V3_API int ggml_v3_cpu_has_metal      (void);
    GGML_V3_API int ggml_v3_cpu_has_f16c       (void);
    GGML_V3_API int ggml_v3_cpu_has_fp16_va    (void);
    GGML_V3_API int ggml_v3_cpu_has_wasm_simd  (void);
    GGML_V3_API int ggml_v3_cpu_has_blas       (void);
    GGML_V3_API int ggml_v3_cpu_has_cublas     (void);
    GGML_V3_API int ggml_v3_cpu_has_clblast    (void);
    GGML_V3_API int ggml_v3_cpu_has_gpublas    (void);
    GGML_V3_API int ggml_v3_cpu_has_sse3       (void);
    GGML_V3_API int ggml_v3_cpu_has_ssse3      (void);
    GGML_V3_API int ggml_v3_cpu_has_vsx        (void);

    //
    // Internal types and functions exposed for tests and benchmarks
    //

#ifdef  __cplusplus
// restrict not standard in C++
#define GGML_V3_RESTRICT
#else
#define GGML_V3_RESTRICT restrict
#endif
    typedef void (*ggml_v3_to_float_t)  (const void  * GGML_V3_RESTRICT x, float * GGML_V3_RESTRICT y, int k);
    typedef void (*ggml_v3_from_float_t)(const float * GGML_V3_RESTRICT x, void  * GGML_V3_RESTRICT y, int k);
    typedef void (*ggml_v3_vec_dot_t)   (const int n, float * GGML_V3_RESTRICT s, const void * GGML_V3_RESTRICT x, const void * GGML_V3_RESTRICT y);

    typedef struct {
        const char      * type_name;
        int               blck_size;
        size_t            type_size;
        bool              is_quantized;
        ggml_v3_to_float_t   to_float;
        ggml_v3_from_float_t from_float;
        ggml_v3_from_float_t from_float_reference;
        ggml_v3_vec_dot_t    vec_dot;
        enum ggml_v3_type    vec_dot_type;
    } ggml_v3_type_traits_t;

    GGML_V3_API ggml_v3_type_traits_t ggml_v3_internal_get_type_traits(enum ggml_v3_type type);

//allocator stuff

GGML_V3_API struct ggml_v3_allocr * ggml_v3_allocr_new(void * data, size_t size, size_t alignment);
GGML_V3_API struct ggml_v3_allocr * ggml_v3_allocr_new_measure(size_t alignment);
// tell the allocator to parse nodes following the order described in the list
// you should call this if your graph are optimized to execute out-of-order
GGML_V3_API void   ggml_v3_allocr_set_parse_seq(struct ggml_v3_allocr * alloc, const int * list, int n);
GGML_V3_API void   ggml_v3_allocr_free(struct ggml_v3_allocr * alloc);
GGML_V3_API bool   ggml_v3_allocr_is_measure(struct ggml_v3_allocr * alloc);
GGML_V3_API void   ggml_v3_allocr_reset(struct ggml_v3_allocr * alloc);
GGML_V3_API void   ggml_v3_allocr_alloc(struct ggml_v3_allocr * alloc, struct ggml_v3_tensor * tensor);
GGML_V3_API size_t ggml_v3_allocr_alloc_graph(struct ggml_v3_allocr * alloc, struct ggml_v3_cgraph * graph);
GGML_V3_API size_t ggml_v3_allocr_max_size(struct ggml_v3_allocr * alloc);

#define GGML_V3_GRAPH_HASHTABLE_SIZE 32771
#define GGML_V3_MAX_NODES 8192

#ifdef  __cplusplus
}
#endif