/** * @file calrt_utils.h * @author your name (you@domain.com) * @brief * @version 0.1 * @date 2024-11-11 * * @copyright Copyright (c) 2024 * */ #pragma once #include "calrt_platform.h" #include #include #include #include #include #include "calrt_error.h" namespace calrt { enum CALRT_API PrimitiveType { U1=0, //1bit PRED=1, //bool, int8 U8=2, S8=3, FP8_143=4, // exp_bias = 8; has subnormal, no inf/nan. when biased_exp=0, it is fixed explained as unbiased_exp=-7; FP8_152=5, // exp_bias = 16; has subnormal, no inf/nan. when biased_exp=0, it is fixed explained as unbiased_exp=-15; U16=6, S16=7, //16b BF16=8, //16b U32=9, //32b S32=10, F32=11, U4=12, S4=13, F35=14, U64=15, // not support S64=16, // u64和s64需要保留,仅做类型转换,因为pytorch要求tensor做index时必须为long或byte或bool。 TF32=17,//19b // not support F16=18, F64=19, C64, C128, TUPLE,//0 TOKEN,//32b OPAQUE_TYPE,//32b INVALID, //0b TYPE_SIZE }; PrimitiveType PrimitiveTypeFromInt(const int32_t& type); std::string PrimitiveTypeToString(const PrimitiveType& type); std::string PrimitiveTypeToString(const int32_t& type); CALRT_API int32_t PrimitiveTypeBitSize(const PrimitiveType& type); struct CALRT_API CalbinTensorInfo_s { std::string name; std::vector shape; PrimitiveType dataType; int64_t devAddr0 = -1; // ping int64_t devAddr1 = -1; // pong int64_t size = -1; }; enum CalbinSectionType_e { CALBIN_SECTION_NULL = 0, CALBIN_SECTION_CPU_CMD , CALBIN_SECTION_CPU_SO , CALBIN_SECTION_CCU_ELF , CALBIN_SECTION_RODATA , CALBIN_SECTION_IBUF , CALBIN_SECTION_OBUF , CALBIN_SECTION_MEMRSVD , CALBIN_SECTION_KV , CALBIN_SECTION_CSR , CALBIN_SECTION_NUM }; constexpr const char * CalbinSecTypeToString(CalbinSectionType_e e) { switch (e) { case CALBIN_SECTION_CPU_CMD: return "CPU command"; case CALBIN_SECTION_CCU_ELF: return "CCU ELF"; case CALBIN_SECTION_CPU_SO: return ".so file"; case CALBIN_SECTION_RODATA: return "parameter"; case CALBIN_SECTION_IBUF: return "input buffer"; case CALBIN_SECTION_OBUF: return "output buffer"; case CALBIN_SECTION_MEMRSVD: return "reserved memory"; case CALBIN_SECTION_KV: return "kv cache"; case CALBIN_SECTION_CSR: return "csr"; default: return "null"; } } enum CalbinSectionPlace_e{ DRAM = 0, SRAM, UNKNOWN }; /** * @brief * @note CalbinSectionType_e == CPU_SO. only need offset(used for prgIdx) devAddr, size, */ struct CALRT_API CalbinSection_s { CalbinSectionPlace_e place = UNKNOWN; CalbinSectionType_e type = CALBIN_SECTION_NULL; // cpu txt, ccu elf, data std::string name = ""; std::string filePath = ""; // must include elf/cpu file path int64_t offset = -1; // [progbit & .bss] entry address int64_t devAddr = -1; // -1 means calrt set memory, for memory and program int64_t size = -1; int64_t progEntry = -1; // program header entry address std::vector chipMask; // mask2, mask1, mask0, small-endian std::vector tensorInfos; // for i/o buf bool operator<(const CalbinSection_s& other) const; bool operator==(const CalbinSection_s& other) const; bool operator!=(const CalbinSection_s& other) const; friend std::ostream& operator<< (std::ostream& os, const CalbinSection_s&); }; struct CALRT_API KV_Cache_s { PrimitiveType dtype = INVALID; PrimitiveType sin_cos_table_data_type = INVALID; uint32_t layer; uint64_t kv_base[2]; //[0] k base address, [1] v base address uint64_t size; uint32_t n_head; uint32_t n_dim; operator bool () const noexcept { return dtype != INVALID; } }; struct CALRT_API CalbinLLM_s { uint32_t max_batch_size = 0; // 16 uint32_t max_seq_len = 0; KV_Cache_s kv_cache; operator bool () const noexcept { return max_batch_size != 0 && max_seq_len != 0; } }; enum ModelChipMode_e { SCHIP=0, MCHIP=1, UNDEFINED_CHIP_MODE }; struct CALRT_API CalbinModel { enum CalDevAccType_e { CAL_DEV_ACC_NULL = 0, CAL_DEV_ACC_CPU = 1, CAL_DEV_ACC_CCU = 2, CAL_DEV_ACC_CPU_CCU = 3 }; bool isConfigured = false; CalDevAccType_e devAccType; ModelChipMode_e chipMode = UNDEFINED_CHIP_MODE; std::string modelType; std::string modelArch; std::string modelName; std::unordered_map> sections; //idx follow above std::vector tarChipN; }; struct CALRT_API CalrtCalbin { uint64_t name; // md5 value or hash value std::vector models; }; enum CalrtBufferDirection_e{ HostToDevice = 0, DeviceToHost = 1, }; enum class PowerMode : uint8_t { BALANCE = 1, HIGH_PERFORMANCE = 2 }; class CALRT_API CalrtTensor { public: CalrtTensor() = delete; CalrtTensor(const std::vector &shape, PrimitiveType elemType, CalrtBufferDirection_e direction, std::string name = ""); template CalrtError_e CheckTensorByDataSize(const std::vector &datas_or_shape, PrimitiveType dtype) { bool success = mElemBitSize == PrimitiveTypeBitSize(dtype); if(!success) return CalrtErrorInvalidDataType; success = success && ((int64_t)datas_or_shape.size() == mElemSize); if(!success) return CalrtErrorInvalidDataShape; return CalrtSuccess; } CalrtError_e CheckTensor(const std::vector &shape, PrimitiveType dtype); /** * @brief safe to copy date to tensor in byte * @note for special custom data type, Using GetDataPtr() to manuelly fill data into tensor. * * @tparam T * @param datas input data * @param dtype data type */ template void Fill(std::vector &datas, PrimitiveType dtype) { FillImpl(static_cast(datas.data()), CheckTensorByDataSize(datas, dtype)); } /** * @brief map host buffer address to device address * @note unsafety for non-tensor check * * @param src */ void MapBuf(void *src); void UnMapBuf(); /** * @brief Get the Data Ptr object. User must guarantee a safety copy action * * @return char* */ uint8_t* GetDataPtr(); PrimitiveType Type() {return mElemType;} const std::vector &Shape() {return mShape;} /** * @brief Get the current tensor byte size * * @return const int64_t */ uint64_t ByteSize() {return mByteSize;} /** * @brief Get the number of element from the current tensor based on dtype * * @return const int64_t */ uint64_t Size() {return mElemSize;} const std::string &Name() {return mName;} CalrtBufferDirection_e Direction(); CalrtError_e SliceTensor(uint64_t offset, uint64_t size); void UndoSlice(); uint64_t Offset(); uint64_t TransSize(); friend std::ostream& operator<<(std::ostream& os, const CalrtTensor& obj); private: void FillImpl(void* src, CalrtError_e err); std::vector mShape; PrimitiveType mElemType; // tensor data elem type CalrtBufferDirection_e mDirection; std::string mName; uint32_t mElemBitSize; // each elem bit size uint64_t mElemSize; // elem size uint64_t mByteSize; // momory byte size uint64_t mOffset; uint64_t mTransDataSize; std::vector mData; uint8_t *mRawPtr = nullptr; }; struct CalrtDevBuf_s { int64_t addr0 = -1; int64_t addr1 = -1; int64_t size = -1; friend std::ostream& operator<<(std::ostream& os, const CalrtDevBuf_s& obj); }; struct CalrtBufferInfo_s{ CalrtBufferDirection_e direction; std::string name; std::vector tensorInfos; std::vector< std::pair> csrTable; }; enum class TaskType_e : uint32_t { TASK_PING = 0, TASK_PONG, UNDEFINED_TASK }; enum class ChipArch_e : uint32_t { SINGLE_CHIP_TASK = 0, MULTIPLE_CHIP_TASK, UNDEFINED }; struct TaskInfo_s { TaskType_e m_taskType; uint32_t m_jobId; uint32_t m_enableCcu; ChipArch_e m_taskChipArch; uint32_t m_syncRegIdx; uint32_t m_syncExpectedVal; }; // struct CalrtVersion // { // uint32_t major; // API or ABI change. E.G. remove/change function, user may need to adapt their code // uint32_t minor; // added new feature // uint32_t patch; // fix bug // }; CALRT_API const char *calrt_version(); CALRT_API void SetFullDebug(bool enable); CALRT_API bool isFullDebug(); }