#pragma once

#include <atomic>
#include <folly/lang/Align.h>
#include <limits>
#include <utility>

#include "common/net/Allocator.h"
#include "common/net/MessageHeader.h"
#include "common/net/RequestOptions.h"
#include "common/serde/MessagePacket.h"
#include "common/serde/Serde.h"
#include "common/utils/DownwardBytes.h"
#include "common/utils/ObjectPool.h"
#include "common/utils/ZSTD.h"

namespace hf3fs::net {

/*
 * A buffer serialized by serde.
 * +----------+-----------------------------------------+
 * |          |             buffer to write             |
 * | headroom +-------------------------------+---------+
 * |          | MessageHeader(checksum, size) | message |
 * +----------+-------------------------------+---------+
 */

;
static_assert(sizeof(SerdeBuffer) == 8);
static_assert(std::is_trivial_v<SerdeBuffer>, "SerdeBuffer is not trivial");

// An item containing a buffer to be written.
struct WriteItem {
  using Pool = ObjectPool<WriteItem, 1024, 64 * 1024>;

  std::atomic<WriteItem *> next = nullptr;
  decltype(SerdeBuffer::create(serde::MessagePacket<>{}, {})) buf;

  uint32_t retryTimes = 0;
  uint32_t maxRetryTimes = kDefaultMaxRetryTimes;

  size_t uuid = std::numeric_limits<size_t>::max();
  bool isReq() const { return uuid != std::numeric_limits<size_t>::max(); }

  template <serde::SerdeType T>
  static auto createMessage(const T &packet, const CoreRequestOptions &options) {
    auto item = Pool::get();
    item->buf = SerdeBuffer::create(packet, options);
    item->maxRetryTimes = options.sendRetryTimes;
    return item;
  }
};
using WriteItemPtr = WriteItem::Pool::Ptr;

class Transport;

class WriteList {
 public:
  WriteList() = default;
  explicit WriteList(WriteItemPtr item) { head_ = tail_ = item.release(); }
  WriteList(WriteItem *head, WriteItem *tail)
      : head_(head),
        tail_(tail) {}
  WriteList(const WriteList &) = delete;
  WriteList(WriteList &&o)
      : head_(std::exchange(o.head_, nullptr)),
        tail_(std::exchange(o.tail_, nullptr)) {}
  ~WriteList() { clear(); }

  bool empty() const { return head_ == nullptr; }

  void clear();

  WriteList extractForRetry();

  void concat(WriteList &&o) {
    if (o.empty()) {
      return;
    }
    if (empty()) {
      head_ = std::exchange(o.head_, nullptr);
      tail_ = std::exchange(o.tail_, nullptr);
    } else {
      tail_->next.store(std::exchange(o.head_, nullptr), std::memory_order_relaxed);
      tail_ = std::exchange(o.tail_, nullptr);
    }
  }

  void setTransport(std::shared_ptr<Transport> tr);

 protected:
  friend class MPSCWriteList;
  WriteItem *head_ = nullptr;
  WriteItem *tail_ = nullptr;
};

class WriteListWithProgress : public WriteList {
 public:
  using WriteList::WriteList;

  uint32_t toIOVec(struct iovec *iovec, uint32_t len, size_t &size);
  void advance(size_t written);

 private:
  // the write offset of the first write item.
  uint32_t firstOffset_ = 0;
};

class MPSCWriteList {
 public:
  ~MPSCWriteList() { takeOut().clear(); }

  // add a list of items. [thread-safe]
  void add(WriteList list);

  // fetch a batch of write items.
  WriteList takeOut();

 private:
  alignas(folly::hardware_destructive_interference_size) std::atomic<WriteItem *> head_;
};

}  // namespace hf3fs::net

class SerdeBuffer {
 public:
  uint8_t *buff() { return self() + headroom_; }
  const uint8_t *buff() const { return self() + headroom_; }
  auto &header() { return *reinterpret_cast<MessageHeader *>(buff()); }
  auto &header() const { return *reinterpret_cast<const MessageHeader *>(buff()); }
  uint8_t *rawMsg() { return buff() + kMessageHeaderSize; }
  const uint8_t *rawMsg() const { return buff() + kMessageHeaderSize; }
  uint32_t length() const { return kMessageHeaderSize + header().size; }
  size_t capacity() const { return capacity_; }
  template <class Allocator>
  struct Deleter {
    void operator()(SerdeBuffer *buf) { Allocator::deallocate(buf->self(), buf->capacity()); }
  };
  template <serde::SerdeType T, class Allocator = net::Allocator<4_KB, 1024, 64 * 1024>>
  static auto create(const T &packet, const CoreRequestOptions &options) {
    serde::Out<DownwardBytes<Allocator>> out;
    serde::serialize(packet, out);
    MessageHeader header;
    header.size = out.bytes().size();
    out.bytes().append(&header, sizeof(header));
    out.bytes().reserve(header.size + kMessageHeaderSize + sizeof(SerdeBuffer));
    uint32_t offset;
    uint32_t capacity;
    std::unique_ptr<SerdeBuffer, Deleter<Allocator>> ptr{
        reinterpret_cast<SerdeBuffer *>(out.bytes().release(offset, capacity))};
    ptr->headroom_ = offset;
    ptr->capacity_ = capacity;
    if (packet.timestamp.has_value()) {
      auto timestamp = reinterpret_cast<hf3fs::serde::Timestamp *>(ptr->rawMsg() + ptr->header().size -
                                                                   sizeof(hf3fs::serde::Timestamp));
      if (packet.isRequest()) {
        timestamp->clientSerialized = UtcClock::now().time_since_epoch().count();
      } else {
        timestamp->serverSerialized = UtcClock::now().time_since_epoch().count();
      }
    }
    auto compress = options.compression.enable(capacity);
    if (compress) {
      auto originalSize = ptr->header().size;
      auto compressedOffset = sizeof(SerdeBuffer) + sizeof(MessageHeader);
      auto compressedBound = ZSTD_compressBound(originalSize);
      auto compressedCapacity = compressedOffset + compressedBound;
      std::unique_ptr<SerdeBuffer, Deleter<Allocator>> compressed{
          reinterpret_cast<SerdeBuffer *>(Allocator::allocate(compressedCapacity))};
      compressed->headroom_ = sizeof(SerdeBuffer);
      compressed->capacity_ = compressedCapacity;
      int ret = ZSTD_compress(compressed->rawMsg(),
                              compressedBound,
                              ptr->rawMsg(),
                              ptr->header().size,
                              options.compression.level);
      if (LIKELY(!ZSTD_isError(ret))) {
        compressed->header().size = ret;
        ptr = std::move(compressed);
      } else {
        compress = false;
      }
    }
    ptr->header().checksum = Checksum::calcSerde(ptr->rawMsg(), ptr->header().size, compress);
    return ptr;
  }
 protected:
  SerdeBuffer() = default;
  uint8_t *self() { return reinterpret_cast<uint8_t *>(this); }
  const uint8_t *self() const { return reinterpret_cast<const uint8_t *>(this); }
 private:
  uint32_t headroom_;
  uint32_t capacity_;
}

#pragma once

#include "common/net/IOWorker.h"
#include "common/net/Transport.h"
#include "common/net/Waiter.h"
#include "common/net/WriteItem.h"
#include "common/net/sync/ConnectionPool.h"
#include "common/serde/ClientContext.h"
#include "common/serde/MessagePacket.h"
#include "common/serde/Serde.h"
#include "common/serde/Service.h"
#include "common/utils/Duration.h"

namespace hf3fs::serde {

class ClientContext {
 public:
  ClientContext(net::IOWorker &ioWorker,
                net::Address destAddr,
                const folly::atomic_shared_ptr<const net::CoreRequestOptions> &options)
      : connectionSource_(&ioWorker),
        destAddr_(destAddr),
        options_(options) {}

  ClientContext(net::sync::ConnectionPool &connectionPool,
                net::Address destAddr,
                const folly::atomic_shared_ptr<const net::CoreRequestOptions> &options)
      : connectionSource_(&connectionPool),
        destAddr_(destAddr),
        options_(options) {}

  ClientContext(const net::TransportPtr &tr);

  template <NameWrapper kServiceName,
            NameWrapper kMethodName,
            class Req,
            class Rsp,
            uint16_t ServiceID,
            uint16_t MethodID>
  CoTryTask<Rsp> call(const Req &req,
                      const net::UserRequestOptions *customOptions = nullptr,
                      Timestamp *timestamp = nullptr) {
    auto options = *options_.load(std::memory_order_acquire);
    if (customOptions != nullptr) {
      options.merge(*customOptions);
    }

    net::Waiter::Item item;
    uint64_t uuid = net::Waiter::instance().bind(item);

    Timestamp ts;
    if ((options.logLongRunningThreshold != 0_ms || options.reportMetrics) && timestamp == nullptr) {
      timestamp = &ts;
    }

    MessagePacket packet(req);
    packet.uuid = uuid;
    packet.serviceId = ServiceID;
    packet.methodId = MethodID;
    packet.flags = EssentialFlags::IsReq;
    if (options.compression) {
      packet.flags |= EssentialFlags::UseCompress;
    }
    if (options.enableRDMAControl) {
      packet.flags |= EssentialFlags::ControlRDMA;
    }
    if (timestamp != nullptr) {
      packet.timestamp = Timestamp{UtcClock::now().time_since_epoch().count()};
    }

    auto writeItem = net::WriteItem::createMessage(packet, options);
    writeItem->uuid = uuid;
    auto requestLength = writeItem->buf->length();
    if (LIKELY(std::holds_alternative<net::IOWorker *>(connectionSource_))) {
      std::get<net::IOWorker *>(connectionSource_)->sendAsync(destAddr_, net::WriteList(std::move(writeItem)));
    } else if (std::holds_alternative<net::Transport *>(connectionSource_)) {
      std::get<net::Transport *>(connectionSource_)->send(net::WriteList(std::move(writeItem)));
    } else {
      co_return MAKE_ERROR_F(StatusCode::kFoundBug,
                             "Sync client call async method: service id {}, method id {}",
                             ServiceID,
                             MethodID);
    }

    net::Waiter::instance().schedule(uuid, options.timeout);
    co_await item.baton;

    if (UNLIKELY(!item.status)) {
      if (item.status.code() == RPCCode::kTimeout && std::holds_alternative<net::IOWorker *>(connectionSource_)) {
        if (item.transport) {
          XLOGF(INFO, "req timeout and close transport {}", fmt::ptr(item.transport.get()));
          co_await item.transport->closeIB();
        } else {
          XLOGF(INFO, "req timeout but no transport");
        }
        std::get<net::IOWorker *>(connectionSource_)->checkConnections(destAddr_, Duration::zero());
      }
      co_return makeError(std::move(item.status));
    }

    Result<Rsp> rsp = makeError(StatusCode::kUnknown);
    auto deserializeResult = serde::deserialize(rsp, item.packet.payload);
    if (UNLIKELY(!deserializeResult)) {
      XLOGF(ERR, "deserialize rsp error: {}", deserializeResult.error());
      if (item.transport) {
        item.transport->invalidate();
      }
      co_return makeError(std::move(deserializeResult.error()));
    }
    if (timestamp != nullptr && item.packet.timestamp.has_value()) {
      *timestamp = *item.packet.timestamp;
      timestamp->clientWaked = UtcClock::now().time_since_epoch().count();
    }
    if (options.logLongRunningThreshold != 0_ms && timestamp->totalLatency() >= options.logLongRunningThreshold) {
      XLOGF(WARNING,
            "req takes too long, total {}, server {}, network {}, queue {}\ndetails {}",
            timestamp->totalLatency(),
            timestamp->serverLatency(),
            timestamp->networkLatency(),
            timestamp->queueLatency(),
            serde::toJsonString(*timestamp));
    }
    if (timestamp && options.reportMetrics) {
      static const auto methodName = fmt::format("{}::{}",
                                                 static_cast<std::string_view>(kServiceName),
                                                 static_cast<std::string_view>(kMethodName));
      static const auto tags = monitor::TagSet::create("instance", methodName);
      reportMetrics(tags, timestamp, requestLength, item.buf->length());
    }
    co_return rsp;
  }

  template <NameWrapper kServiceName,
            NameWrapper kMethodName,
            class Req,
            class Rsp,
            uint16_t ServiceID,
            uint16_t MethodID>
  Result<Rsp> callSync(const Req &req,
                       const net::UserRequestOptions *customOptions = nullptr,
                       Timestamp *timestamp = nullptr) {
    auto options = *options_.load(std::memory_order_acquire);
    if (customOptions != nullptr) {
      options.merge(*customOptions);
    }
    auto startTime = RelativeTime::now();

    MessagePacket packet(req);
    packet.serviceId = ServiceID;
    packet.methodId = MethodID;
    packet.flags = EssentialFlags::IsReq;
    if (timestamp != nullptr) {
      packet.timestamp = Timestamp{UtcClock::now().time_since_epoch().count()};
    }
    auto buff = net::SerdeBuffer::create<MessagePacket<Req>, net::SystemAllocator<>>(packet, options);

    // acquire connection.
    if (UNLIKELY(!std::holds_alternative<net::sync::ConnectionPool *>(connectionSource_))) {
      return makeError(RPCCode::kConnectFailed);
    }
    auto connectionPool = std::get<net::sync::ConnectionPool *>(connectionSource_);
    auto acquireConnectionResult = connectionPool->acquire(destAddr_);
    if (UNLIKELY(!acquireConnectionResult)) {
      return makeError(RPCCode::kConnectFailed);
    }

    auto conn = std::move(acquireConnectionResult.value());
    RETURN_AND_LOG_ON_ERROR(conn->sendSync(buff->buff(), buff->length(), startTime, options.timeout));

    net::MessageHeader header;
    RETURN_AND_LOG_ON_ERROR(
        conn->recvSync(folly::MutableByteRange{reinterpret_cast<uint8_t *>(&header), net::kMessageHeaderSize},
                       startTime,
                       options.timeout));
    auto rspBuff = net::IOBuf::createCombined(header.size);
    rspBuff->append(header.size);
    RETURN_AND_LOG_ON_ERROR(
        conn->recvSync(folly::MutableByteRange{rspBuff->writableData(), header.size}, startTime, options.timeout));

    MessagePacket recv;
    RETURN_AND_LOG_ON_ERROR(
        serde::deserialize(recv, std::string_view(reinterpret_cast<const char *>(rspBuff->buffer()), header.size)));

    Result<Rsp> rsp = makeError(StatusCode::kUnknown);
    RETURN_AND_LOG_ON_ERROR(serde::deserialize(rsp, recv.payload));
    if (timestamp != nullptr && recv.timestamp.has_value()) {
      *timestamp = *recv.timestamp;
      timestamp->clientWaked = UtcClock::now().time_since_epoch().count();
    }
    connectionPool->restore(destAddr_, std::move(conn));
    return rsp;
  }

  net::Address addr() const { return destAddr_; }

 private:
  static void reportMetrics(const monitor::TagSet &tags, Timestamp *ts, uint64_t requestSize, uint64_t responseSize);

  std::variant<net::IOWorker *, net::sync::ConnectionPool *, net::Transport *> connectionSource_;
  net::Address destAddr_{};
  const folly::atomic_shared_ptr<const net::CoreRequestOptions> &options_;
};

}  // namespace hf3fs::serde

#pragma once

#include <bit>
#include <cstdint>
#include <folly/hash/Checksum.h>
#include <type_traits>

#include "common/net/Network.h"
#include "common/utils/Size.h"

namespace hf3fs::net {

constexpr uint32_t kMessageHeaderSize = 8u;
constexpr uint8_t kSerdeMessageMagicNum = 0x86;
// The most extreme case of storage sync:
// target size 30TB / chunk size 512KB * single meta size 37B / compression ratio 5 = 444MB
constexpr size_t kMessageMaxSize = 512_MB;
constexpr size_t kMessageReadBufferSize = 32_KB;

struct MessageHeader {
  uint32_t checksum;
  uint32_t size;

  bool isSerdeMessage() const { return (checksum & 0xfeu) == kSerdeMessageMagicNum; }
  bool isCompressed() const { return isCompressed(checksum); }
  static bool isCompressed(uint32_t checksum) { return checksum & 1; }
};
static_assert(sizeof(MessageHeader) == kMessageHeaderSize, "sizeof(MessageHeader) != kMessageHeaderSize");
static_assert(kMessageHeaderSize % sizeof(uint64_t) == 0, "MessageHeader is not aligned by sizeof(uint64_t)!");
static_assert(std::is_trivial_v<MessageHeader>, "std::is_trivial_v<MessageHeader> is false!");

struct Checksum {
  // calculate crc32c and overwrite the lower 8 bits with serde magic number.
  static inline uint32_t calcSerde(const uint8_t *data, size_t size, bool compressed = false) {
    uint32_t crc32c = folly::crc32c(data, size, 0);
    return (crc32c & ~0xffu) | kSerdeMessageMagicNum | compressed;
  }
};

class MessageWrapper : folly::MoveOnly {
 public:
  MessageWrapper(IOBufPtr buf)
      : buf_(std::move(buf)) {}

  // get the capacity of the total buffer.
  size_t capacity() const { return buf_->capacity(); }

  // get the length of the remaining buffer.
  size_t length() const { return buf_->length(); }

  // check whether the current header is complete.
  bool headerComplete() const { return kMessageHeaderSize <= length(); }

  // get the header of current message.
  auto &header() const { return *reinterpret_cast<const MessageHeader *>(buf_->data()); }

  // get the length of the current message.
  size_t messageLength() const { return kMessageHeaderSize + header().size; }

  // check whether the current message is complete.
  bool messageComplete() const { return messageLength() <= length(); }

  // clone current message.
  IOBufPtr cloneMessage() {
    auto newBuf = buf_->cloneOne();
    newBuf->trimEnd(length() - messageLength());
    newBuf->trimStart(sizeof(MessageHeader));
    return newBuf;
  }

  // seek to next message header.
  void next() { buf_->trimStart(messageLength()); }

  // allocate a empty buffer.
  static IOBufPtr allocate(size_t size);

  // create a new buffer with the remaining data.
  IOBufPtr createFromRemain(size_t minSize);

  // check whether at least one message is complete in this wrapper.
  bool hasMessages() const { return buf_->headroom(); }

  // seek to begin position.
  void seekBegin() { buf_->prepend(buf_->headroom()); }

  // detach the buffer.
  IOBufPtr detach() { return std::move(buf_); }

  // is serde message.
  bool isSerdeMessage() const { return header().isSerdeMessage(); }

 private:
  IOBufPtr buf_;
};

}  // namespace hf3fs::net

#pragma once

#include <cstring>
#include <exception>
#include <folly/Likely.h>
#include <folly/logging/xlog.h>
#include <memory>
#include <stdexcept>

namespace hf3fs {

template <class Allocator>
class DownwardBytes {
 public:
  DownwardBytes() = default;
  DownwardBytes(const DownwardBytes &o)
      : offset_(o.offset_),
        capacity_(o.capacity_),
        data_(Allocator::allocate(capacity_)) {
    std::memcpy(this->data(), o.data(), o.size());
  }
  DownwardBytes(DownwardBytes &&o)
      : offset_(std::exchange(o.offset_, 0)),
        capacity_(std::exchange(o.capacity_, 0)),
        data_(std::exchange(o.data_, nullptr)) {}
  ~DownwardBytes() { reset(); }

  DownwardBytes &operator=(const DownwardBytes &o) {
    if (std::addressof(o) != this) {
      reserve(o.size());
      offset_ = capacity_ - o.size();
      std::memcpy(this->data(), o.data(), o.size());
    }
    return *this;
  }
  DownwardBytes &operator=(DownwardBytes &&o) {
    if (std::addressof(o) != this) {
      offset_ = std::exchange(o.offset_, 0);
      capacity_ = std::exchange(o.capacity_, 0);
      data_ = std::exchange(o.data_, nullptr);
    }
    return *this;
  }

  auto size() const { return capacity_ - offset_; }
  auto capacity() const { return capacity_; }
  auto data() { return data_ + offset_; }
  auto data() const { return data_ + offset_; }
  auto &operator[](size_t idx) { return data()[idx]; }
  auto &operator[](size_t idx) const { return data()[idx]; }
  operator std::string_view() const { return std::string_view{reinterpret_cast<const char *>(data()), size()}; }

  void append(auto *data, uint32_t size) {
    reserve(this->size() + size);
    offset_ -= size;
    std::memcpy(this->data(), data, size);
  }

  void append(uint8_t value) {
    reserve(this->size() + 1);
    offset_ -= 1;
    *data() = value;
  }

  void reset() {
    if (data_) {
      Allocator::deallocate(std::exchange(data_, nullptr), std::exchange(capacity_, 0));
    }
    offset_ = 0;
  }

  void reserve(uint32_t size) {
    if (UNLIKELY(size > capacity_)) {
      auto newCap = std::max(size, capacity_ * 2);
      auto newBuf = Allocator::allocate(newCap);
      if (auto length = this->size()) {
        auto newOffset = newCap - length;
        std::memcpy(newBuf + newOffset, this->data(), length);
        offset_ = newOffset;
      } else {
        offset_ = newCap;
      }
      Allocator::deallocate(data_, capacity_);
      capacity_ = newCap;
      data_ = newBuf;
    }
  }

  uint8_t *release(uint32_t &offset, uint32_t &capacity) {
    offset = std::exchange(offset_, 0);
    capacity = std::exchange(capacity_, 0);
    return std::exchange(data_, nullptr);
  }

  std::string toString() const { return std::string{*this}; }

 private:
  uint32_t offset_ = Allocator::kDefaultSize;
  uint32_t capacity_ = Allocator::kDefaultSize;
  std::uint8_t *data_ = Allocator::allocate(Allocator::kDefaultSize);
};

template <>
class DownwardBytes<void> {
 public:
  auto size() const { return length_; }
  auto &operator[](size_t) { return dummy_; }
  void append(auto *, uint32_t size) { length_ += size; }

 private:
  uint32_t length_ = 0;
  uint32_t dummy_ = 0;
};

struct UserBufferAllocator {};

template <>
class DownwardBytes<UserBufferAllocator> {
 public:
  DownwardBytes() = default;

  void setBuffer(std::uint8_t *data, uint32_t capacity) {
    data_ = data;
    offset_ = capacity_ = capacity;
  }

  auto size() const { return capacity_ - offset_; }
  auto capacity() const { return capacity_; }
  auto data() { return data_ + offset_; }
  auto data() const { return data_ + offset_; }
  auto &operator[](size_t idx) { return data()[idx]; }
  auto &operator[](size_t idx) const { return data()[idx]; }
  operator std::string_view() const { return std::string_view{reinterpret_cast<const char *>(data()), size()}; }

  void append(auto *data, uint32_t size) {
    if (offset_ < size) {
      XLOGF(FATAL, "insufficient capacity! cap {} offset {} size {}", capacity_, offset_, size);
    }
    offset_ -= size;
    std::memcpy(this->data(), data, size);
  }

  void append(uint8_t value) {
    if (offset_ < 1) {
      XLOGF(FATAL, "insufficient capacity! cap {} offset {} size {}", capacity_, offset_, 1);
    }
    offset_ -= 1;
    *data() = value;
  }

  std::string toString() const { return std::string{*this}; }

 private:
  uint32_t offset_{};
  uint32_t capacity_{};
  std::uint8_t *data_{};
};

}  // namespace hf3fs

#pragma once

#include <algorithm>
#include <array>
#include <asm-generic/errno-base.h>
#include <atomic>
#include <chrono>
#include <condition_variable>
#include <cstddef>
#include <cstdint>
#include <cstdlib>
#include <deque>
#include <folly/Likely.h>
#include <folly/Range.h>
#include <folly/Utility.h>
#include <folly/experimental/coro/Mutex.h>
#include <folly/fibers/Semaphore.h>
#include <folly/futures/detail/Types.h>
#include <folly/io/IOBufQueue.h>
#include <folly/logging/xlog.h>
#include <gtest/gtest_prod.h>
#include <infiniband/verbs.h>
#include <map>
#include <memory>
#include <mutex>
#include <optional>
#include <span>
#include <sys/socket.h>
#include <utility>

#include "common/net/ib/IBDevice.h"
#include "common/serde/Serde.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Result.h"

namespace hf3fs::net {

using RDMABufMR = ibv_mr *;

class RDMARemoteBuf {
  struct Rkey {
    uint32_t rkey = 0;
    int devId = -1;

    bool operator==(const Rkey &) const = default;
  };

 public:
  RDMARemoteBuf()
      : addr_(0),
        length_(0),
        rkeys_() {}

  RDMARemoteBuf(uint64_t addr, size_t length, std::array<Rkey, IBDevice::kMaxDeviceCnt> rkeys)
      : addr_(addr),
        length_(length),
        rkeys_(rkeys) {}

  ~RDMARemoteBuf() = default;

  uint64_t addr() const { return addr_; }
  size_t size() const { return length_; }

  std::optional<uint32_t> getRkey(int devId) const {
    for (auto &rkey : rkeys_) {
      if (rkey.devId == devId) {
        return rkey.rkey;
      }
    }
    return std::nullopt;
  }

  bool advance(size_t len) {
    if (UNLIKELY(length_ < len)) {
      return false;
    }
    addr_ += len;
    length_ -= len;
    return true;
  }
  bool subtract(size_t n) {
    if (UNLIKELY(n > size())) {
      return false;
    }
    length_ -= n;
    return true;
  }

  RDMARemoteBuf subrange(size_t offset, size_t len) const {
    if (UNLIKELY(offset + len > length_)) {
      return RDMARemoteBuf();
    }
    RDMARemoteBuf remote = *this;
    remote.addr_ = addr_ + offset;
    remote.length_ = len;
    return remote;
  }

  RDMARemoteBuf first(size_t len) const { return subrange(0, len); }
  RDMARemoteBuf takeFirst(size_t len) {
    auto buf = first(len);
    advance(len);
    return buf;
  }

  RDMARemoteBuf last(size_t len) const {
    if (UNLIKELY(len > length_)) {
      return RDMARemoteBuf();
    }
    return subrange(length_ - len, len);
  }
  RDMARemoteBuf takeLast(size_t len) {
    auto buf = last(len);
    subtract(len);
    return buf;
  }

  bool valid() const { return addr_ != 0; }
  explicit operator bool() const { return valid(); }

  bool operator==(const RDMARemoteBuf &o) const = default;

  auto &rkeys() { return rkeys_; }
  const auto &rkeys() const { return rkeys_; }

 private:
  friend class RDMABuf;
  FRIEND_TEST(TestRDMARemoteBuf, Basic);
  FRIEND_TEST(TestIBSocket, RDMAFailure);

  uint64_t addr_;
  uint64_t length_;
  std::array<Rkey, IBDevice::kMaxDeviceCnt> rkeys_;
};

class RDMABufPool;

class RDMABuf {
 public:
  RDMABuf()
      : RDMABuf(nullptr, nullptr, 0) {}

  RDMABuf(const RDMABuf &) = default;
  RDMABuf(RDMABuf &&) = default;

  RDMABuf &operator=(const RDMABuf &) = default;
  RDMABuf &operator=(RDMABuf &&) = default;

  static RDMABuf allocate(size_t size) { return allocate(size, {}); }

  bool valid() const { return buf_ != nullptr; }
  explicit operator bool() const { return valid(); }
  auto raw() const { return buf_.get(); }

  const uint8_t *ptr() const { return begin_; }
  uint8_t *ptr() { return begin_; }

  size_t capacity() const { return buf_ ? buf_->capacity() : 0; }
  size_t size() const { return length_; }
  bool empty() const { return size() == 0; }

  bool contains(const uint8_t *data, uint32_t len) const { return ptr() <= data && data + len <= ptr() + capacity(); }

  void resetRange() {
    if (LIKELY(buf_ != nullptr)) {
      begin_ = buf_->ptr();
      length_ = buf_->capacity();
    }
  }

  bool advance(size_t n) {
    if (UNLIKELY(n > size())) {
      return false;
    }
    begin_ += n;
    length_ -= n;
    return true;
  }
  bool subtract(size_t n) {
    if (UNLIKELY(n > size())) {
      return false;
    }
    length_ -= n;
    return true;
  }

  RDMABuf subrange(size_t offset, size_t length) const {
    if (UNLIKELY(offset + length > size())) {
      return RDMABuf();
    }
    return RDMABuf(buf_, begin_ + offset, length);
  }

  RDMABuf first(size_t length) const { return subrange(0, length); }
  RDMABuf takeFirst(size_t length) {
    auto buf = first(length);
    advance(length);
    return buf;
  }

  RDMABuf last(size_t length) const {
    if (UNLIKELY(length > size())) {
      return RDMABuf();
    }
    return subrange(size() - length, length);
  }
  RDMABuf takeLast(size_t length) {
    auto buf = last(length);
    subtract(length);
    return buf;
  }

  RDMABufMR getMR(int dev) const {
    if (UNLIKELY(buf_ == nullptr)) {
      return RDMABufMR();
    }
    return buf_->getMR(dev);
  }

  RDMARemoteBuf toRemoteBuf() const {
    std::array<RDMARemoteBuf::Rkey, IBDevice::kMaxDeviceCnt> rkeys;
    if (UNLIKELY(!buf_ || !buf_->getRkeys(rkeys))) {
      return RDMARemoteBuf();
    }
    return RDMARemoteBuf((uint64_t)begin_, length_, rkeys);
  }

  operator RDMARemoteBuf() { return toRemoteBuf(); }

  operator std::span<const uint8_t>() const { return {ptr(), size()}; }
  operator std::span<uint8_t>() { return {ptr(), size()}; }

  operator folly::MutableByteRange() { return {ptr(), size()}; }
  operator folly::ByteRange() const { return {ptr(), size()}; }

  bool operator==(const RDMABuf &o) const = default;

 private:
  friend class RDMABufPool;

  class Inner : folly::MoveOnly {
   public:
    static constexpr int kAccessFlags =
        IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_RELAXED_ORDERING;

    Inner(size_t capacity)
        : Inner(std::weak_ptr<RDMABufPool>(), capacity) {}

    Inner(std::weak_ptr<RDMABufPool> pool, size_t capacity)
        : pool_(std::move(pool)),
          ptr_(nullptr),
          capacity_(capacity),
          mrs_(),
          userBuffer_(false) {}

    Inner(uint8_t *buf, size_t len)
        : pool_(std::weak_ptr<RDMABufPool>()),
          ptr_(buf),
          capacity_(len),
          mrs_(),
          userBuffer_(true) {}

    Inner(Inner &&o)
        : pool_(std::move(o.pool_)),
          ptr_(std::exchange(o.ptr_, nullptr)),
          capacity_(std::exchange(o.capacity_, 0)),
          mrs_(std::exchange(o.mrs_, std::array<RDMABufMR, IBDevice::kMaxDeviceCnt>())),
          userBuffer_(std::exchange(o.userBuffer_, false)) {}

    ~Inner();

    static void deallocate(Inner *ptr);

    int init();
    int allocateMemory();
    int registerMemory();

    RDMABufMR getMR(int dev) const;
    bool getRkeys(std::array<RDMARemoteBuf::Rkey, IBDevice::kMaxDeviceCnt> &rkeys) const;

    const uint8_t *ptr() const { return ptr_; }
    uint8_t *ptr() { return ptr_; }
    size_t capacity() const { return capacity_; }

   private:
    std::weak_ptr<RDMABufPool> pool_;
    uint8_t *ptr_;
    size_t capacity_;
    std::array<RDMABufMR, IBDevice::kMaxDeviceCnt> mrs_;
    bool userBuffer_;
  };

  static RDMABuf allocate(size_t size, std::weak_ptr<RDMABufPool> pool);

 public:
  RDMABuf(Inner *buf)
      : RDMABuf(std::shared_ptr<Inner>(buf, Inner::deallocate)) {}

  RDMABuf(std::shared_ptr<Inner> buf)
      : buf_(std::move(buf)) {
    begin_ = buf_->ptr();
    length_ = buf_->capacity();
  }

  RDMABuf(std::shared_ptr<Inner> buf, uint8_t *begin, size_t length)
      : buf_(std::move(buf)),
        begin_(begin),
        length_(length) {}

  static RDMABuf createFromUserBuffer(uint8_t *buf, size_t len);

 private:
  std::shared_ptr<Inner> buf_;
  uint8_t *begin_;
  size_t length_;
};

class RDMABufPool : public std::enable_shared_from_this<RDMABufPool>, folly::MoveOnly {
  struct PrivateTag {};

 public:
  RDMABufPool(PrivateTag, size_t bufSize, size_t bufCnt)
      : bufSize_(bufSize),
        bufAllocated_(0),
        sem_(bufCnt),
        mutex_(),
        freeList_() {}

  ~RDMABufPool();

  static std::shared_ptr<RDMABufPool> create(size_t bufSize, size_t bufCnt) {
    return std::make_shared<RDMABufPool>(PrivateTag(), bufSize, bufCnt);
  }

  CoTask<RDMABuf> allocate(std::optional<folly::Duration> timeout = std::nullopt);
  void deallocate(RDMABuf::Inner *buf);

  size_t bufSize() const { return bufSize_; }
  size_t freeCnt() const { return sem_.getAvailableTokens(); }
  size_t totalCnt() const { return sem_.getCapacity(); }

 private:
  size_t bufSize_;
  std::atomic<size_t> bufAllocated_;

  folly::fibers::Semaphore sem_;
  std::mutex mutex_;
  std::deque<RDMABuf::Inner *> freeList_;
};

}  // namespace hf3fs::net

template <>
struct ::hf3fs::serde::SerdeMethod<::hf3fs::net::RDMARemoteBuf> {
  static constexpr auto serialize(const net::RDMARemoteBuf &buf, auto &out) {
    uint8_t len = 0;
    for (auto &rkey : buf.rkeys()) {
      if (rkey.devId == -1) {
        break;
      } else {
        ++len;
      }
    }

    for (int8_t i = len - 1; i >= 0; --i) {
      serde::serialize(buf.rkeys()[i].devId, out);
      serde::serialize(buf.rkeys()[i].rkey, out);
    }
    serde::serialize(len, out);

    serde::serialize(buf.size(), out);
    serde::serialize(buf.addr(), out);
  }

  static Result<Void> deserialize(net::RDMARemoteBuf &buf, auto &&in) {
    uint64_t addr;
    uint64_t size;
    RETURN_AND_LOG_ON_ERROR(serde::deserialize(addr, in));
    RETURN_AND_LOG_ON_ERROR(serde::deserialize(size, in));

    int8_t len;
    RETURN_AND_LOG_ON_ERROR(serde::deserialize(len, in));

    auto rkeys = buf.rkeys();
    for (auto &rkey : rkeys) {
      if (len-- > 0) {
        RETURN_AND_LOG_ON_ERROR(serde::deserialize(rkey.rkey, in));
        RETURN_AND_LOG_ON_ERROR(serde::deserialize(rkey.devId, in));
      } else {
        rkey.devId = -1;
      }
    }
    buf = net::RDMARemoteBuf(addr, size, rkeys);
    return Void{};
  }

  static auto serializeReadable(const net::RDMARemoteBuf &buf, auto &out) {
    auto start = out.tableBegin(false);
    {
      out.key("addr");
      serde::serialize(buf.addr(), out);
      out.key("size");
      serde::serialize(buf.size(), out);

      out.key("rkeys");
      out.arrayBegin();
      {
        for (auto &rkey : buf.rkeys()) {
          if (rkey.devId == -1) {
            break;
          }

          auto start = out.tableBegin(false);
          out.key("rkey");
          serde::serialize(rkey.rkey, out);
          out.key("devId");
          serde::serialize(rkey.devId, out);
          out.tableEnd(start);
        }
      }
      out.arrayEnd(0);
    }
    out.tableEnd(start);
  };

  static std::string serdeToReadable(const net::RDMARemoteBuf &rdmabuf) { return serde::toJsonString(rdmabuf); }

  static Result<net::RDMARemoteBuf> serdeFromReadable(const std::string &str) {
    net::RDMARemoteBuf rdmabuf;
    auto result = serde::fromJsonString(rdmabuf, str);
    if (result) return rdmabuf;
    return makeError(result.error());
  }
};

#pragma once

#include <atomic>
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <folly/experimental/coro/ViaIfAsync.h>
#include <limits>
#include <thread>

#include "common/net/MessageHeader.h"
#include "common/net/Network.h"
#include "common/net/Transport.h"
#include "common/net/Waiter.h"
#include "common/serde/CallContext.h"
#include "common/serde/MessagePacket.h"
#include "common/serde/Serde.h"
#include "common/serde/Services.h"
#include "common/utils/CPUExecutorGroup.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/CoroutinesPool.h"
#include "common/utils/DynamicCoroutinesPool.h"
#include "common/utils/Uuid.h"
#include "common/utils/ZSTD.h"

namespace hf3fs::net {

class IOWorker;

class Processor {
 public:
  struct Config : public ConfigBase<Config> {
    // TODO: directly use CoroutinesPool::Config
    CONFIG_ITEM(max_processing_requests_num, 4096ul);
    CONFIG_ITEM(max_coroutines_num, 256ul);
    CONFIG_HOT_UPDATED_ITEM(enable_coroutines_pool, true);
    CONFIG_HOT_UPDATED_ITEM(response_compression_level, 1u);
    CONFIG_HOT_UPDATED_ITEM(response_compression_threshold, 128_KB);
  };

  struct Job {
    IOBufPtr buf;
    serde::MessagePacket<> packet;
    TransportPtr tr;
  };

  explicit Processor(serde::Services &serdeServices, CPUExecutorGroup &executor, const Config &config)
      : serdeServices_(serdeServices),
        executor_(executor),
        config_(config) {}

  void processMsg(MessageWrapper wrapper, TransportPtr tr) {
    auto num = processingRequestsNum(flags_.load(std::memory_order_acquire));
    if (UNLIKELY(num >= config_.max_processing_requests_num())) {
      XLOGF(WARN, "Too many processing requests: {}, unpack message in current thread.", num);
      unpackMsg(wrapper, tr);
    } else {
      executor_.pickNext().add(
          [this, wrapper = std::move(wrapper), tr = std::move(tr)]() mutable { unpackMsg(wrapper, tr); });
    }
  }

  void setCoroutinesPoolGetter(auto &&g) { coroutinesPoolGetter_ = std::forward<decltype(g)>(g); }

  Result<Void> start(std::string_view = {}) { return Void{}; }

  void stopAndJoin() {
    // Mark as stopped, and refuse new requests.
    auto flags = flags_.fetch_or(kStopFlag);
    while (processingRequestsNum(flags) != 0) {
      XLOGF(INFO, "Waiting for {} requests to finish...", processingRequestsNum(flags));
      std::this_thread::sleep_for(100ms);
      flags = flags_.load(std::memory_order_acquire);
    }
  }

  void setFrozen(bool frozen, bool isRDMA) {
    auto flags = (isRDMA ? kFrozenRDMA : kFrozenTCP);
    if (frozen) {
      flags_ |= flags;
    } else {
      flags_ &= ~flags;
    }
  }

 protected:
  void unpackMsg(MessageWrapper &wrapper, TransportPtr &tr) {
    while (wrapper.length()) {
      // 1. check the message is complete.
      if (UNLIKELY(!wrapper.headerComplete() || !wrapper.messageComplete())) {
        XLOGF(ERR,
              "Message is not complete! {} < {}, peer: {}",
              wrapper.length(),
              wrapper.headerComplete() ? wrapper.header().size : 0,
              tr->describe());
        tr->invalidate();
        return;
      }

      if (LIKELY(wrapper.isSerdeMessage())) {
        unpackSerdeMsg(wrapper.cloneMessage(), wrapper.header().checksum, tr);
        wrapper.next();
        continue;
      }

      XLOGF(ERR, "Message is invalid", tr->describe());
      tr->invalidate();
      return;
    }
  }

  Result<Void> decompressSerdeMsg(IOBufPtr &buf, TransportPtr &tr);

  void unpackSerdeMsg(IOBufPtr buf, uint32_t checksumIn, TransportPtr tr) {
    // 1. check checksum. (without timestamp part)
    bool isCompressed = MessageHeader::isCompressed(checksumIn);
    auto checksum = Checksum::calcSerde(buf->data(), buf->length(), isCompressed);
    if (UNLIKELY(checksumIn != checksum)) {
      XLOGF(ERR, "checksum is not matched! {:X} != {:X}, peer: {}", checksumIn, checksum, tr->describe());
      tr->invalidate();
      return;
    }
    // message is verified by checksum and serde deserialize.

    if (isCompressed) {
      auto result = decompressSerdeMsg(buf, tr);
      if (UNLIKELY(!result)) {
        return;
      }
    }

    auto view = std::string_view{reinterpret_cast<const char *>(buf->data()), buf->length()};
    serde::MessagePacket<> packet;
    auto deserializeResult = serde::deserialize(packet, view);
    if (UNLIKELY(!deserializeResult)) {
      XLOGF(ERR, "Message buffer deserialize failed {}!, peer: {}", deserializeResult.error(), tr->describe());
      tr->invalidate();
      return;
    }

    // 2. check this message is request or response.
    if (packet.timestamp) {
      if (packet.isRequest()) {
        packet.timestamp->serverReceived = UtcClock::now().time_since_epoch().count();
      } else {
        packet.timestamp->clientReceived = UtcClock::now().time_since_epoch().count();
      }
    }
    if (packet.flags & serde::EssentialFlags::IsReq) {
      // is request.
      XLOGF(DBG, "receive request {}:{}", packet.serviceId, packet.methodId);
      tryToProcessSerdeRequest(std::move(buf), packet, std::move(tr));
    } else {
      // is response.
      XLOGF(DBG, "receive response {}:{}", packet.serviceId, packet.methodId);
      Waiter::instance().post(packet, std::move(buf));
    }
  }

  CoTask<void> processSerdeRequest(IOBufPtr buf, serde::MessagePacket<> packet, TransportPtr tr) {
    // decrease the count in any case.
    auto guard = folly::makeGuard([&] { flags_ -= kCountInc; });

    (void)buf;  // keep alive.
    auto &service = serdeServices_.getServiceById(packet.serviceId, tr->isRDMA());
    serde::CallContext ctx(packet, std::move(tr), service);
    if (packet.useCompress()) {
      ctx.responseOptions().compression = {config_.response_compression_level(),
                                           config_.response_compression_threshold()};
    }
    co_await ctx.handle();
  }

  void tryToProcessSerdeRequest(IOBufPtr buff, serde::MessagePacket<> &packet, TransportPtr tr) {
    auto flags = flags_.load(std::memory_order_acquire);
    if (UNLIKELY(needRefuse(flags))) {
      refuseSerdeRequest(packet, std::move(tr), flags);
      return;
    }
    if (UNLIKELY(isFrozen(flags, tr->isRDMA()))) {
      // do nothing and return.
      return;
    }

    flags = flags_.fetch_add(kCountInc);
    if (UNLIKELY(needRefuse(flags))) {
      refuseSerdeRequest(packet, std::move(tr), flags);
      flags_ -= kCountInc;  // resume count.
      return;
    }

    if (coroutinesPoolGetter_) {
      coroutinesPoolGetter_(packet).enqueue(processSerdeRequest(std::move(buff), packet, std::move(tr)));
    } else {
      processSerdeRequest(std::move(buff), packet, std::move(tr)).scheduleOn(&executor_.pickNext()).start();
    }
  }

  void refuseSerdeRequest(serde::MessagePacket<> &packet, TransportPtr tr, size_t flags) {
    auto msg =
        fmt::format("Refuse requests, stopped: {}, processing: {}", isStopped(flags), processingRequestsNum(flags));
    XLOG(WARN, msg);
    auto &service = serdeServices_.getServiceById(packet.serviceId, tr->isRDMA());
    serde::CallContext ctx(packet, std::move(tr), service);
    ctx.onError(makeError(RPCCode::kRequestRefused, msg));
  }

  inline bool needRefuse(size_t flags) const {
    return isStopped(flags) || processingRequestsNum(flags) >= config_.max_processing_requests_num();
  }
  inline bool isStopped(size_t flags) const { return flags & kStopFlag; }
  inline bool isFrozen(size_t flags, bool isRDMA) const { return flags & (isRDMA ? kFrozenRDMA : kFrozenTCP); }
  inline size_t processingRequestsNum(size_t flags) const { return flags / kCountInc; }

 private:
  serde::Services &serdeServices_;
  CPUExecutorGroup &executor_;
  const Config &config_;
  using CoroutinesPoolGetter = std::function<DynamicCoroutinesPool &(const serde::MessagePacket<> &)>;
  CoroutinesPoolGetter coroutinesPoolGetter_{};

  constexpr static size_t kStopFlag = 1;
  constexpr static size_t kFrozenTCP = 2;
  constexpr static size_t kFrozenRDMA = 4;
  constexpr static size_t kCountInc = 8;
  std::atomic<size_t> flags_{0};
};

}  // namespace hf3fs::net

#pragma once

#include <algorithm>
#include <folly/Likely.h>
#include <iterator>
#include <limits>
#include <scn/scn.h>
#include <string_view>
#include <tuple>
#include <type_traits>

#include "common/net/Allocator.h"
#include "common/serde/TypeName.h"
#include "common/utils/DownwardBytes.h"
#include "common/utils/MagicEnum.hpp"
#include "common/utils/NameWrapper.h"
#include "common/utils/Path.h"
#include "common/utils/Reflection.h"
#include "common/utils/Result.h"
#include "common/utils/Thief.h"
#include "common/utils/TypeTraits.h"
#include "common/utils/Varint32.h"
#include "common/utils/Varint64.h"

#define SERDE_CLASS_TYPED_FIELD(TYPE, NAME, DEFAULT, ...)                                                          \
 private:                                                                                \
  friend struct ::hf3fs::refl::Helper;                                                                             \
  struct T##NAME : std::type_identity<REFL_NOW> {};                                                                \
  TYPE NAME##_ = DEFAULT;                                                                                \
                                                                                \
 public:                                                                                \
  auto &NAME() { return NAME##_; }                                                                                \
  const auto &NAME() const { return NAME##_; }                                                                     \
                                                                                \
 protected:                                                                                \
  constexpr auto T##NAME()->::hf3fs::thief::steal<struct T##NAME, std::decay_t<decltype(*this)>>;                  \
  REFL_ADD_SAFE(                                                                                \
      (::hf3fs::serde::FieldInfo<#NAME,                                                                            \
                                 &::hf3fs::thief::retrieve<struct T##NAME>::NAME##_ __VA_OPT__(, __VA_ARGS__)>{}), \
      typename T##NAME::type)

#define SERDE_STRUCT_TYPED_FIELD(TYPE, NAME, DEFAULT, ...)                                                             \
 private:                                                                                \
  friend struct ::hf3fs::refl::Helper;                                                                                \
  struct T##NAME : std::type_identity<REFL_NOW> {};                                                                    \
                                                                                \
 public:                                                                                \
  TYPE NAME = DEFAULT;                                                                                \
                                                                                \
 protected:                                                                                \
  constexpr auto T##NAME()->::hf3fs::thief::steal<struct T##NAME, std::decay_t<decltype(*this)>>;                      \
  REFL_ADD_SAFE(                                                                                \
      (::hf3fs::serde::FieldInfo<#NAME, &::hf3fs::thief::retrieve<struct T##NAME>::NAME __VA_OPT__(, __VA_ARGS__)>{}), \
      typename T##NAME::type)

#define SERDE_CLASS_FIELD(NAME, DEFAULT, ...) \
  SERDE_CLASS_TYPED_FIELD(std::decay_t<decltype(DEFAULT)>, NAME, DEFAULT __VA_OPT__(, __VA_ARGS__))

#define SERDE_STRUCT_FIELD(NAME, DEFAULT, ...) \
  SERDE_STRUCT_TYPED_FIELD(std::decay_t<decltype(DEFAULT)>, NAME, DEFAULT __VA_OPT__(, __VA_ARGS__))

namespace hf3fs::serde {

template <NameWrapper Name, auto Getter, auto Checker = nullptr>
struct FieldInfo {
  static constexpr std::string_view name = Name;
  static constexpr auto getter = Getter;
  static constexpr auto checker = Checker;
};

template <auto... Args>
constexpr inline std::true_type is_field_info(FieldInfo<Args...>);
template <class T>
constexpr inline bool is_field_infos = false;
template <class... Ts>
constexpr inline bool is_field_infos<std::tuple<Ts...>> = (requires { is_field_info(std::declval<Ts>()); } && ...);

template <typename T>
concept SerdeType = bool(refl::Helper::Size<T>) && is_field_infos<refl::Helper::FieldInfoList<T>>;

template <typename T>
concept SerdeCopyable =
    std::is_same_v<T, bool> || std::is_integral_v<T> || std::is_floating_point_v<T> || std::is_enum_v<T> || requires {
  typename T::is_serde_copyable;
};

template <SerdeType T>
constexpr inline auto count() {
  return refl::Helper::Size<T>;
}
template <SerdeType T, size_t Index>
constexpr inline auto name() {
  return refl::Helper::FieldInfo<T, Index>::name;
}
template <SerdeType T, size_t Index>
constexpr inline auto getter() {
  return refl::Helper::FieldInfo<T, Index>::getter;
}
constexpr inline auto count(auto &&o) { return count<std::decay_t<decltype(o)>>(); }
template <size_t Index>
constexpr inline auto name(auto &&o) {
  return name<std::decay_t<decltype(o)>, Index>();
}
template <size_t Index>
constexpr inline auto &value(auto &&o) {
  return o.*getter<std::decay_t<decltype(o)>, Index>();
}

template <size_t I = 0>
constexpr inline auto iterate(auto &&f, auto &&o, auto &&...args) {
  return refl::Helper::iterate<std::decay_t<decltype(o)>>(
      [&](auto type) { return f(type.name, o.*type.getter, args...); });
}

template <class T>
struct SerdeMethod {
  static auto serialize(const T &o, auto &out) = delete;
  static auto serializeReadable(const T &o, auto &out) = delete;
  static auto deserialize(T &o, auto &in) = delete;
  static Result<T> deserializeReadable(T &o, auto &out) = delete;

  static auto serdeTo(const T &t) = delete;
  static auto serdeToReadable(const T &o) = delete;
  static Result<T> serdeFrom(auto) = delete;
  static Result<T> serdeFromReadable(auto) = delete;
};

template <typename T>
struct DefaultConstructor {
  static T construct() { return T{}; }
};

template <>
struct DefaultConstructor<Status> {
  static Status construct() { return Status::OK; }
};

template <typename T>
struct DefaultConstructor<Result<T>> {
  static Result<T> construct() { return T{}; }
};
}  // namespace hf3fs::serde

template <>
struct hf3fs::serde::SerdeMethod<hf3fs::Varint32> {
  static auto serialize(const Varint32 &o, auto &out) {
    constexpr int B = 128;
    uint8_t buf[5];
    auto ptr = buf;
    if (o < (1 << 7)) {
      *(ptr++) = o;
    } else if (o < (1 << 14)) {
      *(ptr++) = o | B;
      *(ptr++) = o >> 7;
    } else if (o < (1 << 21)) {
      *(ptr++) = o | B;
      *(ptr++) = (o >> 7) | B;
      *(ptr++) = o >> 14;
    } else if (o < (1 << 28)) {
      *(ptr++) = o | B;
      *(ptr++) = (o >> 7) | B;
      *(ptr++) = (o >> 14) | B;
      *(ptr++) = o >> 21;
    } else {
      *(ptr++) = o | B;
      *(ptr++) = (o >> 7) | B;
      *(ptr++) = (o >> 14) | B;
      *(ptr++) = (o >> 21) | B;
      *(ptr++) = o >> 28;
    }
    out.append(buf, ptr - buf);
  }

  static Result<Void> deserialize(Varint32 &o, auto &in) {
    o.value = 0;
    for (uint32_t shift = 0; shift <= 28 && !in.str().empty(); shift += 7) {
      uint32_t byte = static_cast<uint8_t>(in.str()[0]);
      in.str().remove_prefix(1);
      if (byte & 128) {
        // More bytes are present
        o.value |= ((byte & 127) << shift);
      } else {
        o.value |= (byte << shift);
        return Void{};
      }
    }
    return makeError(StatusCode::kSerdeInsufficientLength, fmt::format("varint32 is short"));
  }

  static uint32_t serdeToReadable(Varint32 o) { return o; }
};

template <>
struct hf3fs::serde::SerdeMethod<hf3fs::Varint64> {
  static auto serialize(const Varint64 &o, auto &out) {
    static const int B = 128;
    uint8_t buf[10];
    auto ptr = reinterpret_cast<uint8_t *>(buf);

    uint64_t v = o;
    while (v >= B) {
      *(ptr++) = o | B;
      v >>= 7;
    }
    *(ptr++) = static_cast<uint8_t>(v);
    out.append(buf, ptr - buf);
  }

  static Result<Void> deserialize(Varint64 &o, auto &in) {
    o.value = 0;
    for (uint32_t shift = 0; shift <= 63 && !in.str().empty(); shift += 7) {
      uint64_t byte = static_cast<uint8_t>(in.str()[0]);
      in.str().remove_prefix(1);
      if (byte & 128) {
        // More bytes are present
        o.value |= ((byte & 127) << shift);
      } else {
        o.value |= (byte << shift);
        return Void{};
      }
    }
    return makeError(StatusCode::kSerdeInsufficientLength, fmt::format("varint64 is short"));
  }

  static uint64_t serdeToReadable(Varint64 o) { return o; }
};

namespace hf3fs::serde {

enum class Optional : uint8_t { NullOpt, HasValue };
struct VariantIndex {
  uint8_t index;
};

template <class T>
class Out {
 public:
  auto tableBegin(bool isInline);  // begin a table.
  void tableEnd(auto);             // end a table.
  void arrayBegin();               // begin an array.
  void arrayEnd(uint32_t size);    // end an array.
  void key(std::string_view key);  // prepare the key in a key-value pair.
  void value(auto &&value);        // prepare the value in a key-value pair or an array.
};

template <class T>
class In {
 public:
  auto parseKey(std::string_view);
  auto parseTable();

  // for binary.
  Result<Void> parseCopyable(auto &o);

  // for Json/Toml.
  Result<bool> parseBoolean();
  Result<int64_t> parseInteger();
  Result<double> parseFloat();

  // for both.
  Result<std::string_view> parseString();
  Result<Void> parseOptional(Optional &optional);
  Result<std::pair<std::string_view, In>> parseVariant();
};

template <class O>
inline void serialize(auto &&o, Out<O> &out) {
  using T = std::decay_t<decltype(o)>;
  constexpr bool isBinaryOut = requires { typename Out<O>::is_binary_out; };
  if constexpr (requires { o.serdeToReadable(); } && !isBinaryOut) {
    serialize(o.serdeToReadable(), out);
  } else if constexpr (requires { SerdeMethod<T>::serializeReadable(o, out); } && !isBinaryOut) {
    SerdeMethod<T>::serializeReadable(o, out);
  } else if constexpr (requires { SerdeMethod<T>::serdeToReadable(o); } && !isBinaryOut) {
    serialize(SerdeMethod<T>::serdeToReadable(o), out);
  } else if constexpr (requires { SerdeMethod<T>::serialize(o, out); }) {
    static_assert(requires(T t, In<std::string_view> in) { Result<Void>{SerdeMethod<T>::deserialize(t, in)}; });
    SerdeMethod<T>::serialize(o, out);
  } else if constexpr (requires { SerdeMethod<T>::serdeTo(o); }) {
    static_assert(requires { SerdeMethod<T>::serdeFrom(SerdeMethod<T>::serdeTo(o)); });
    serialize(SerdeMethod<T>::serdeTo(o), out);
  } else if constexpr (SerdeType<T>) {
    auto start = out.tableBegin(false);
    if constexpr (isBinaryOut) {
      refl::Helper::iterate<T, true>([&](auto type) { serialize(o.*type.getter, out); },
                                     [&] { out.tableEnd(start), start = out.tableBegin(false); });
    } else {
      refl::Helper::iterate<T>([&](auto type) { out.key(type.name), serialize(o.*type.getter, out); });
    }
    out.tableEnd(start);
  } else if constexpr (std::is_same_v<T, std::string>) {
    out.value(o);
  } else if constexpr (std::is_convertible_v<T, std::string_view>) {
    out.value(std::string_view(o));
  } else if constexpr (is_optional_v<T>) {
    if (o.has_value()) {
      serialize(o.value(), out);
      out.value(Optional::HasValue);
    } else {
      out.value(Optional::NullOpt);
    }
  } else if constexpr (is_unique_ptr_v<T> || is_shared_ptr_v<T>) {
    if (o) {
      serialize(*o, out);
      out.value(Optional::HasValue);
    } else {
      out.value(Optional::NullOpt);
    }
  } else if constexpr (is_variant_v<T> && isBinaryOut) {
    static_assert(std::variant_size_v<T> <= std::numeric_limits<uint8_t>::max());
    std::visit(
        [&out](auto &&v) {
          serialize(v, out);
          serialize(type_name_v<std::decay_t<decltype(v)>>, out);
        },
        o);
  } else if constexpr (is_variant_v<T>) {
    static_assert(std::variant_size_v<T> <= std::numeric_limits<uint8_t>::max());
    auto start = out.tableBegin(true);
    out.key("type");
    std::visit(
        [&out](auto &&v) {
          serialize(type_name_v<std::decay_t<decltype(v)>>, out);
          out.key("value");
          serialize(v, out);
        },
        o);
    out.tableEnd(start);
  } else if constexpr (is_generic_pair_v<T> && isBinaryOut) {
    serialize(o.second, out);
    serialize(o.first, out);
  } else if constexpr (Container<T> && isBinaryOut) {
    out.arrayBegin();
    if constexpr (requires { o.rbegin(); }) {
      for (auto it = o.rbegin(); it != o.rend(); ++it) {
        serialize(*it, out);
      }
    } else {
      for (auto &item : o) {
        serialize(item, out);
      }
    }
    out.arrayEnd(o.size());
  } else if constexpr (is_vector_v<T> && std::is_arithmetic_v<T> && isBinaryOut) {
    out.value(o);
  } else if constexpr (is_vector_v<T> || is_set_v<T>) {
    out.arrayBegin();
    for (const auto &item : o) {
      serialize(item, out);
    }
    out.arrayEnd(o.size());
  } else if constexpr (is_map_v<T>) {
    auto start = out.tableBegin(true);
    for (const auto &pair : o) {
      if constexpr (requires { out.key(pair.first); }) {
        out.key(pair.first);
      } else if constexpr (requires { pair.first.toUnderType(); }) {
        out.key(fmt::format("{}", pair.first.toUnderType()));
      } else {
        out.key(fmt::format("{}", pair.first));
      }
      serialize(pair.second, out);
    }
    out.tableEnd(start);
  } else if constexpr (isBinaryOut && SerdeCopyable<T>) {
    out.value(o);
  } else if constexpr (std::is_same_v<T, bool>) {
    out.value(o);
  } else if constexpr (std::is_integral_v<T>) {
    out.value(int64_t(o));
  } else if constexpr (std::is_floating_point_v<T>) {
    out.value(double(o));
  } else if constexpr (std::is_enum_v<T>) {
    out.value(magic_enum::enum_name(o));
  } else {
    return notSupportToSerialize(o);
  }
}

struct JsonObject;
struct TomlObject;

template <class T>
concept IsJsonOrToml = std::is_same_v<T, JsonObject> || std::is_same_v<T, TomlObject>;
template <class T>
concept IsBinaryOut = std::is_same_v<T, std::string> || is_specialization_of_v<T, DownwardBytes>;

template <IsJsonOrToml T>
class Out<T> {
 public:
  Out();
  ~Out();
  int tableBegin(bool);
  void tableEnd(auto) { end(); }
  void arrayBegin();
  void arrayEnd(uint32_t) { end(); }
  void end();

  void key(std::string_view key) { add(key); }
  void value(bool value) { add(value); }
  void value(int64_t value) { add(value); }
  void value(double value) { add(value); }
  void value(std::string &&str) { add(std::move(str)); }
  void value(std::string_view str) { add(str); }
  void value(Optional);
  void value(VariantIndex) {}

  std::string toString(bool sortKeys = false, bool prettyFormatting = false);

 protected:
  void add(auto &&v);
  bool inTable();
  bool inArray();

 private:
  std::vector<T> root_;
};

template <IsBinaryOut T>
class Out<T> {
 public:
  using is_binary_out = void;

  uint32_t tableBegin(bool) { return out_.size(); }
  void tableEnd(uint32_t start) { serde::serialize(Varint32(out_.size() - start), *this); }
  void arrayBegin() {}
  void arrayEnd(uint32_t size) { serde::serialize(Varint32(size), *this); }

  inline void key(std::string_view) {}  // ignore key.
  void value(auto &&value) requires(std::is_trivially_copyable_v<std::decay_t<decltype(value)>>) {
    append(&value, sizeof(value));
  }
  void value(std::string_view str) {
    append(str.data(), str.size());
    serde::serialize(Varint32(str.size()), *this);
  }
  template <class V>
  requires(std::is_arithmetic_v<V>) void value(const std::vector<V> &vec) {
    append(vec.data(), vec.size() * sizeof(V));
    serde::serialize(Varint32(vec.size()), *this);
  }

  auto &bytes() { return out_; }

  inline void append(auto *data, size_t size) { out_.append(reinterpret_cast<const char *>(data), size); }

 private:
  T out_;
};

struct UnknownVariantType {
  SERDE_STRUCT_FIELD(type, String{});
  // TODO: add serializedBytes
};

template <typename... Ts>
using AutoFallbackVariant = std::variant<UnknownVariantType, Ts...>;

template <typename T>
constexpr bool is_auto_fallback_variant_v = false;

template <typename... Ts>
constexpr bool is_auto_fallback_variant_v<AutoFallbackVariant<Ts...>> = true;

inline Result<Void> deserialize(auto &o, auto &&in) requires is_specialization<std::decay_t<decltype(in)>, In> {
  using T = std::decay_t<decltype(o)>;
  using I = std::decay_t<decltype(in)>;
  constexpr bool isBinaryIn = requires { typename I::is_binary_in; };
  if constexpr (!isBinaryIn && requires { function_first_parameter_t<&T::serdeFromReadable>{}; }) {
    // 0. custom serde impl.
    function_first_parameter_t<&T::serdeFromReadable> from{};
    RETURN_AND_LOG_ON_ERROR(deserialize(from, in));
    auto result = T::serdeFromReadable(from);
    RETURN_AND_LOG_ON_ERROR(result);
    o = std::move(*result);
    return Void{};
  } else if constexpr (!isBinaryIn && requires { function_first_parameter_t<&SerdeMethod<T>::serdeFromReadable>{}; }) {
    function_first_parameter_t<&SerdeMethod<T>::serdeFromReadable> from{};
    RETURN_AND_LOG_ON_ERROR(deserialize(from, in));
    auto result = SerdeMethod<T>::serdeFromReadable(from);
    RETURN_AND_LOG_ON_ERROR(result);
    o = std::move(*result);
    return Void{};
  } else if constexpr (requires { function_first_parameter_t<&SerdeMethod<T>::serdeFrom>{}; }) {
    // 0. custom serde impl.
    function_first_parameter_t<&SerdeMethod<T>::serdeFrom> from{};
    RETURN_AND_LOG_ON_ERROR(deserialize(from, in));
    auto result = SerdeMethod<T>::serdeFrom(from);
    RETURN_AND_LOG_ON_ERROR(result);
    o = std::move(*result);
    return Void{};
  } else if constexpr (!isBinaryIn && requires { SerdeMethod<T>::deserializeReadable(o, in); }) {
    return SerdeMethod<T>::deserializeReadable(o, in);
  } else if constexpr (requires { SerdeMethod<T>::deserialize(o, in); }) {
    return SerdeMethod<T>::deserialize(o, in);
  } else if constexpr (SerdeType<T>) {
    auto table = in.parseTable();
    RETURN_AND_LOG_ON_ERROR(table);
    if constexpr (isBinaryIn) {
      return refl::Helper::iterate<T>(
          [&](auto type) -> Result<Void> {
            if (LIKELY(*table)) {
              return deserialize(o.*type.getter, *table);
            }
            // Missing fields at the end are acceptable.
            return Void{};
          },
          [&]() -> Result<Void> {
            table = in.parseTable();
            RETURN_AND_LOG_ON_ERROR(table);
            return Void{};
          });
    } else {
      return refl::Helper::iterate<T>([&](auto type) -> Result<Void> {
        auto value = table->parseKey(type.name);
        if (LIKELY(bool(value))) {
          return deserialize(o.*type.getter, *value);
        } else {
          using ItemType = std::decay_t<decltype(o.*type.getter)>;
          if constexpr (is_optional_v<ItemType>) {
            o.*type.getter = std::nullopt;
          } else if constexpr (is_unique_ptr_v<ItemType> || is_shared_ptr_v<ItemType>) {
            o.*type.getter = nullptr;
          }
        }
        return Void{};
      });
    }
  } else if constexpr (std::is_same_v<std::string, T> || std::is_same_v<std::string_view, T>) {
    auto result = in.parseString();
    RETURN_AND_LOG_ON_ERROR(result);
    o = *result;
    return Void{};
  } else if constexpr (is_optional_v<T>) {
    Optional optional;
    RETURN_AND_LOG_ON_ERROR(in.parseOptional(optional));
    if (optional == Optional::HasValue) {
      std::remove_cv_t<typename T::value_type> value;
      RETURN_AND_LOG_ON_ERROR(deserialize(value, in));
      o = std::move(value);
    } else {
      o = std::nullopt;
    }
    return Void{};
  } else if constexpr (is_unique_ptr_v<T> || is_shared_ptr_v<T>) {
    Optional optional;
    RETURN_AND_LOG_ON_ERROR(in.parseOptional(optional));
    if (optional == Optional::HasValue) {
      if constexpr (is_unique_ptr_v<T>) {
        o = std::make_unique<typename T::element_type>();
      } else {
        o = std::make_shared<typename T::element_type>();
      }
      RETURN_AND_LOG_ON_ERROR(deserialize(*o, in));
    } else {
      o = nullptr;
    }
    return Void{};
  } else if constexpr (is_variant_v<T>) {
    auto variant = in.parseVariant();
    RETURN_AND_LOG_ON_ERROR(variant);
    return callByIdx<type_list_t<T>>(
        [&](auto type) -> Result<Void> {
          if constexpr (std::is_same_v<decltype(type), std::nullptr_t>) {
            if constexpr (is_auto_fallback_variant_v<T>) {
              UnknownVariantType uvt;
              uvt.type = variant->first;
              o = std::move(uvt);
              return Void{};
            } else {
              return makeError(StatusCode::kSerdeVariantIndexExceeded);
            }
          } else {
            RETURN_AND_LOG_ON_ERROR(deserialize(type, variant->second));
            o = std::move(type);
            return Void{};
          }
        },
        variantTypeNameToIndex<T>(variant->first));
    return Void{};
  } else if constexpr (is_generic_pair_v<T> && isBinaryIn) {
    RETURN_AND_LOG_ON_ERROR(deserialize(o.first, in));
    RETURN_AND_LOG_ON_ERROR(deserialize(o.second, in));
    return Void{};
  } else if constexpr (Container<T> && isBinaryIn) {
    Varint64 size = 0;
    RETURN_AND_LOG_ON_ERROR(deserialize(size, in));
    o.clear();
    if constexpr (requires { o.reserve(size); }) {
      o.reserve(size);
    }
    auto inserter = std::inserter(o, o.end());
    for (uint32_t i = 0; i < size; ++i) {
      if constexpr (is_generic_pair_v<typename T::value_type>) {
        std::remove_cv_t<typename T::value_type::first_type> first;
        RETURN_AND_LOG_ON_ERROR(deserialize(first, in));
        std::remove_cv_t<typename T::value_type::second_type> second;
        RETURN_AND_LOG_ON_ERROR(deserialize(second, in));
        inserter++ = typename T::value_type{std::move(first), std::move(second)};
      } else {
        auto value = DefaultConstructor<std::remove_cv_t<typename T::value_type>>::construct();
        RETURN_AND_LOG_ON_ERROR(deserialize(value, in));
        inserter++ = std::move(value);
      }
    }
    return Void{};
  } else if constexpr (is_vector_v<T> || is_set_v<T> || is_map_v<T>) {
    auto containerResult = in.parseContainer();
    RETURN_AND_LOG_ON_ERROR(containerResult);
    auto size = containerResult->first;
    auto it = std::move(containerResult->second);
    o.clear();
    if constexpr (requires { o.reserve(size); }) {
      o.reserve(size);
    }
    auto inserter = std::inserter(o, o.end());
    for (uint32_t i = 0; i < size; ++i) {
      if constexpr (is_map_v<T>) {
        auto keyResult = in.fetchKey(it);
        RETURN_AND_LOG_ON_ERROR(keyResult);
        std::string_view key = *keyResult;
        using KeyType = std::remove_cv_t<typename T::value_type::first_type>;
        KeyType first;
        if constexpr (requires { first = KeyType{key}; }) {
          first = KeyType{key};
        } else if constexpr (requires { scn::scan(key, "{}", first); }) {
          auto result = scn::scan(key, "{}", first);
          if (!result) {
            return makeError(StatusCode::kInvalidArg);
          }
        } else if constexpr (requires { scn::scan(key, "{}", first.toUnderType()); }) {
          auto result = scn::scan(key, "{}", first.toUnderType());
          if (!result) {
            return makeError(StatusCode::kInvalidArg);
          }
        } else {
          return makeError(StatusCode::kInvalidArg);
        }
        std::remove_cv_t<typename T::value_type::second_type> second;
        RETURN_AND_LOG_ON_ERROR(deserialize(second, in.fetchValue(it)));
        inserter++ = typename T::value_type{std::move(first), std::move(second)};
        in.next(it);
      } else {
        std::remove_cv_t<typename T::value_type> value;
        RETURN_AND_LOG_ON_ERROR(deserialize(value, in.fetchAndNext(it)));
        inserter++ = std::move(value);
      }
    }
    return Void{};
  } else if constexpr (isBinaryIn && SerdeCopyable<T>) {
    return in.parseCopyable(o);
  } else if constexpr (std::is_same_v<T, bool>) {
    auto result = in.parseBoolean();
    RETURN_AND_LOG_ON_ERROR(result);
    o = *result;
    return Void{};
  } else if constexpr (std::is_integral_v<T>) {
    auto result = in.parseInteger();
    RETURN_AND_LOG_ON_ERROR(result);
    o = *result;
    return Void{};
  } else if constexpr (std::is_floating_point_v<T>) {
    auto result = in.parseFloat();
    RETURN_AND_LOG_ON_ERROR(result);
    o = *result;
    return Void{};
  } else if constexpr (std::is_enum_v<T>) {
    std::string_view str;
    RETURN_AND_LOG_ON_ERROR(deserialize(str, in));
    auto result = magic_enum::enum_cast<T>(str);
    if (result.has_value()) {
      o = *result;
      return Void{};
    }
    return makeError(StatusCode::kSerdeUnknownEnumValue,
                     fmt::format("unknown enum value {} for type {}", str, nameof::nameof_full_type<T>()));
  } else {
    return notSupportToDeserialize(o, in);
  }
}

template <>
class In<std::string_view> {
 public:
  using is_binary_in = void;

  In(std::string_view str)
      : str_(str) {}

  operator bool() const { return !str_.empty(); }

  Result<In> parseTable() {
    Varint64 length;
    RETURN_AND_LOG_ON_ERROR(serde::deserialize(length, *this));
    if (UNLIKELY(length > str_.length())) {
      return makeError(StatusCode::kSerdeInsufficientLength,
                       fmt::format("string short {} > {}", length, str_.length()));
    }
    auto table = In{str_.substr(0, length)};
    str_.remove_prefix(length);
    return table;
  }

  Result<std::string_view> parseString() {
    Varint64 length;
    RETURN_AND_LOG_ON_ERROR(serde::deserialize(length, *this));
    if (UNLIKELY(length > str_.length())) {
      return makeError(StatusCode::kSerdeInsufficientLength,
                       fmt::format("string short {} > {}", length, str_.length()));
    }
    auto value = str_.substr(0, length);
    str_.remove_prefix(length);
    return value;
  }

  Result<std::pair<std::string_view, In &>> parseVariant() {
    std::string_view typeName;
    RETURN_AND_LOG_ON_ERROR(serde::deserialize(typeName, *this));
    return std::pair<std::string_view, In &>(typeName, *this);
  }

  Result<Void> parseOptional(Optional &optional) { return serde::deserialize(optional, *this); }

  Result<Void> parseCopyable(auto &o) {
    if (UNLIKELY(sizeof(o) > str_.length())) {
      return makeError(StatusCode::kSerdeInsufficientLength,
                       fmt::format("trivially copyable {} > {}", sizeof(o), str_.length()));
    }
    std::memcpy(&o, str_.data(), sizeof(o));
    str_.remove_prefix(sizeof(o));
    return Void{};
  }

  auto &str() { return str_; }

 private:
  std::string_view str_;
};

template <>
class In<JsonObject> {
 public:
  In(const JsonObject &obj)
      : obj_(obj) {}
  template <class T>
  In(const T &obj);
  static Result<Void> parse(std::string_view str, std::function<Result<Void>(const JsonObject &)> func);

  Result<In> parseKey(std::string_view key) const;
  Result<In> parseTable() const;

  Result<bool> parseBoolean() const;
  Result<int64_t> parseInteger() const;
  Result<double> parseFloat() const;

  Result<std::string_view> parseString() const;
  Result<std::pair<std::string_view, In>> parseVariant() const;

  Result<Void> parseOptional(Optional &optional) const;

  Result<std::pair<uint32_t, std::unique_ptr<void, void (*)(void *)>>> parseContainer() const;

  In fetchAndNext(std::unique_ptr<void, void (*)(void *)> &it) const;
  Result<std::string_view> fetchKey(std::unique_ptr<void, void (*)(void *)> &it) const;
  In fetchValue(std::unique_ptr<void, void (*)(void *)> &it) const;
  void next(std::unique_ptr<void, void (*)(void *)> &it) const;

 private:
  const JsonObject &obj_;
};

template <>
class In<TomlObject> {
 public:
  In(const TomlObject &obj)
      : obj_(obj) {}
  template <class T>
  In(const T &obj);
  static Result<Void> parse(std::string_view str, std::function<Result<Void>(const TomlObject &)> func);
  static Result<Void> parseFile(const Path &path, std::function<Result<Void>(const TomlObject &)> func);

  Result<In> parseKey(std::string_view key) const;
  Result<In> parseTable() const;

  Result<bool> parseBoolean() const;
  Result<int64_t> parseInteger() const;
  Result<double> parseFloat() const;

  Result<std::string_view> parseString() const;
  Result<std::pair<std::string_view, In>> parseVariant() const;

  Result<Void> parseOptional(Optional &optional) const;

  Result<std::pair<uint32_t, std::unique_ptr<void, void (*)(void *)>>> parseContainer() const;

  In fetchAndNext(std::unique_ptr<void, void (*)(void *)> &it) const;
  Result<std::string_view> fetchKey(std::unique_ptr<void, void (*)(void *)> &it) const;
  In fetchValue(std::unique_ptr<void, void (*)(void *)> &it) const;
  void next(std::unique_ptr<void, void (*)(void *)> &it) const;

 private:
  const TomlObject &obj_;
};

inline std::string serialize(const auto &o) {
  Out<DownwardBytes<net::Allocator<>>> out;
  serialize(o, out);
  return out.bytes().toString();
}

inline DownwardBytes<net::Allocator<>> serializeBytes(const auto &o) {
  Out<DownwardBytes<net::Allocator<>>> out;
  serialize(o, out);
  return std::move(out.bytes());
}

inline void serializeToUserBuffer(const auto &o, uint8_t *data, uint32_t capacity) {
  Out<DownwardBytes<UserBufferAllocator>> out;
  out.bytes().setBuffer(data, capacity);
  serialize(o, out);
}

inline auto serializeLength(const auto &o) {
  Out<DownwardBytes<void>> out;
  serialize(o, out);
  return out.bytes().size();
}

inline Result<Void> deserialize(auto &o, std::string_view str) {
  serde::In<std::string_view> in(str);
  return deserialize(o, in);
}

inline std::string toTomlString(const auto &o) {
  Out<TomlObject> out;
  serialize(o, out);
  return out.toString();
}

inline std::string toJsonString(const auto &o, bool sortKeys = false, bool prettyFormatting = false) {
  Out<JsonObject> out;
  serialize(o, out);
  return out.toString(sortKeys, prettyFormatting);
}

inline Result<Void> fromJsonString(auto &o, std::string_view str) {
  return In<JsonObject>::parse(str, [&](const JsonObject &obj) { return deserialize(o, In<JsonObject>(obj)); });
}

inline Result<Void> fromTomlString(auto &o, std::string_view str) {
  return In<TomlObject>::parse(str, [&](const TomlObject &obj) { return deserialize(o, In<TomlObject>(obj)); });
}

inline Result<Void> fromTomlFile(auto &o, const Path &path) {
  return In<TomlObject>::parseFile(path, [&](const TomlObject &obj) { return deserialize(o, In<TomlObject>(obj)); });
}

template <typename T>
concept SerializableToBytes = requires(const T &o) {
  serialize(o);
};

template <typename T>
concept SerializableToToml = requires(const T &o) {
  toTomlString(o);
};

template <typename T>
concept SerializableToJson = requires(const T &o) {
  toJsonString(o);
};

template <typename T>
concept Serializable = requires(const T &o) {
  serialize(o);
  toJsonString(o);
};

}  // namespace hf3fs::serde

template <>
struct ::hf3fs::serde::SerdeMethod<::hf3fs::Void> {
  static constexpr auto serialize(Void, auto &&) {}                 // DO NOTHING.
  static Result<Void> deserialize(Void, auto &) { return Void{}; }  // DO NOTHING.
  static constexpr std::string_view serdeToReadable(Void) { return "Void"; };
};

template <>
struct ::hf3fs::serde::SerdeMethod<::hf3fs::Status> {
  static constexpr auto serialize(const Status &status, auto &out) {
    std::optional<std::string_view> msg = std::nullopt;
    if (!status.message().empty()) {
      msg = status.message();
    }
    serde::serialize(msg, out);
    serde::serialize(status.code(), out);
  }

  static Result<Void> deserialize(Status &status, auto &&in) {
    status_code_t code;
    RETURN_AND_LOG_ON_ERROR(serde::deserialize(code, in));
    std::optional<std::string_view> msg;
    RETURN_AND_LOG_ON_ERROR(serde::deserialize(msg, in));
    if (msg.has_value()) {
      status = Status(code, msg.value());
    } else {
      status = Status(code);
    }
    return Void{};
  }

  struct InnerStatus {
    SERDE_STRUCT_FIELD(code, uint16_t{});
    SERDE_STRUCT_FIELD(msg, std::string{});
  };

  static auto serdeToReadable(const Status &status) {
    return InnerStatus{status.code(), std::string(status.message())};
  }

  static Result<Status> serdeFromReadable(const InnerStatus &inner) {
    if (inner.msg.empty()) {
      return folly::makeExpected<Status>(Status{inner.code});
    } else {
      return folly::makeExpected<Status>(Status{inner.code, inner.msg});
    }
  }
};

template <class T>
struct ::hf3fs::serde::SerdeMethod<::hf3fs::Result<T>> {
  static auto serialize(const ::hf3fs::Result<T> &result, auto &out) {
    bool hasValue = result.hasValue();
    if (hasValue) {
      serde::serialize(result.value(), out);
    } else {
      serde::serialize(result.error(), out);
    }
    out.value(hasValue);
  }

  static auto serializeReadable(const ::hf3fs::Result<T> &result, auto &out) {
    auto start = out.tableBegin(false);
    {
      if (result.hasValue()) {
        out.key("value");
        serde::serialize(result.value(), out);
      } else {
        out.key("error");
        serde::serialize(result.error(), out);
      }
    }
    out.tableEnd(start);
  }

  static Result<Void> deserialize(::hf3fs::Result<T> &result, auto &&in) {
    bool hasValue = false;
    RETURN_AND_LOG_ON_ERROR(serde::deserialize(hasValue, in));
    if (hasValue) {
      T value;
      RETURN_AND_LOG_ON_ERROR(serde::deserialize(value, in));
      result = std::move(value);
    } else {
      Status status = Status::OK;
      RETURN_AND_LOG_ON_ERROR(serde::deserialize(status, in));
      result = makeError(std::move(status));
    }
    return Void{};
  }
};

template <>
struct hf3fs::serde::SerdeMethod<hf3fs::Path> {
  static auto serdeTo(const hf3fs::Path &p) { return p.string(); }
  static Result<hf3fs::Path> serdeFrom(std::string_view str) { return hf3fs::Path{std::string{str}}; }
};

FMT_BEGIN_NAMESPACE

template <::hf3fs::serde::SerdeType T>
requires(::hf3fs::serde::SerializableToJson<T>) struct formatter<T> : formatter<std::string_view> {
  detail::dynamic_format_specs<char> specs_;
  /* Copy from https://fmt.dev/latest/api.html#formatting-user-defined-types */
  template <typename ParseContext>
  constexpr auto parse(ParseContext &ctx) -> decltype(ctx.begin()) {
    auto end = detail::parse_format_specs(ctx.begin(), ctx.end(), specs_, ctx, detail::type::char_type);
    detail::check_char_specs(specs_);
    return end;
  }

  template <typename FormatContext>
  auto format(const T &t, FormatContext &ctx) const {
    bool debug = specs_.type == presentation_type::debug;
    return formatter<std::string_view>::format(
        hf3fs::serde::toJsonString(t, debug /*sortKeys*/, debug /*prettyFormatting*/),
        ctx);
  }
};

FMT_END_NAMESPACE

//                 ______  _____                 ______                _________
//  ______________ ___  /_ ___(_)_______         ___  /_ ______ ______ ______  /
//  __  ___/_  __ \__  __ \__  / __  __ \        __  __ \_  __ \_  __ \_  __  /
//  _  /    / /_/ /_  /_/ /_  /  _  / / /        _  / / // /_/ // /_/ // /_/ /
//  /_/     \____/ /_.___/ /_/   /_/ /_/ ________/_/ /_/ \____/ \____/ \__,_/
//                                      _/_____/
//
// Fast & memory efficient hashtable based on robin hood hashing for C++11/14/17/20
// https://github.com/martinus/robin-hood-hashing
//
// Licensed under the MIT License <http://opensource.org/licenses/MIT>.
// SPDX-License-Identifier: MIT
// Copyright (c) 2018-2021 Martin Ankerl <http://martin.ankerl.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

#ifndef ROBIN_HOOD_H_INCLUDED
#define ROBIN_HOOD_H_INCLUDED

// see https://semver.org/
#define ROBIN_HOOD_VERSION_MAJOR 3   // for incompatible API changes
#define ROBIN_HOOD_VERSION_MINOR 11  // for adding functionality in a backwards-compatible manner
#define ROBIN_HOOD_VERSION_PATCH 5   // for backwards-compatible bug fixes

#include <algorithm>
#include <cstdlib>
#include <cstring>
#include <functional>
#include <limits>
#include <memory>  // only to support hash of smart pointers
#include <stdexcept>
#include <string>
#include <type_traits>
#include <utility>

#include "memory/common/GlobalMemoryAllocator.h"

#if __cplusplus >= 201703L
#include <string_view>
#endif

// #define ROBIN_HOOD_LOG_ENABLED
#ifdef ROBIN_HOOD_LOG_ENABLED
#include <iostream>
#define ROBIN_HOOD_LOG(...) std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << __VA_ARGS__ << std::endl;
#else
#define ROBIN_HOOD_LOG(x)
#endif

// #define ROBIN_HOOD_TRACE_ENABLED
#ifdef ROBIN_HOOD_TRACE_ENABLED
#include <iostream>
#define ROBIN_HOOD_TRACE(...) std::cout << __FUNCTION__ << "@" << __LINE__ << ": " << __VA_ARGS__ << std::endl;
#else
#define ROBIN_HOOD_TRACE(x)
#endif

// #define ROBIN_HOOD_COUNT_ENABLED
#ifdef ROBIN_HOOD_COUNT_ENABLED
#include <iostream>
#define ROBIN_HOOD_COUNT(x) ++counts().x;
namespace robin_hood {
struct Counts {
  uint64_t shiftUp{};
  uint64_t shiftDown{};
};
inline std::ostream &operator<<(std::ostream &os, Counts const &c) {
  return os << c.shiftUp << " shiftUp" << std::endl << c.shiftDown << " shiftDown" << std::endl;
}

static Counts &counts() {
  static Counts counts{};
  return counts;
}
}  // namespace robin_hood
#else
#define ROBIN_HOOD_COUNT(x)
#endif

// all non-argument macros should use this facility. See
// https://www.fluentcpp.com/2019/05/28/better-macros-better-flags/
#define ROBIN_HOOD(x) ROBIN_HOOD_PRIVATE_DEFINITION_##x()

// mark unused members with this macro
#define ROBIN_HOOD_UNUSED(identifier)

// bitness
#if SIZE_MAX == UINT32_MAX
#define ROBIN_HOOD_PRIVATE_DEFINITION_BITNESS() 32
#elif SIZE_MAX == UINT64_MAX
#define ROBIN_HOOD_PRIVATE_DEFINITION_BITNESS() 64
#else
#error Unsupported bitness
#endif

// endianess
#ifdef _MSC_VER
#define ROBIN_HOOD_PRIVATE_DEFINITION_LITTLE_ENDIAN() 1
#define ROBIN_HOOD_PRIVATE_DEFINITION_BIG_ENDIAN() 0
#else
#define ROBIN_HOOD_PRIVATE_DEFINITION_LITTLE_ENDIAN() (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
#define ROBIN_HOOD_PRIVATE_DEFINITION_BIG_ENDIAN() (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
#endif

// inline
#ifdef _MSC_VER
#define ROBIN_HOOD_PRIVATE_DEFINITION_NOINLINE() __declspec(noinline)
#else
#define ROBIN_HOOD_PRIVATE_DEFINITION_NOINLINE() __attribute__((noinline))
#endif

// exceptions
#if !defined(__cpp_exceptions) && !defined(__EXCEPTIONS) && !defined(_CPPUNWIND)
#define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_EXCEPTIONS() 0
#else
#define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_EXCEPTIONS() 1
#endif

// count leading/trailing bits
#if !defined(ROBIN_HOOD_DISABLE_INTRINSICS)
#ifdef _MSC_VER
#if ROBIN_HOOD(BITNESS) == 32
#define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward
#else
#define ROBIN_HOOD_PRIVATE_DEFINITION_BITSCANFORWARD() _BitScanForward64
#endif
#include <intrin.h>
#pragma intrinsic(ROBIN_HOOD(BITSCANFORWARD))
#define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x)                                                          \
  [](size_t mask) noexcept -> int {                                                                  \
    unsigned long index;                                                                             \
    return ROBIN_HOOD(BITSCANFORWARD)(&index, mask) ? static_cast<int>(index) : ROBIN_HOOD(BITNESS); \
  }(x)
#else
#if ROBIN_HOOD(BITNESS) == 32
#define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzl
#define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzl
#else
#define ROBIN_HOOD_PRIVATE_DEFINITION_CTZ() __builtin_ctzll
#define ROBIN_HOOD_PRIVATE_DEFINITION_CLZ() __builtin_clzll
#endif
#define ROBIN_HOOD_COUNT_LEADING_ZEROES(x) ((x) ? ROBIN_HOOD(CLZ)(x) : ROBIN_HOOD(BITNESS))
#define ROBIN_HOOD_COUNT_TRAILING_ZEROES(x) ((x) ? ROBIN_HOOD(CTZ)(x) : ROBIN_HOOD(BITNESS))
#endif
#endif

// fallthrough
#ifndef __has_cpp_attribute  // For backwards compatibility
#define __has_cpp_attribute(x) 0
#endif
#if __has_cpp_attribute(clang::fallthrough)
#define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() [[clang::fallthrough]]
#elif __has_cpp_attribute(gnu::fallthrough)
#define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH() [[gnu::fallthrough]]
#else
#define ROBIN_HOOD_PRIVATE_DEFINITION_FALLTHROUGH()
#endif

// likely/unlikely
#ifdef _MSC_VER
#define ROBIN_HOOD_LIKELY(condition) condition
#define ROBIN_HOOD_UNLIKELY(condition) condition
#else
#define ROBIN_HOOD_LIKELY(condition) __builtin_expect(condition, 1)
#define ROBIN_HOOD_UNLIKELY(condition) __builtin_expect(condition, 0)
#endif

// detect if native wchar_t type is availiable in MSVC
#ifdef _MSC_VER
#ifdef _NATIVE_WCHAR_T_DEFINED
#define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 1
#else
#define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 0
#endif
#else
#define ROBIN_HOOD_PRIVATE_DEFINITION_HAS_NATIVE_WCHART() 1
#endif

// detect if MSVC supports the pair(std::piecewise_construct_t,...) consructor being constexpr
#ifdef _MSC_VER
#if _MSC_VER <= 1900
#define ROBIN_HOOD_PRIVATE_DEFINITION_BROKEN_CONSTEXPR() 1
#else
#define ROBIN_HOOD_PRIVATE_DEFINITION_BROKEN_CONSTEXPR() 0
#endif
#else
#define ROBIN_HOOD_PRIVATE_DEFINITION_BROKEN_CONSTEXPR() 0
#endif

// helpers for C++ versions, see https://gcc.gnu.org/onlinedocs/cpp/Standard-Predefined-Macros.html
#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX() __cplusplus
#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX98() 199711L
#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX11() 201103L
#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX14() 201402L
#define ROBIN_HOOD_PRIVATE_DEFINITION_CXX17() 201703L

#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX17)
#define ROBIN_HOOD_PRIVATE_DEFINITION_NODISCARD() [[nodiscard]]
#else
#define ROBIN_HOOD_PRIVATE_DEFINITION_NODISCARD()
#endif

namespace robin_hood {

#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX14)
#define ROBIN_HOOD_STD std
#else

// c++11 compatibility layer
namespace ROBIN_HOOD_STD {
template <class T>
struct alignment_of : std::integral_constant<std::size_t, alignof(typename std::remove_all_extents<T>::type)> {};

template <class T, T... Ints>
class integer_sequence {
 public:
  using value_type = T;
  static_assert(std::is_integral<value_type>::value, "not integral type");
  static constexpr std::size_t size() noexcept { return sizeof...(Ints); }
};
template <std::size_t... Inds>
using index_sequence = integer_sequence<std::size_t, Inds...>;

namespace detail_ {
template <class T, T Begin, T End, bool>
struct IntSeqImpl {
  using TValue = T;
  static_assert(std::is_integral<TValue>::value, "not integral type");
  static_assert(Begin >= 0 && Begin < End, "unexpected argument (Begin<0 || Begin<=End)");

  template <class, class>
  struct IntSeqCombiner;

  template <TValue... Inds0, TValue... Inds1>
  struct IntSeqCombiner<integer_sequence<TValue, Inds0...>, integer_sequence<TValue, Inds1...>> {
    using TResult = integer_sequence<TValue, Inds0..., Inds1...>;
  };

  using TResult = typename IntSeqCombiner<
      typename IntSeqImpl<TValue, Begin, Begin + (End - Begin) / 2, (End - Begin) / 2 == 1>::TResult,
      typename IntSeqImpl<TValue, Begin + (End - Begin) / 2, End, (End - Begin + 1) / 2 == 1>::TResult>::TResult;
};

template <class T, T Begin>
struct IntSeqImpl<T, Begin, Begin, false> {
  using TValue = T;
  static_assert(std::is_integral<TValue>::value, "not integral type");
  static_assert(Begin >= 0, "unexpected argument (Begin<0)");
  using TResult = integer_sequence<TValue>;
};

template <class T, T Begin, T End>
struct IntSeqImpl<T, Begin, End, true> {
  using TValue = T;
  static_assert(std::is_integral<TValue>::value, "not integral type");
  static_assert(Begin >= 0, "unexpected argument (Begin<0)");
  using TResult = integer_sequence<TValue, Begin>;
};
}  // namespace detail_

template <class T, T N>
using make_integer_sequence = typename detail_::IntSeqImpl<T, 0, N, (N - 0) == 1>::TResult;

template <std::size_t N>
using make_index_sequence = make_integer_sequence<std::size_t, N>;

template <class... T>
using index_sequence_for = make_index_sequence<sizeof...(T)>;

}  // namespace ROBIN_HOOD_STD

#endif

namespace detail {

// make sure we static_cast to the correct type for hash_int
#if ROBIN_HOOD(BITNESS) == 64
using SizeT = uint64_t;
#else
using SizeT = uint32_t;
#endif

template <typename T>
T rotr(T x, unsigned k) {
  return (x >> k) | (x << (8U * sizeof(T) - k));
}

// This cast gets rid of warnings like "cast from 'uint8_t*' {aka 'unsigned char*'} to
// 'uint64_t*' {aka 'long unsigned int*'} increases required alignment of target type". Use with
// care!
template <typename T>
inline T reinterpret_cast_no_cast_align_warning(void *ptr) noexcept {
  return reinterpret_cast<T>(ptr);
}

template <typename T>
inline T reinterpret_cast_no_cast_align_warning(void const *ptr) noexcept {
  return reinterpret_cast<T>(ptr);
}

// make sure this is not inlined as it is slow and dramatically enlarges code, thus making other
// inlinings more difficult. Throws are also generally the slow path.
template <typename E, typename... Args>
[[noreturn]] ROBIN_HOOD(NOINLINE)
#if ROBIN_HOOD(HAS_EXCEPTIONS)
    void doThrow(Args &&...args) {
  // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-array-to-pointer-decay)
  throw E(std::forward<Args>(args)...);
}
#else
    void doThrow(Args &&...ROBIN_HOOD_UNUSED(args) /*unused*/) {
  abort();
}
#endif

template <typename E, typename T, typename... Args>
T *assertNotNull(T *t, Args &&...args) {
  if (ROBIN_HOOD_UNLIKELY(nullptr == t)) {
    doThrow<E>(std::forward<Args>(args)...);
  }
  return t;
}

template <typename T>
inline T unaligned_load(void const *ptr) noexcept {
  // using memcpy so we don't get into unaligned load problems.
  // compiler should optimize this very well anyways.
  T t;
  std::memcpy(&t, ptr, sizeof(T));
  return t;
}

// Allocates bulks of memory for objects of type T. This deallocates the memory in the destructor,
// and keeps a linked list of the allocated memory around. Overhead per allocation is the size of a
// pointer.
template <typename T, size_t MinNumAllocs = 4, size_t MaxNumAllocs = 256>
class BulkPoolAllocator {
 public:
  BulkPoolAllocator() noexcept = default;

  // does not copy anything, just creates a new allocator.
  BulkPoolAllocator(const BulkPoolAllocator &ROBIN_HOOD_UNUSED(o) /*unused*/) noexcept
      : mHead(nullptr),
        mListForFree(nullptr) {}

  BulkPoolAllocator(BulkPoolAllocator &&o) noexcept
      : mHead(o.mHead),
        mListForFree(o.mListForFree) {
    o.mListForFree = nullptr;
    o.mHead = nullptr;
  }

  BulkPoolAllocator &operator=(BulkPoolAllocator &&o) noexcept {
    reset();
    mHead = o.mHead;
    mListForFree = o.mListForFree;
    o.mListForFree = nullptr;
    o.mHead = nullptr;
    return *this;
  }

  BulkPoolAllocator &
  // NOLINTNEXTLINE(bugprone-unhandled-self-assignment,cert-oop54-cpp)
  operator=(const BulkPoolAllocator &ROBIN_HOOD_UNUSED(o) /*unused*/) noexcept {
    // does not do anything
    return *this;
  }

  ~BulkPoolAllocator() noexcept { reset(); }

  // Deallocates all allocated memory.
  void reset() noexcept {
    while (mListForFree) {
      T *tmp = *mListForFree;
      ROBIN_HOOD_LOG("hf3fs::memory::deallocate")
      hf3fs::memory::deallocate(mListForFree);
      mListForFree = reinterpret_cast_no_cast_align_warning<T **>(tmp);
    }
    mHead = nullptr;
  }

  // allocates, but does NOT initialize. Use in-place new constructor, e.g.
  //   T* obj = pool.allocate();
  //   ::new (static_cast<void*>(obj)) T();
  T *allocate() {
    T *tmp = mHead;
    if (!tmp) {
      tmp = performAllocation();
    }

    mHead = *reinterpret_cast_no_cast_align_warning<T **>(tmp);
    return tmp;
  }

  // does not actually deallocate but puts it in store.
  // make sure you have already called the destructor! e.g. with
  //  obj->~T();
  //  pool.deallocate(obj);
  void deallocate(T *obj) noexcept {
    *reinterpret_cast_no_cast_align_warning<T **>(obj) = mHead;
    mHead = obj;
  }

  // Adds an already allocated block of memory to the allocator. This allocator is from now on
  // responsible for freeing the data (with free()). If the provided data is not large enough to
  // make use of, it is immediately freed. Otherwise it is reused and freed in the destructor.
  void addOrFree(void *ptr, const size_t numBytes) noexcept {
    // calculate number of available elements in ptr
    if (numBytes < ALIGNMENT + ALIGNED_SIZE) {
      // not enough data for at least one element. Free and return.
      ROBIN_HOOD_LOG("hf3fs::memory::deallocate")
      hf3fs::memory::deallocate(ptr);
    } else {
      ROBIN_HOOD_LOG("add to buffer")
      add(ptr, numBytes);
    }
  }

  void swap(BulkPoolAllocator<T, MinNumAllocs, MaxNumAllocs> &other) noexcept {
    using std::swap;
    swap(mHead, other.mHead);
    swap(mListForFree, other.mListForFree);
  }

 private:
  // iterates the list of allocated memory to calculate how many to alloc next.
  // Recalculating this each time saves us a size_t member.
  // This ignores the fact that memory blocks might have been added manually with addOrFree. In
  // practice, this should not matter much.
  ROBIN_HOOD(NODISCARD) size_t calcNumElementsToAlloc() const noexcept {
    auto tmp = mListForFree;
    size_t numAllocs = MinNumAllocs;

    while (numAllocs * 2 <= MaxNumAllocs && tmp) {
      auto x = reinterpret_cast<T ***>(tmp);
      tmp = *x;
      numAllocs *= 2;
    }

    return numAllocs;
  }

  // WARNING: Underflow if numBytes < ALIGNMENT! This is guarded in addOrFree().
  void add(void *ptr, const size_t numBytes) noexcept {
    const size_t numElements = (numBytes - ALIGNMENT) / ALIGNED_SIZE;

    auto data = reinterpret_cast<T **>(ptr);

    // link free list
    auto x = reinterpret_cast<T ***>(data);
    *x = mListForFree;
    mListForFree = data;

    // create linked list for newly allocated data
    auto *const headT = reinterpret_cast_no_cast_align_warning<T *>(reinterpret_cast<char *>(ptr) + ALIGNMENT);

    auto *const head = reinterpret_cast<char *>(headT);

    // Visual Studio compiler automatically unrolls this loop, which is pretty cool
    for (size_t i = 0; i < numElements; ++i) {
      *reinterpret_cast_no_cast_align_warning<char **>(head + i * ALIGNED_SIZE) = head + (i + 1) * ALIGNED_SIZE;
    }

    // last one points to 0
    *reinterpret_cast_no_cast_align_warning<T **>(head + (numElements - 1) * ALIGNED_SIZE) = mHead;
    mHead = headT;
  }

  // Called when no memory is available (mHead == 0).
  // Don't inline this slow path.
  ROBIN_HOOD(NOINLINE) T *performAllocation() {
    size_t const numElementsToAlloc = calcNumElementsToAlloc();

    // alloc new memory: [prev |T, T, ... T]
    size_t const bytes = ALIGNMENT + ALIGNED_SIZE * numElementsToAlloc;
    ROBIN_HOOD_LOG("hf3fs::memory::allocate " << bytes << " = " << ALIGNMENT << " + " << ALIGNED_SIZE << " * "
                                              << numElementsToAlloc)
    add(assertNotNull<std::bad_alloc>(hf3fs::memory::allocate(bytes)), bytes);
    return mHead;
  }

  // enforce byte alignment of the T's
#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX14)
  static constexpr size_t ALIGNMENT = (std::max)(std::alignment_of<T>::value, std::alignment_of<T *>::value);
#else
  static const size_t ALIGNMENT = (ROBIN_HOOD_STD::alignment_of<T>::value > ROBIN_HOOD_STD::alignment_of<T *>::value)
                                      ? ROBIN_HOOD_STD::alignment_of<T>::value
                                      : +ROBIN_HOOD_STD::alignment_of<T *>::value;  // the + is for walkarround
#endif

  static constexpr size_t ALIGNED_SIZE = ((sizeof(T) - 1) / ALIGNMENT + 1) * ALIGNMENT;

  static_assert(MinNumAllocs >= 1, "MinNumAllocs");
  static_assert(MaxNumAllocs >= MinNumAllocs, "MaxNumAllocs");
  static_assert(ALIGNED_SIZE >= sizeof(T *), "ALIGNED_SIZE");
  static_assert(0 == (ALIGNED_SIZE % sizeof(T *)), "ALIGNED_SIZE mod");
  static_assert(ALIGNMENT >= sizeof(T *), "ALIGNMENT");

  T *mHead{nullptr};
  T **mListForFree{nullptr};
};

template <typename T, size_t MinSize, size_t MaxSize, bool IsFlat>
struct NodeAllocator;

// dummy allocator that does nothing
template <typename T, size_t MinSize, size_t MaxSize>
struct NodeAllocator<T, MinSize, MaxSize, true> {
  // we are not using the data, so just free it.
  void addOrFree(void *ptr, size_t ROBIN_HOOD_UNUSED(numBytes) /*unused*/) noexcept {
    ROBIN_HOOD_LOG("hf3fs::memory::deallocate")
    hf3fs::memory::deallocate(ptr);
  }
};

template <typename T, size_t MinSize, size_t MaxSize>
struct NodeAllocator<T, MinSize, MaxSize, false> : public BulkPoolAllocator<T, MinSize, MaxSize> {};

// c++14 doesn't have is_nothrow_swappable, and clang++ 6.0.1 doesn't like it either, so I'm making
// my own here.
namespace swappable {
#if ROBIN_HOOD(CXX) < ROBIN_HOOD(CXX17)
using std::swap;
template <typename T>
struct nothrow {
  static const bool value = noexcept(swap(std::declval<T &>(), std::declval<T &>()));
};
#else
template <typename T>
struct nothrow {
  static const bool value = std::is_nothrow_swappable<T>::value;
};
#endif
}  // namespace swappable

}  // namespace detail

struct is_transparent_tag {};

// A custom pair implementation is used in the map because std::pair is not is_trivially_copyable,
// which means it would  not be allowed to be used in std::memcpy. This struct is copyable, which is
// also tested.
template <typename T1, typename T2>
struct pair {
  using first_type = T1;
  using second_type = T2;

  template <typename U1 = T1,
            typename U2 = T2,
            typename = typename std::enable_if<std::is_default_constructible<U1>::value &&
                                               std::is_default_constructible<U2>::value>::type>
  constexpr pair() noexcept(noexcept(U1()) &&noexcept(U2()))
      : first(),
        second() {}

  // pair constructors are explicit so we don't accidentally call this ctor when we don't have to.
  explicit constexpr pair(std::pair<T1, T2> const &o) noexcept(
      noexcept(T1(std::declval<T1 const &>())) &&noexcept(T2(std::declval<T2 const &>())))
      : first(o.first),
        second(o.second) {}

  // pair constructors are explicit so we don't accidentally call this ctor when we don't have to.
  explicit constexpr pair(std::pair<T1, T2> &&o) noexcept(
      noexcept(T1(std::move(std::declval<T1 &&>()))) &&noexcept(T2(std::move(std::declval<T2 &&>()))))
      : first(std::move(o.first)),
        second(std::move(o.second)) {}

  constexpr pair(T1 &&a, T2 &&b) noexcept(
      noexcept(T1(std::move(std::declval<T1 &&>()))) &&noexcept(T2(std::move(std::declval<T2 &&>()))))
      : first(std::move(a)),
        second(std::move(b)) {}

  template <typename U1, typename U2>
  constexpr pair(U1 &&a, U2 &&b) noexcept(
      noexcept(T1(std::forward<U1>(std::declval<U1 &&>()))) &&noexcept(T2(std::forward<U2>(std::declval<U2 &&>()))))
      : first(std::forward<U1>(a)),
        second(std::forward<U2>(b)) {}

  template <typename... U1, typename... U2>
  // MSVC 2015 produces error "C2476: ‘constexpr’ constructor does not initialize all members"
  // if this constructor is constexpr
#if !ROBIN_HOOD(BROKEN_CONSTEXPR)
  constexpr
#endif
      pair(std::piecewise_construct_t /*unused*/,
           std::tuple<U1...> a,
           std::tuple<U2...> b) noexcept(noexcept(pair(std::declval<std::tuple<U1...> &>(),
                                                       std::declval<std::tuple<U2...> &>(),
                                                       ROBIN_HOOD_STD::index_sequence_for<U1...>(),
                                                       ROBIN_HOOD_STD::index_sequence_for<U2...>())))
      : pair(a, b, ROBIN_HOOD_STD::index_sequence_for<U1...>(), ROBIN_HOOD_STD::index_sequence_for<U2...>()) {
  }

  // constructor called from the std::piecewise_construct_t ctor
  template <typename... U1, size_t... I1, typename... U2, size_t... I2>
  pair(std::tuple<U1...> &a,
       std::tuple<U2...> &b,
       ROBIN_HOOD_STD::index_sequence<I1...> /*unused*/,
       ROBIN_HOOD_STD::index_sequence<
           I2...> /*unused*/) noexcept(noexcept(T1(std::forward<U1>(std::get<I1>(std::declval<std::tuple<U1...>
                                                                                &>()))...))
                                           &&noexcept(T2(
                                               std::forward<U2>(std::get<I2>(std::declval<std::tuple<U2...> &>()))...)))
      : first(std::forward<U1>(std::get<I1>(a))...),
        second(std::forward<U2>(std::get<I2>(b))...) {
    // make visual studio compiler happy about warning about unused a & b.
    // Visual studio's pair implementation disables warning 4100.
    (void)a;
    (void)b;
  }

  void swap(pair<T1, T2> &o) noexcept((detail::swappable::nothrow<T1>::value) &&
                                      (detail::swappable::nothrow<T2>::value)) {
    using std::swap;
    swap(first, o.first);
    swap(second, o.second);
  }

  T1 first;   // NOLINT(misc-non-private-member-variables-in-classes)
  T2 second;  // NOLINT(misc-non-private-member-variables-in-classes)
};

template <typename A, typename B>
inline void swap(pair<A, B> &a,
                 pair<A, B> &b) noexcept(noexcept(std::declval<pair<A, B> &>().swap(std::declval<pair<A, B> &>()))) {
  a.swap(b);
}

template <typename A, typename B>
inline constexpr bool operator==(pair<A, B> const &x, pair<A, B> const &y) {
  return (x.first == y.first) && (x.second == y.second);
}
template <typename A, typename B>
inline constexpr bool operator!=(pair<A, B> const &x, pair<A, B> const &y) {
  return !(x == y);
}
template <typename A, typename B>
inline constexpr bool operator<(pair<A, B> const &x, pair<A, B> const &y) noexcept(
    noexcept(std::declval<A const &>() < std::declval<A const &>()) &&noexcept(std::declval<B const &>() <
                                                                               std::declval<B const &>())) {
  return x.first < y.first || (!(y.first < x.first) && x.second < y.second);
}
template <typename A, typename B>
inline constexpr bool operator>(pair<A, B> const &x, pair<A, B> const &y) {
  return y < x;
}
template <typename A, typename B>
inline constexpr bool operator<=(pair<A, B> const &x, pair<A, B> const &y) {
  return !(x > y);
}
template <typename A, typename B>
inline constexpr bool operator>=(pair<A, B> const &x, pair<A, B> const &y) {
  return !(x < y);
}

inline size_t hash_bytes(void const *ptr, size_t len) noexcept {
  static constexpr uint64_t m = UINT64_C(0xc6a4a7935bd1e995);
  static constexpr uint64_t seed = UINT64_C(0xe17a1465);
  static constexpr unsigned int r = 47;

  auto const *const data64 = static_cast<uint64_t const *>(ptr);
  uint64_t h = seed ^ (len * m);

  size_t const n_blocks = len / 8;
  for (size_t i = 0; i < n_blocks; ++i) {
    auto k = detail::unaligned_load<uint64_t>(data64 + i);

    k *= m;
    k ^= k >> r;
    k *= m;

    h ^= k;
    h *= m;
  }

  auto const *const data8 = reinterpret_cast<uint8_t const *>(data64 + n_blocks);
  switch (len & 7U) {
    case 7:
      h ^= static_cast<uint64_t>(data8[6]) << 48U;
      ROBIN_HOOD(FALLTHROUGH);  // FALLTHROUGH
    case 6:
      h ^= static_cast<uint64_t>(data8[5]) << 40U;
      ROBIN_HOOD(FALLTHROUGH);  // FALLTHROUGH
    case 5:
      h ^= static_cast<uint64_t>(data8[4]) << 32U;
      ROBIN_HOOD(FALLTHROUGH);  // FALLTHROUGH
    case 4:
      h ^= static_cast<uint64_t>(data8[3]) << 24U;
      ROBIN_HOOD(FALLTHROUGH);  // FALLTHROUGH
    case 3:
      h ^= static_cast<uint64_t>(data8[2]) << 16U;
      ROBIN_HOOD(FALLTHROUGH);  // FALLTHROUGH
    case 2:
      h ^= static_cast<uint64_t>(data8[1]) << 8U;
      ROBIN_HOOD(FALLTHROUGH);  // FALLTHROUGH
    case 1:
      h ^= static_cast<uint64_t>(data8[0]);
      h *= m;
      ROBIN_HOOD(FALLTHROUGH);  // FALLTHROUGH
    default:
      break;
  }

  h ^= h >> r;

  // not doing the final step here, because this will be done by keyToIdx anyways
  // h *= m;
  // h ^= h >> r;
  return static_cast<size_t>(h);
}

inline size_t hash_int(uint64_t x) noexcept {
  // tried lots of different hashes, let's stick with murmurhash3. It's simple, fast, well tested,
  // and doesn't need any special 128bit operations.
  x ^= x >> 33U;
  x *= UINT64_C(0xff51afd7ed558ccd);
  x ^= x >> 33U;

  // not doing the final step here, because this will be done by keyToIdx anyways
  // x *= UINT64_C(0xc4ceb9fe1a85ec53);
  // x ^= x >> 33U;
  return static_cast<size_t>(x);
}

// A thin wrapper around std::hash, performing an additional simple mixing step of the result.
template <typename T, typename Enable = void>
struct hash : public std::hash<T> {
  size_t operator()(T const &obj) const
      noexcept(noexcept(std::declval<std::hash<T>>().operator()(std::declval<T const &>()))) {
    // call base hash
    auto result = std::hash<T>::operator()(obj);
    // return mixed of that, to be save against identity has
    return hash_int(static_cast<detail::SizeT>(result));
  }
};

template <typename CharT>
struct hash<std::basic_string<CharT>> {
  size_t operator()(std::basic_string<CharT> const &str) const noexcept {
    return hash_bytes(str.data(), sizeof(CharT) * str.size());
  }
};

#if ROBIN_HOOD(CXX) >= ROBIN_HOOD(CXX17)
template <typename CharT>
struct hash<std::basic_string_view<CharT>> {
  size_t operator()(std::basic_string_view<CharT> const &sv) const noexcept {
    return hash_bytes(sv.data(), sizeof(CharT) * sv.size());
  }
};
#endif

template <class T>
struct hash<T *> {
  size_t operator()(T *ptr) const noexcept { return hash_int(reinterpret_cast<detail::SizeT>(ptr)); }
};

template <class T>
struct hash<std::unique_ptr<T>> {
  size_t operator()(std::unique_ptr<T> const &ptr) const noexcept {
    return hash_int(reinterpret_cast<detail::SizeT>(ptr.get()));
  }
};

template <class T>
struct hash<std::shared_ptr<T>> {
  size_t operator()(std::shared_ptr<T> const &ptr) const noexcept {
    return hash_int(reinterpret_cast<detail::SizeT>(ptr.get()));
  }
};

template <typename Enum>
struct hash<Enum, typename std::enable_if<std::is_enum<Enum>::value>::type> {
  size_t operator()(Enum e) const noexcept {
    using Underlying = typename std::underlying_type<Enum>::type;
    return hash<Underlying>{}(static_cast<Underlying>(e));
  }
};

#define ROBIN_HOOD_HASH_INT(T)                                                                      \
  template <>                                                                                \
  struct hash<T> {                                                                                \
    size_t operator()(T const &obj) const noexcept { return hash_int(static_cast<uint64_t>(obj)); } \
  }

#if defined(__GNUC__) && !defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wuseless-cast"
#endif
// see https://en.cppreference.com/w/cpp/utility/hash
ROBIN_HOOD_HASH_INT(bool);
ROBIN_HOOD_HASH_INT(char);
ROBIN_HOOD_HASH_INT(signed char);
ROBIN_HOOD_HASH_INT(unsigned char);
ROBIN_HOOD_HASH_INT(char16_t);
ROBIN_HOOD_HASH_INT(char32_t);
#if ROBIN_HOOD(HAS_NATIVE_WCHART)
ROBIN_HOOD_HASH_INT(wchar_t);
#endif
ROBIN_HOOD_HASH_INT(short);
ROBIN_HOOD_HASH_INT(unsigned short);
ROBIN_HOOD_HASH_INT(int);
ROBIN_HOOD_HASH_INT(unsigned int);
ROBIN_HOOD_HASH_INT(long);
ROBIN_HOOD_HASH_INT(long long);
ROBIN_HOOD_HASH_INT(unsigned long);
ROBIN_HOOD_HASH_INT(unsigned long long);
#if defined(__GNUC__) && !defined(__clang__)
#pragma GCC diagnostic pop
#endif
namespace detail {

template <typename T>
struct void_type {
  using type = void;
};

template <typename T, typename = void>
struct has_is_transparent : public std::false_type {};

template <typename T>
struct has_is_transparent<T, typename void_type<typename T::is_transparent>::type> : public std::true_type {};

// using wrapper classes for hash and key_equal prevents the diamond problem when the same type
// is used. see https://stackoverflow.com/a/28771920/48181
template <typename T>
struct WrapHash : public T {
  WrapHash() = default;
  explicit WrapHash(T const &o) noexcept(noexcept(T(std::declval<T const &>())))
      : T(o) {}
};

template <typename T>
struct WrapKeyEqual : public T {
  WrapKeyEqual() = default;
  explicit WrapKeyEqual(T const &o) noexcept(noexcept(T(std::declval<T const &>())))
      : T(o) {}
};

// A highly optimized hashmap implementation, using the Robin Hood algorithm.
//
// In most cases, this map should be usable as a drop-in replacement for std::unordered_map, but
// be about 2x faster in most cases and require much less allocations.
//
// This implementation uses the following memory layout:
//
// [Node, Node, ... Node | info, info, ... infoSentinel ]
//
// * Node: either a DataNode that directly has the std::pair<key, val> as member,
//   or a DataNode with a pointer to std::pair<key,val>. Which DataNode representation to use
//   depends on how fast the swap() operation is. Heuristically, this is automatically choosen
//   based on sizeof(). there are always 2^n Nodes.
//
// * info: Each Node in the map has a corresponding info byte, so there are 2^n info bytes.
//   Each byte is initialized to 0, meaning the corresponding Node is empty. Set to 1 means the
//   corresponding node contains data. Set to 2 means the corresponding Node is filled, but it
//   actually belongs to the previous position and was pushed out because that place is already
//   taken.
//
// * infoSentinel: Sentinel byte set to 1, so that iterator's ++ can stop at end() without the
//   need for a idx variable.
//
// According to STL, order of templates has effect on throughput. That's why I've moved the
// boolean to the front.
// https://www.reddit.com/r/cpp/comments/ahp6iu/compile_time_binary_size_reductions_and_cs_future/eeguck4/
template <bool IsFlat, size_t MaxLoadFactor100, typename Key, typename T, typename Hash, typename KeyEqual>
class Table
    : public WrapHash<Hash>,
      public WrapKeyEqual<KeyEqual>,
      detail::NodeAllocator<
          typename std::conditional<std::is_void<T>::value,
                                    Key,
                                    robin_hood::pair<typename std::conditional<IsFlat, Key, Key const>::type, T>>::type,
          4,
          16384,
          IsFlat> {
 public:
  static constexpr bool is_flat = IsFlat;
  static constexpr bool is_map = !std::is_void<T>::value;
  static constexpr bool is_set = !is_map;
  static constexpr bool is_transparent = has_is_transparent<Hash>::value && has_is_transparent<KeyEqual>::value;

  using key_type = Key;
  using mapped_type = T;
  using value_type = typename std::
      conditional<is_set, Key, robin_hood::pair<typename std::conditional<is_flat, Key, Key const>::type, T>>::type;
  using size_type = size_t;
  using hasher = Hash;
  using key_equal = KeyEqual;
  using Self = Table<IsFlat, MaxLoadFactor100, key_type, mapped_type, hasher, key_equal>;

 private:
  static_assert(MaxLoadFactor100 > 10 && MaxLoadFactor100 < 100, "MaxLoadFactor100 needs to be >10 && < 100");

  using WHash = WrapHash<Hash>;
  using WKeyEqual = WrapKeyEqual<KeyEqual>;

  // configuration defaults

  // make sure we have 8 elements, needed to quickly rehash mInfo
  static constexpr size_t InitialNumElements = sizeof(uint64_t);
  static constexpr uint32_t InitialInfoNumBits = 5;
  static constexpr uint8_t InitialInfoInc = 1U << InitialInfoNumBits;
  static constexpr size_t InfoMask = InitialInfoInc - 1U;
  static constexpr uint8_t InitialInfoHashShift = 0;
  using DataPool = detail::NodeAllocator<value_type, 4, 16384, IsFlat>;

  // type needs to be wider than uint8_t.
  using InfoType = uint32_t;

  // DataNode ////////////////////////////////////////////////////////

  // Primary template for the data node. We have special implementations for small and big
  // objects. For large objects it is assumed that swap() is fairly slow, so we allocate these
  // on the heap so swap merely swaps a pointer.
  template <typename M, bool>
  class DataNode {};

  // Small: just allocate on the stack.
  template <typename M>
  class DataNode<M, true> final {
   public:
    template <typename... Args>
    explicit DataNode(M &ROBIN_HOOD_UNUSED(map) /*unused*/,
                      Args &&...args) noexcept(noexcept(value_type(std::forward<Args>(args)...)))
        : mData(std::forward<Args>(args)...) {}

    DataNode(M &ROBIN_HOOD_UNUSED(map) /*unused*/,
             DataNode<M, true> &&n) noexcept(std::is_nothrow_move_constructible<value_type>::value)
        : mData(std::move(n.mData)) {}

    // doesn't do anything
    void destroy(M &ROBIN_HOOD_UNUSED(map) /*unused*/) noexcept {}
    void destroyDoNotDeallocate() noexcept {}

    value_type const *operator->() const noexcept { return &mData; }
    value_type *operator->() noexcept { return &mData; }

    const value_type &operator*() const noexcept { return mData; }

    value_type &operator*() noexcept { return mData; }

    template <typename VT = value_type>
    ROBIN_HOOD(NODISCARD)
    typename std::enable_if<is_map, typename VT::first_type &>::type getFirst() noexcept {
      return mData.first;
    }
    template <typename VT = value_type>
    ROBIN_HOOD(NODISCARD)
    typename std::enable_if<is_set, VT &>::type getFirst() noexcept {
      return mData;
    }

    template <typename VT = value_type>
    ROBIN_HOOD(NODISCARD)
    typename std::enable_if<is_map, typename VT::first_type const &>::type getFirst() const noexcept {
      return mData.first;
    }
    template <typename VT = value_type>
    ROBIN_HOOD(NODISCARD)
    typename std::enable_if<is_set, VT const &>::type getFirst() const noexcept {
      return mData;
    }

    template <typename MT = mapped_type>
    ROBIN_HOOD(NODISCARD)
    typename std::enable_if<is_map, MT &>::type getSecond() noexcept {
      return mData.second;
    }

    template <typename MT = mapped_type>
    ROBIN_HOOD(NODISCARD)
    typename std::enable_if<is_set, MT const &>::type getSecond() const noexcept {
      return mData.second;
    }

    void swap(DataNode<M, true> &o) noexcept(noexcept(std::declval<value_type>().swap(std::declval<value_type>()))) {
      mData.swap(o.mData);
    }

   private:
    value_type mData;
  };

  // big object: allocate on heap.
  template <typename M>
  class DataNode<M, false> {
   public:
    template <typename... Args>
    explicit DataNode(M &map, Args &&...args)
        : mData(map.allocate()) {
      ::new (static_cast<void *>(mData)) value_type(std::forward<Args>(args)...);
    }

    DataNode(M &ROBIN_HOOD_UNUSED(map) /*unused*/, DataNode<M, false> &&n) noexcept
        : mData(std::move(n.mData)) {}

    void destroy(M &map) noexcept {
      // don't deallocate, just put it into list of datapool.
      mData->~value_type();
      map.deallocate(mData);
    }

    void destroyDoNotDeallocate() noexcept { mData->~value_type(); }

    value_type const *operator->() const noexcept { return mData; }

    value_type *operator->() noexcept { return mData; }

    const value_type &operator*() const { return *mData; }

    value_type &operator*() { return *mData; }

    template <typename VT = value_type>
    ROBIN_HOOD(NODISCARD)
    typename std::enable_if<is_map, typename VT::first_type &>::type getFirst() noexcept {
      return mData->first;
    }
    template <typename VT = value_type>
    ROBIN_HOOD(NODISCARD)
    typename std::enable_if<is_set, VT &>::type getFirst() noexcept {
      return *mData;
    }

    template <typename VT = value_type>
    ROBIN_HOOD(NODISCARD)
    typename std::enable_if<is_map, typename VT::first_type const &>::type getFirst() const noexcept {
      return mData->first;
    }
    template <typename VT = value_type>
    ROBIN_HOOD(NODISCARD)
    typename std::enable_if<is_set, VT const &>::type getFirst() const noexcept {
      return *mData;
    }

    template <typename MT = mapped_type>
    ROBIN_HOOD(NODISCARD)
    typename std::enable_if<is_map, MT &>::type getSecond() noexcept {
      return mData->second;
    }

    template <typename MT = mapped_type>
    ROBIN_HOOD(NODISCARD)
    typename std::enable_if<is_map, MT const &>::type getSecond() const noexcept {
      return mData->second;
    }

    void swap(DataNode<M, false> &o) noexcept {
      using std::swap;
      swap(mData, o.mData);
    }

   private:
    value_type *mData;
  };

  using Node = DataNode<Self, IsFlat>;

  // helpers for insertKeyPrepareEmptySpot: extract first entry (only const required)
  ROBIN_HOOD(NODISCARD) key_type const &getFirstConst(Node const &n) const noexcept { return n.getFirst(); }

  // in case we have void mapped_type, we are not using a pair, thus we just route k through.
  // No need to disable this because it's just not used if not applicable.
  ROBIN_HOOD(NODISCARD) key_type const &getFirstConst(key_type const &k) const noexcept { return k; }

  // in case we have non-void mapped_type, we have a standard robin_hood::pair
  template <typename Q = mapped_type>
  ROBIN_HOOD(NODISCARD)
  typename std::enable_if<!std::is_void<Q>::value, key_type const &>::type
      getFirstConst(value_type const &vt) const noexcept {
    return vt.first;
  }

  // Cloner //////////////////////////////////////////////////////////

  template <typename M, bool UseMemcpy>
  struct Cloner;

  // fast path: Just copy data, without allocating anything.
  template <typename M>
  struct Cloner<M, true> {
    void operator()(M const &source, M &target) const {
      auto const *const src = reinterpret_cast<char const *>(source.mKeyVals);
      auto *tgt = reinterpret_cast<char *>(target.mKeyVals);
      auto const numElementsWithBuffer = target.calcNumElementsWithBuffer(target.mMask + 1);
      std::copy(src, src + target.calcNumBytesTotal(numElementsWithBuffer), tgt);
    }
  };

  template <typename M>
  struct Cloner<M, false> {
    void operator()(M const &s, M &t) const {
      auto const numElementsWithBuffer = t.calcNumElementsWithBuffer(t.mMask + 1);
      std::copy(s.mInfo, s.mInfo + t.calcNumBytesInfo(numElementsWithBuffer), t.mInfo);

      for (size_t i = 0; i < numElementsWithBuffer; ++i) {
        if (t.mInfo[i]) {
          ::new (static_cast<void *>(t.mKeyVals + i)) Node(t, *s.mKeyVals[i]);
        }
      }
    }
  };

  // Destroyer ///////////////////////////////////////////////////////

  template <typename M, bool IsFlatAndTrivial>
  struct Destroyer {};

  template <typename M>
  struct Destroyer<M, true> {
    void nodes(M &m) const noexcept { m.mNumElements = 0; }

    void nodesDoNotDeallocate(M &m) const noexcept { m.mNumElements = 0; }
  };

  template <typename M>
  struct Destroyer<M, false> {
    void nodes(M &m) const noexcept {
      m.mNumElements = 0;
      // clear also resets mInfo to 0, that's sometimes not necessary.
      auto const numElementsWithBuffer = m.calcNumElementsWithBuffer(m.mMask + 1);

      for (size_t idx = 0; idx < numElementsWithBuffer; ++idx) {
        if (0 != m.mInfo[idx]) {
          Node &n = m.mKeyVals[idx];
          n.destroy(m);
          n.~Node();
        }
      }
    }

    void nodesDoNotDeallocate(M &m) const noexcept {
      m.mNumElements = 0;
      // clear also resets mInfo to 0, that's sometimes not necessary.
      auto const numElementsWithBuffer = m.calcNumElementsWithBuffer(m.mMask + 1);
      for (size_t idx = 0; idx < numElementsWithBuffer; ++idx) {
        if (0 != m.mInfo[idx]) {
          Node &n = m.mKeyVals[idx];
          n.destroyDoNotDeallocate();
          n.~Node();
        }
      }
    }
  };

  // Iter ////////////////////////////////////////////////////////////

  struct fast_forward_tag {};

  // generic iterator for both const_iterator and iterator.
  template <bool IsConst>
  // NOLINTNEXTLINE(hicpp-special-member-functions,cppcoreguidelines-special-member-functions)
  class Iter {
   private:
    using NodePtr = typename std::conditional<IsConst, Node const *, Node *>::type;

   public:
    using difference_type = std::ptrdiff_t;
    using value_type = typename Self::value_type;
    using reference = typename std::conditional<IsConst, value_type const &, value_type &>::type;
    using pointer = typename std::conditional<IsConst, value_type const *, value_type *>::type;
    using iterator_category = std::forward_iterator_tag;

    // default constructed iterator can be compared to itself, but WON'T return true when
    // compared to end().
    Iter() = default;

    // Rule of zero: nothing specified. The conversion constructor is only enabled for
    // iterator to const_iterator, so it doesn't accidentally work as a copy ctor.

    // Conversion constructor from iterator to const_iterator.
    template <bool OtherIsConst, typename = typename std::enable_if<IsConst && !OtherIsConst>::type>
    // NOLINTNEXTLINE(hicpp-explicit-conversions)
    Iter(Iter<OtherIsConst> const &other) noexcept
        : mKeyVals(other.mKeyVals),
          mInfo(other.mInfo) {}

    Iter(NodePtr valPtr, uint8_t const *infoPtr) noexcept
        : mKeyVals(valPtr),
          mInfo(infoPtr) {}

    Iter(NodePtr valPtr, uint8_t const *infoPtr, fast_forward_tag ROBIN_HOOD_UNUSED(tag) /*unused*/) noexcept
        : mKeyVals(valPtr),
          mInfo(infoPtr) {
      fastForward();
    }

    template <bool OtherIsConst, typename = typename std::enable_if<IsConst && !OtherIsConst>::type>
    Iter &operator=(Iter<OtherIsConst> const &other) noexcept {
      mKeyVals = other.mKeyVals;
      mInfo = other.mInfo;
      return *this;
    }

    // prefix increment. Undefined behavior if we are at end()!
    Iter &operator++() noexcept {
      mInfo++;
      mKeyVals++;
      fastForward();
      return *this;
    }

    Iter operator++(int) noexcept {
      Iter tmp = *this;
      ++(*this);
      return tmp;
    }

    reference operator*() const { return **mKeyVals; }

    pointer operator->() const { return &**mKeyVals; }

    template <bool O>
    bool operator==(Iter<O> const &o) const noexcept {
      return mKeyVals == o.mKeyVals;
    }

    template <bool O>
    bool operator!=(Iter<O> const &o) const noexcept {
      return mKeyVals != o.mKeyVals;
    }

   private:
    // fast forward to the next non-free info byte
    // I've tried a few variants that don't depend on intrinsics, but unfortunately they are
    // quite a bit slower than this one. So I've reverted that change again. See map_benchmark.
    void fastForward() noexcept {
      size_t n = 0;
      while (0U == (n = detail::unaligned_load<size_t>(mInfo))) {
        mInfo += sizeof(size_t);
        mKeyVals += sizeof(size_t);
      }
#if defined(ROBIN_HOOD_DISABLE_INTRINSICS)
      // we know for certain that within the next 8 bytes we'll find a non-zero one.
      if (ROBIN_HOOD_UNLIKELY(0U == detail::unaligned_load<uint32_t>(mInfo))) {
        mInfo += 4;
        mKeyVals += 4;
      }
      if (ROBIN_HOOD_UNLIKELY(0U == detail::unaligned_load<uint16_t>(mInfo))) {
        mInfo += 2;
        mKeyVals += 2;
      }
      if (ROBIN_HOOD_UNLIKELY(0U == *mInfo)) {
        mInfo += 1;
        mKeyVals += 1;
      }
#else
#if ROBIN_HOOD(LITTLE_ENDIAN)
      auto inc = ROBIN_HOOD_COUNT_TRAILING_ZEROES(n) / 8;
#else
      auto inc = ROBIN_HOOD_COUNT_LEADING_ZEROES(n) / 8;
#endif
      mInfo += inc;
      mKeyVals += inc;
#endif
    }

    friend class Table<IsFlat, MaxLoadFactor100, key_type, mapped_type, hasher, key_equal>;
    NodePtr mKeyVals{nullptr};
    uint8_t const *mInfo{nullptr};
  };

  ////////////////////////////////////////////////////////////////////

  // highly performance relevant code.
  // Lower bits are used for indexing into the array (2^n size)
  // The upper 1-5 bits need to be a reasonable good hash, to save comparisons.
  template <typename HashKey>
  void keyToIdx(HashKey &&key, size_t *idx, InfoType *info) const {
    // In addition to whatever hash is used, add another mul & shift so we get better hashing.
    // This serves as a bad hash prevention, if the given data is
    // badly mixed.
    auto h = static_cast<uint64_t>(WHash::operator()(key));

    h *= mHashMultiplier;
    h ^= h >> 33U;

    // the lower InitialInfoNumBits are reserved for info.
    *info = mInfoInc + static_cast<InfoType>((h & InfoMask) >> mInfoHashShift);
    *idx = (static_cast<size_t>(h) >> InitialInfoNumBits) & mMask;
  }

  // forwards the index by one, wrapping around at the end
  void next(InfoType *info, size_t *idx) const noexcept {
    *idx = *idx + 1;
    *info += mInfoInc;
  }

  void nextWhileLess(InfoType *info, size_t *idx) const noexcept {
    // unrolling this by hand did not bring any speedups.
    while (*info < mInfo[*idx]) {
      next(info, idx);
    }
  }

  // Shift everything up by one element. Tries to move stuff around.
  void shiftUp(size_t startIdx, size_t const insertion_idx) noexcept(std::is_nothrow_move_assignable<Node>::value) {
    auto idx = startIdx;
    ::new (static_cast<void *>(mKeyVals + idx)) Node(std::move(mKeyVals[idx - 1]));
    while (--idx != insertion_idx) {
      mKeyVals[idx] = std::move(mKeyVals[idx - 1]);
    }

    idx = startIdx;
    while (idx != insertion_idx) {
      ROBIN_HOOD_COUNT(shiftUp)
      mInfo[idx] = static_cast<uint8_t>(mInfo[idx - 1] + mInfoInc);
      if (ROBIN_HOOD_UNLIKELY(mInfo[idx] + mInfoInc > 0xFF)) {
        mMaxNumElementsAllowed = 0;
      }
      --idx;
    }
  }

  void shiftDown(size_t idx) noexcept(std::is_nothrow_move_assignable<Node>::value) {
    // until we find one that is either empty or has zero offset.
    // TODO(martinus) we don't need to move everything, just the last one for the same
    // bucket.
    mKeyVals[idx].destroy(*this);

    // until we find one that is either empty or has zero offset.
    while (mInfo[idx + 1] >= 2 * mInfoInc) {
      ROBIN_HOOD_COUNT(shiftDown)
      mInfo[idx] = static_cast<uint8_t>(mInfo[idx + 1] - mInfoInc);
      mKeyVals[idx] = std::move(mKeyVals[idx + 1]);
      ++idx;
    }

    mInfo[idx] = 0;
    // don't destroy, we've moved it
    // mKeyVals[idx].destroy(*this);
    mKeyVals[idx].~Node();
  }

  // copy of find(), except that it returns iterator instead of const_iterator.
  template <typename Other>
  ROBIN_HOOD(NODISCARD)
  size_t findIdx(Other const &key) const {
    size_t idx{};
    InfoType info{};
    keyToIdx(key, &idx, &info);

    do {
      // unrolling this twice gives a bit of a speedup. More unrolling did not help.
      if (info == mInfo[idx] && ROBIN_HOOD_LIKELY(WKeyEqual::operator()(key, mKeyVals[idx].getFirst()))) {
        return idx;
      }
      next(&info, &idx);
      if (info == mInfo[idx] && ROBIN_HOOD_LIKELY(WKeyEqual::operator()(key, mKeyVals[idx].getFirst()))) {
        return idx;
      }
      next(&info, &idx);
    } while (info <= mInfo[idx]);

    // nothing found!
    return mMask == 0
               ? 0
               : static_cast<size_t>(std::distance(mKeyVals, reinterpret_cast_no_cast_align_warning<Node *>(mInfo)));
  }

  void cloneData(const Table &o) { Cloner<Table, IsFlat && std::is_trivially_copyable<Node>::value>()(o, *this); }

  // inserts a keyval that is guaranteed to be new, e.g. when the hashmap is resized.
  // @return True on success, false if something went wrong
  void insert_move(Node &&keyval) {
    // we don't retry, fail if overflowing
    // don't need to check max num elements
    if (0 == mMaxNumElementsAllowed && !try_increase_info()) {
      throwOverflowError();
    }

    size_t idx{};
    InfoType info{};
    keyToIdx(keyval.getFirst(), &idx, &info);

    // skip forward. Use <= because we are certain that the element is not there.
    while (info <= mInfo[idx]) {
      idx = idx + 1;
      info += mInfoInc;
    }

    // key not found, so we are now exactly where we want to insert it.
    auto const insertion_idx = idx;
    auto const insertion_info = static_cast<uint8_t>(info);
    if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) {
      mMaxNumElementsAllowed = 0;
    }

    // find an empty spot
    while (0 != mInfo[idx]) {
      next(&info, &idx);
    }

    auto &l = mKeyVals[insertion_idx];
    if (idx == insertion_idx) {
      ::new (static_cast<void *>(&l)) Node(std::move(keyval));
    } else {
      shiftUp(idx, insertion_idx);
      l = std::move(keyval);
    }

    // put at empty spot
    mInfo[insertion_idx] = insertion_info;

    ++mNumElements;
  }

 public:
  using iterator = Iter<false>;
  using const_iterator = Iter<true>;

  Table() noexcept(noexcept(Hash()) &&noexcept(KeyEqual()))
      : WHash(),
        WKeyEqual() {
    ROBIN_HOOD_TRACE(this)
  }

  // Creates an empty hash map. Nothing is allocated yet, this happens at the first insert.
  // This tremendously speeds up ctor & dtor of a map that never receives an element. The
  // penalty is payed at the first insert, and not before. Lookup of this empty map works
  // because everybody points to DummyInfoByte::b. parameter bucket_count is dictated by the
  // standard, but we can ignore it.
  explicit Table(size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/,
                 const Hash &h = Hash{},
                 const KeyEqual &equal = KeyEqual{}) noexcept(noexcept(Hash(h)) &&noexcept(KeyEqual(equal)))
      : WHash(h),
        WKeyEqual(equal) {
    ROBIN_HOOD_TRACE(this)
  }

  template <typename Iter>
  Table(Iter first,
        Iter last,
        size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0,
        const Hash &h = Hash{},
        const KeyEqual &equal = KeyEqual{})
      : WHash(h),
        WKeyEqual(equal) {
    ROBIN_HOOD_TRACE(this)
    insert(first, last);
  }

  Table(std::initializer_list<value_type> initlist,
        size_t ROBIN_HOOD_UNUSED(bucket_count) /*unused*/ = 0,
        const Hash &h = Hash{},
        const KeyEqual &equal = KeyEqual{})
      : WHash(h),
        WKeyEqual(equal) {
    ROBIN_HOOD_TRACE(this)
    insert(initlist.begin(), initlist.end());
  }

  Table(Table &&o) noexcept
      : WHash(std::move(static_cast<WHash &>(o))),
        WKeyEqual(std::move(static_cast<WKeyEqual &>(o))),
        DataPool(std::move(static_cast<DataPool &>(o))) {
    ROBIN_HOOD_TRACE(this)
    if (o.mMask) {
      mHashMultiplier = std::move(o.mHashMultiplier);
      mKeyVals = std::move(o.mKeyVals);
      mInfo = std::move(o.mInfo);
      mNumElements = std::move(o.mNumElements);
      mMask = std::move(o.mMask);
      mMaxNumElementsAllowed = std::move(o.mMaxNumElementsAllowed);
      mInfoInc = std::move(o.mInfoInc);
      mInfoHashShift = std::move(o.mInfoHashShift);
      // set other's mask to 0 so its destructor won't do anything
      o.init();
    }
  }

  Table &operator=(Table &&o) noexcept {
    ROBIN_HOOD_TRACE(this)
    if (&o != this) {
      if (o.mMask) {
        // only move stuff if the other map actually has some data
        destroy();
        mHashMultiplier = std::move(o.mHashMultiplier);
        mKeyVals = std::move(o.mKeyVals);
        mInfo = std::move(o.mInfo);
        mNumElements = std::move(o.mNumElements);
        mMask = std::move(o.mMask);
        mMaxNumElementsAllowed = std::move(o.mMaxNumElementsAllowed);
        mInfoInc = std::move(o.mInfoInc);
        mInfoHashShift = std::move(o.mInfoHashShift);
        WHash::operator=(std::move(static_cast<WHash &>(o)));
        WKeyEqual::operator=(std::move(static_cast<WKeyEqual &>(o)));
        DataPool::operator=(std::move(static_cast<DataPool &>(o)));

        o.init();

      } else {
        // nothing in the other map => just clear us.
        clear();
      }
    }
    return *this;
  }

  Table(const Table &o)
      : WHash(static_cast<const WHash &>(o)),
        WKeyEqual(static_cast<const WKeyEqual &>(o)),
        DataPool(static_cast<const DataPool &>(o)) {
    ROBIN_HOOD_TRACE(this)
    if (!o.empty()) {
      // not empty: create an exact copy. it is also possible to just iterate through all
      // elements and insert them, but copying is probably faster.

      auto const numElementsWithBuffer = calcNumElementsWithBuffer(o.mMask + 1);
      auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer);

      ROBIN_HOOD_LOG("hf3fs::memory::allocate " << numBytesTotal << " = calcNumBytesTotal(" << numElementsWithBuffer
                                                << ")")
      mHashMultiplier = o.mHashMultiplier;
      mKeyVals = static_cast<Node *>(detail::assertNotNull<std::bad_alloc>(hf3fs::memory::allocate(numBytesTotal)));
      // no need for calloc because clonData does memcpy
      mInfo = reinterpret_cast<uint8_t *>(mKeyVals + numElementsWithBuffer);
      mNumElements = o.mNumElements;
      mMask = o.mMask;
      mMaxNumElementsAllowed = o.mMaxNumElementsAllowed;
      mInfoInc = o.mInfoInc;
      mInfoHashShift = o.mInfoHashShift;
      cloneData(o);
    }
  }

  // Creates a copy of the given map. Copy constructor of each entry is used.
  // Not sure why clang-tidy thinks this doesn't handle self assignment, it does
  // NOLINTNEXTLINE(bugprone-unhandled-self-assignment,cert-oop54-cpp)
  Table &operator=(Table const &o) {
    ROBIN_HOOD_TRACE(this)
    if (&o == this) {
      // prevent assigning of itself
      return *this;
    }

    // we keep using the old allocator and not assign the new one, because we want to keep
    // the memory available. when it is the same size.
    if (o.empty()) {
      if (0 == mMask) {
        // nothing to do, we are empty too
        return *this;
      }

      // not empty: destroy what we have there
      // clear also resets mInfo to 0, that's sometimes not necessary.
      destroy();
      init();
      WHash::operator=(static_cast<const WHash &>(o));
      WKeyEqual::operator=(static_cast<const WKeyEqual &>(o));
      DataPool::operator=(static_cast<DataPool const &>(o));

      return *this;
    }

    // clean up old stuff
    Destroyer<Self, IsFlat && std::is_trivially_destructible<Node>::value>{}.nodes(*this);

    if (mMask != o.mMask) {
      // no luck: we don't have the same array size allocated, so we need to realloc.
      if (0 != mMask) {
        // only deallocate if we actually have data!
        ROBIN_HOOD_LOG("hf3fs::memory::deallocate")
        hf3fs::memory::deallocate(mKeyVals);
      }

      auto const numElementsWithBuffer = calcNumElementsWithBuffer(o.mMask + 1);
      auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer);
      ROBIN_HOOD_LOG("hf3fs::memory::allocate " << numBytesTotal << " = calcNumBytesTotal(" << numElementsWithBuffer
                                                << ")")
      mKeyVals = static_cast<Node *>(detail::assertNotNull<std::bad_alloc>(hf3fs::memory::allocate(numBytesTotal)));

      // no need for calloc here because cloneData performs a memcpy.
      mInfo = reinterpret_cast<uint8_t *>(mKeyVals + numElementsWithBuffer);
      // sentinel is set in cloneData
    }
    WHash::operator=(static_cast<const WHash &>(o));
    WKeyEqual::operator=(static_cast<const WKeyEqual &>(o));
    DataPool::operator=(static_cast<DataPool const &>(o));
    mHashMultiplier = o.mHashMultiplier;
    mNumElements = o.mNumElements;
    mMask = o.mMask;
    mMaxNumElementsAllowed = o.mMaxNumElementsAllowed;
    mInfoInc = o.mInfoInc;
    mInfoHashShift = o.mInfoHashShift;
    cloneData(o);

    return *this;
  }

  // Swaps everything between the two maps.
  void swap(Table &o) {
    ROBIN_HOOD_TRACE(this)
    using std::swap;
    swap(o, *this);
  }

  // Clears all data, without resizing.
  void clear() {
    ROBIN_HOOD_TRACE(this)
    if (empty()) {
      // don't do anything! also important because we don't want to write to
      // DummyInfoByte::b, even though we would just write 0 to it.
      return;
    }

    Destroyer<Self, IsFlat && std::is_trivially_destructible<Node>::value>{}.nodes(*this);

    auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1);
    // clear everything, then set the sentinel again
    uint8_t const z = 0;
    std::fill(mInfo, mInfo + calcNumBytesInfo(numElementsWithBuffer), z);
    mInfo[numElementsWithBuffer] = 1;

    mInfoInc = InitialInfoInc;
    mInfoHashShift = InitialInfoHashShift;
  }

  // Destroys the map and all it's contents.
  ~Table() {
    ROBIN_HOOD_TRACE(this)
    destroy();
  }

  // Checks if both tables contain the same entries. Order is irrelevant.
  bool operator==(const Table &other) const {
    ROBIN_HOOD_TRACE(this)
    if (other.size() != size()) {
      return false;
    }
    for (auto const &otherEntry : other) {
      if (!has(otherEntry)) {
        return false;
      }
    }

    return true;
  }

  bool operator!=(const Table &other) const {
    ROBIN_HOOD_TRACE(this)
    return !operator==(other);
  }

  template <typename Q = mapped_type>
  typename std::enable_if<!std::is_void<Q>::value, Q &>::type operator[](const key_type &key) {
    ROBIN_HOOD_TRACE(this)
    auto idxAndState = insertKeyPrepareEmptySpot(key);
    switch (idxAndState.second) {
      case InsertionState::key_found:
        break;

      case InsertionState::new_node:
        ::new (static_cast<void *>(&mKeyVals[idxAndState.first]))
            Node(*this, std::piecewise_construct, std::forward_as_tuple(key), std::forward_as_tuple());
        break;

      case InsertionState::overwrite_node:
        mKeyVals[idxAndState.first] =
            Node(*this, std::piecewise_construct, std::forward_as_tuple(key), std::forward_as_tuple());
        break;

      case InsertionState::overflow_error:
        throwOverflowError();
    }

    return mKeyVals[idxAndState.first].getSecond();
  }

  template <typename Q = mapped_type>
  typename std::enable_if<!std::is_void<Q>::value, Q &>::type operator[](key_type &&key) {
    ROBIN_HOOD_TRACE(this)
    auto idxAndState = insertKeyPrepareEmptySpot(key);
    switch (idxAndState.second) {
      case InsertionState::key_found:
        break;

      case InsertionState::new_node:
        ::new (static_cast<void *>(&mKeyVals[idxAndState.first]))
            Node(*this, std::piecewise_construct, std::forward_as_tuple(std::move(key)), std::forward_as_tuple());
        break;

      case InsertionState::overwrite_node:
        mKeyVals[idxAndState.first] =
            Node(*this, std::piecewise_construct, std::forward_as_tuple(std::move(key)), std::forward_as_tuple());
        break;

      case InsertionState::overflow_error:
        throwOverflowError();
    }

    return mKeyVals[idxAndState.first].getSecond();
  }

  template <typename Iter>
  void insert(Iter first, Iter last) {
    for (; first != last; ++first) {
      // value_type ctor needed because this might be called with std::pair's
      insert(value_type(*first));
    }
  }

  void insert(std::initializer_list<value_type> ilist) {
    for (auto &&vt : ilist) {
      insert(std::move(vt));
    }
  }

  template <typename... Args>
  std::pair<iterator, bool> emplace(Args &&...args) {
    ROBIN_HOOD_TRACE(this)
    Node n{*this, std::forward<Args>(args)...};
    auto idxAndState = insertKeyPrepareEmptySpot(getFirstConst(n));
    switch (idxAndState.second) {
      case InsertionState::key_found:
        n.destroy(*this);
        break;

      case InsertionState::new_node:
        ::new (static_cast<void *>(&mKeyVals[idxAndState.first])) Node(*this, std::move(n));
        break;

      case InsertionState::overwrite_node:
        mKeyVals[idxAndState.first] = std::move(n);
        break;

      case InsertionState::overflow_error:
        n.destroy(*this);
        throwOverflowError();
        break;
    }

    return std::make_pair(iterator(mKeyVals + idxAndState.first, mInfo + idxAndState.first),
                          InsertionState::key_found != idxAndState.second);
  }

  template <typename... Args>
  iterator emplace_hint(const_iterator position, Args &&...args) {
    (void)position;
    return emplace(std::forward<Args>(args)...).first;
  }

  template <typename... Args>
  std::pair<iterator, bool> try_emplace(const key_type &key, Args &&...args) {
    return try_emplace_impl(key, std::forward<Args>(args)...);
  }

  template <typename... Args>
  std::pair<iterator, bool> try_emplace(key_type &&key, Args &&...args) {
    return try_emplace_impl(std::move(key), std::forward<Args>(args)...);
  }

  template <typename... Args>
  iterator try_emplace(const_iterator hint, const key_type &key, Args &&...args) {
    (void)hint;
    return try_emplace_impl(key, std::forward<Args>(args)...).first;
  }

  template <typename... Args>
  iterator try_emplace(const_iterator hint, key_type &&key, Args &&...args) {
    (void)hint;
    return try_emplace_impl(std::move(key), std::forward<Args>(args)...).first;
  }

  template <typename Mapped>
  std::pair<iterator, bool> insert_or_assign(const key_type &key, Mapped &&obj) {
    return insertOrAssignImpl(key, std::forward<Mapped>(obj));
  }

  template <typename Mapped>
  std::pair<iterator, bool> insert_or_assign(key_type &&key, Mapped &&obj) {
    return insertOrAssignImpl(std::move(key), std::forward<Mapped>(obj));
  }

  template <typename Mapped>
  iterator insert_or_assign(const_iterator hint, const key_type &key, Mapped &&obj) {
    (void)hint;
    return insertOrAssignImpl(key, std::forward<Mapped>(obj)).first;
  }

  template <typename Mapped>
  iterator insert_or_assign(const_iterator hint, key_type &&key, Mapped &&obj) {
    (void)hint;
    return insertOrAssignImpl(std::move(key), std::forward<Mapped>(obj)).first;
  }

  std::pair<iterator, bool> insert(const value_type &keyval) {
    ROBIN_HOOD_TRACE(this)
    return emplace(keyval);
  }

  iterator insert(const_iterator hint, const value_type &keyval) {
    (void)hint;
    return emplace(keyval).first;
  }

  std::pair<iterator, bool> insert(value_type &&keyval) { return emplace(std::move(keyval)); }

  iterator insert(const_iterator hint, value_type &&keyval) {
    (void)hint;
    return emplace(std::move(keyval)).first;
  }

  // Returns 1 if key is found, 0 otherwise.
  size_t count(const key_type &key) const {  // NOLINT(modernize-use-nodiscard)
    ROBIN_HOOD_TRACE(this)
    auto kv = mKeyVals + findIdx(key);
    if (kv != reinterpret_cast_no_cast_align_warning<Node *>(mInfo)) {
      return 1;
    }
    return 0;
  }

  template <typename OtherKey, typename Self_ = Self>
  // NOLINTNEXTLINE(modernize-use-nodiscard)
  typename std::enable_if<Self_::is_transparent, size_t>::type count(const OtherKey &key) const {
    ROBIN_HOOD_TRACE(this)
    auto kv = mKeyVals + findIdx(key);
    if (kv != reinterpret_cast_no_cast_align_warning<Node *>(mInfo)) {
      return 1;
    }
    return 0;
  }

  bool contains(const key_type &key) const {  // NOLINT(modernize-use-nodiscard)
    return 1U == count(key);
  }

  template <typename OtherKey, typename Self_ = Self>
  // NOLINTNEXTLINE(modernize-use-nodiscard)
  typename std::enable_if<Self_::is_transparent, bool>::type contains(const OtherKey &key) const {
    return 1U == count(key);
  }

  // Returns a reference to the value found for key.
  // Throws std::out_of_range if element cannot be found
  template <typename Q = mapped_type>
  // NOLINTNEXTLINE(modernize-use-nodiscard)
  typename std::enable_if<!std::is_void<Q>::value, Q &>::type at(key_type const &key) {
    ROBIN_HOOD_TRACE(this)
    auto kv = mKeyVals + findIdx(key);
    if (kv == reinterpret_cast_no_cast_align_warning<Node *>(mInfo)) {
      doThrow<std::out_of_range>("key not found");
    }
    return kv->getSecond();
  }

  // Returns a reference to the value found for key.
  // Throws std::out_of_range if element cannot be found
  template <typename Q = mapped_type>
  // NOLINTNEXTLINE(modernize-use-nodiscard)
  typename std::enable_if<!std::is_void<Q>::value, Q const &>::type at(key_type const &key) const {
    ROBIN_HOOD_TRACE(this)
    auto kv = mKeyVals + findIdx(key);
    if (kv == reinterpret_cast_no_cast_align_warning<Node *>(mInfo)) {
      doThrow<std::out_of_range>("key not found");
    }
    return kv->getSecond();
  }

  const_iterator find(const key_type &key) const {  // NOLINT(modernize-use-nodiscard)
    ROBIN_HOOD_TRACE(this)
    const size_t idx = findIdx(key);
    return const_iterator{mKeyVals + idx, mInfo + idx};
  }

  template <typename OtherKey>
  const_iterator find(const OtherKey &key, is_transparent_tag /*unused*/) const {
    ROBIN_HOOD_TRACE(this)
    const size_t idx = findIdx(key);
    return const_iterator{mKeyVals + idx, mInfo + idx};
  }

  template <typename OtherKey, typename Self_ = Self>
  typename std::enable_if<Self_::is_transparent,  // NOLINT(modernize-use-nodiscard)
                          const_iterator>::type   // NOLINT(modernize-use-nodiscard)
  find(const OtherKey &key) const {               // NOLINT(modernize-use-nodiscard)
    ROBIN_HOOD_TRACE(this)
    const size_t idx = findIdx(key);
    return const_iterator{mKeyVals + idx, mInfo + idx};
  }

  iterator find(const key_type &key) {
    ROBIN_HOOD_TRACE(this)
    const size_t idx = findIdx(key);
    return iterator{mKeyVals + idx, mInfo + idx};
  }

  template <typename OtherKey>
  iterator find(const OtherKey &key, is_transparent_tag /*unused*/) {
    ROBIN_HOOD_TRACE(this)
    const size_t idx = findIdx(key);
    return iterator{mKeyVals + idx, mInfo + idx};
  }

  template <typename OtherKey, typename Self_ = Self>
  typename std::enable_if<Self_::is_transparent, iterator>::type find(const OtherKey &key) {
    ROBIN_HOOD_TRACE(this)
    const size_t idx = findIdx(key);
    return iterator{mKeyVals + idx, mInfo + idx};
  }

  iterator begin() {
    ROBIN_HOOD_TRACE(this)
    if (empty()) {
      return end();
    }
    return iterator(mKeyVals, mInfo, fast_forward_tag{});
  }
  const_iterator begin() const {  // NOLINT(modernize-use-nodiscard)
    ROBIN_HOOD_TRACE(this)
    return cbegin();
  }
  const_iterator cbegin() const {  // NOLINT(modernize-use-nodiscard)
    ROBIN_HOOD_TRACE(this)
    if (empty()) {
      return cend();
    }
    return const_iterator(mKeyVals, mInfo, fast_forward_tag{});
  }

  iterator end() {
    ROBIN_HOOD_TRACE(this)
    // no need to supply valid info pointer: end() must not be dereferenced, and only node
    // pointer is compared.
    return iterator{reinterpret_cast_no_cast_align_warning<Node *>(mInfo), nullptr};
  }
  const_iterator end() const {  // NOLINT(modernize-use-nodiscard)
    ROBIN_HOOD_TRACE(this)
    return cend();
  }
  const_iterator cend() const {  // NOLINT(modernize-use-nodiscard)
    ROBIN_HOOD_TRACE(this)
    return const_iterator{reinterpret_cast_no_cast_align_warning<Node *>(mInfo), nullptr};
  }

  iterator erase(const_iterator pos) {
    ROBIN_HOOD_TRACE(this)
    // its safe to perform const cast here
    // NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast)
    return erase(iterator{const_cast<Node *>(pos.mKeyVals), const_cast<uint8_t *>(pos.mInfo)});
  }

  // Erases element at pos, returns iterator to the next element.
  iterator erase(iterator pos) {
    ROBIN_HOOD_TRACE(this)
    // we assume that pos always points to a valid entry, and not end().
    auto const idx = static_cast<size_t>(pos.mKeyVals - mKeyVals);

    shiftDown(idx);
    --mNumElements;

    if (*pos.mInfo) {
      // we've backward shifted, return this again
      return pos;
    }

    // no backward shift, return next element
    return ++pos;
  }

  size_t erase(const key_type &key) {
    ROBIN_HOOD_TRACE(this)
    size_t idx{};
    InfoType info{};
    keyToIdx(key, &idx, &info);

    // check while info matches with the source idx
    do {
      if (info == mInfo[idx] && WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) {
        shiftDown(idx);
        --mNumElements;
        return 1;
      }
      next(&info, &idx);
    } while (info <= mInfo[idx]);

    // nothing found to delete
    return 0;
  }

  // reserves space for the specified number of elements. Makes sure the old data fits.
  // exactly the same as reserve(c).
  void rehash(size_t c) {
    // forces a reserve
    reserve(c, true);
  }

  // reserves space for the specified number of elements. Makes sure the old data fits.
  // Exactly the same as rehash(c). Use rehash(0) to shrink to fit.
  void reserve(size_t c) {
    // reserve, but don't force rehash
    reserve(c, false);
  }

  // If possible reallocates the map to a smaller one. This frees the underlying table.
  // Does not do anything if load_factor is too large for decreasing the table's size.
  void compact() {
    ROBIN_HOOD_TRACE(this)
    auto newSize = InitialNumElements;
    while (calcMaxNumElementsAllowed(newSize) < mNumElements && newSize != 0) {
      newSize *= 2;
    }
    if (ROBIN_HOOD_UNLIKELY(newSize == 0)) {
      throwOverflowError();
    }

    ROBIN_HOOD_LOG("newSize > mMask + 1: " << newSize << " > " << mMask << " + 1")

    // only actually do anything when the new size is bigger than the old one. This prevents to
    // continuously allocate for each reserve() call.
    if (newSize < mMask + 1) {
      rehashPowerOfTwo(newSize, true);
    }
  }

  size_type size() const noexcept {  // NOLINT(modernize-use-nodiscard)
    ROBIN_HOOD_TRACE(this)
    return mNumElements;
  }

  size_type max_size() const noexcept {  // NOLINT(modernize-use-nodiscard)
    ROBIN_HOOD_TRACE(this)
    return static_cast<size_type>(-1);
  }

  ROBIN_HOOD(NODISCARD) bool empty() const noexcept {
    ROBIN_HOOD_TRACE(this)
    return 0 == mNumElements;
  }

  float max_load_factor() const noexcept {  // NOLINT(modernize-use-nodiscard)
    ROBIN_HOOD_TRACE(this)
    return MaxLoadFactor100 / 100.0F;
  }

  // Average number of elements per bucket. Since we allow only 1 per bucket
  float load_factor() const noexcept {  // NOLINT(modernize-use-nodiscard)
    ROBIN_HOOD_TRACE(this)
    return static_cast<float>(size()) / static_cast<float>(mMask + 1);
  }

  ROBIN_HOOD(NODISCARD) size_t mask() const noexcept {
    ROBIN_HOOD_TRACE(this)
    return mMask;
  }

  ROBIN_HOOD(NODISCARD) size_t calcMaxNumElementsAllowed(size_t maxElements) const noexcept {
    if (ROBIN_HOOD_LIKELY(maxElements <= (std::numeric_limits<size_t>::max)() / 100)) {
      return maxElements * MaxLoadFactor100 / 100;
    }

    // we might be a bit inprecise, but since maxElements is quite large that doesn't matter
    return (maxElements / 100) * MaxLoadFactor100;
  }

  ROBIN_HOOD(NODISCARD) size_t calcNumBytesInfo(size_t numElements) const noexcept {
    // we add a uint64_t, which houses the sentinel (first byte) and padding so we can load
    // 64bit types.
    return numElements + sizeof(uint64_t);
  }

  ROBIN_HOOD(NODISCARD)
  size_t calcNumElementsWithBuffer(size_t numElements) const noexcept {
    auto maxNumElementsAllowed = calcMaxNumElementsAllowed(numElements);
    return numElements + (std::min)(maxNumElementsAllowed, (static_cast<size_t>(0xFF)));
  }

  // calculation only allowed for 2^n values
  ROBIN_HOOD(NODISCARD) size_t calcNumBytesTotal(size_t numElements) const {
#if ROBIN_HOOD(BITNESS) == 64
    return numElements * sizeof(Node) + calcNumBytesInfo(numElements);
#else
    // make sure we're doing 64bit operations, so we are at least safe against 32bit overflows.
    auto const ne = static_cast<uint64_t>(numElements);
    auto const s = static_cast<uint64_t>(sizeof(Node));
    auto const infos = static_cast<uint64_t>(calcNumBytesInfo(numElements));

    auto const total64 = ne * s + infos;
    auto const total = static_cast<size_t>(total64);

    if (ROBIN_HOOD_UNLIKELY(static_cast<uint64_t>(total) != total64)) {
      throwOverflowError();
    }
    return total;
#endif
  }

 private:
  template <typename Q = mapped_type>
  ROBIN_HOOD(NODISCARD)
  typename std::enable_if<!std::is_void<Q>::value, bool>::type has(const value_type &e) const {
    ROBIN_HOOD_TRACE(this)
    auto it = find(e.first);
    return it != end() && it->second == e.second;
  }

  template <typename Q = mapped_type>
  ROBIN_HOOD(NODISCARD)
  typename std::enable_if<std::is_void<Q>::value, bool>::type has(const value_type &e) const {
    ROBIN_HOOD_TRACE(this)
    return find(e) != end();
  }

  void reserve(size_t c, bool forceRehash) {
    ROBIN_HOOD_TRACE(this)
    auto const minElementsAllowed = (std::max)(c, mNumElements);
    auto newSize = InitialNumElements;
    while (calcMaxNumElementsAllowed(newSize) < minElementsAllowed && newSize != 0) {
      newSize *= 2;
    }
    if (ROBIN_HOOD_UNLIKELY(newSize == 0)) {
      throwOverflowError();
    }

    ROBIN_HOOD_LOG("newSize > mMask + 1: " << newSize << " > " << mMask << " + 1")

    // only actually do anything when the new size is bigger than the old one. This prevents to
    // continuously allocate for each reserve() call.
    if (forceRehash || newSize > mMask + 1) {
      rehashPowerOfTwo(newSize, false);
    }
  }

  // reserves space for at least the specified number of elements.
  // only works if numBuckets if power of two
  // True on success, false otherwise
  void rehashPowerOfTwo(size_t numBuckets, bool forceFree) {
    ROBIN_HOOD_TRACE(this)

    Node *const oldKeyVals = mKeyVals;
    uint8_t const *const oldInfo = mInfo;

    const size_t oldMaxElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1);

    // resize operation: move stuff
    initData(numBuckets);
    if (oldMaxElementsWithBuffer > 1) {
      for (size_t i = 0; i < oldMaxElementsWithBuffer; ++i) {
        if (oldInfo[i] != 0) {
          // might throw an exception, which is really bad since we are in the middle of
          // moving stuff.
          insert_move(std::move(oldKeyVals[i]));
          // destroy the node but DON'T destroy the data.
          oldKeyVals[i].~Node();
        }
      }

      // this check is not necessary as it's guarded by the previous if, but it helps
      // silence g++'s overeager "attempt to free a non-heap object 'map'
      // [-Werror=free-nonheap-object]" warning.
      if (oldKeyVals != reinterpret_cast_no_cast_align_warning<Node *>(&mMask)) {
        // don't destroy old data: put it into the pool instead
        if (forceFree) {
          hf3fs::memory::deallocate(oldKeyVals);
        } else {
          DataPool::addOrFree(oldKeyVals, calcNumBytesTotal(oldMaxElementsWithBuffer));
        }
      }
    }
  }

  ROBIN_HOOD(NOINLINE) void throwOverflowError() const {
#if ROBIN_HOOD(HAS_EXCEPTIONS)
    throw std::overflow_error("robin_hood::map overflow");
#else
    abort();
#endif
  }

  template <typename OtherKey, typename... Args>
  std::pair<iterator, bool> try_emplace_impl(OtherKey &&key, Args &&...args) {
    ROBIN_HOOD_TRACE(this)
    auto idxAndState = insertKeyPrepareEmptySpot(key);
    switch (idxAndState.second) {
      case InsertionState::key_found:
        break;

      case InsertionState::new_node:
        ::new (static_cast<void *>(&mKeyVals[idxAndState.first]))
            Node(*this,
                 std::piecewise_construct,
                 std::forward_as_tuple(std::forward<OtherKey>(key)),
                 std::forward_as_tuple(std::forward<Args>(args)...));
        break;

      case InsertionState::overwrite_node:
        mKeyVals[idxAndState.first] = Node(*this,
                                           std::piecewise_construct,
                                           std::forward_as_tuple(std::forward<OtherKey>(key)),
                                           std::forward_as_tuple(std::forward<Args>(args)...));
        break;

      case InsertionState::overflow_error:
        throwOverflowError();
        break;
    }

    return std::make_pair(iterator(mKeyVals + idxAndState.first, mInfo + idxAndState.first),
                          InsertionState::key_found != idxAndState.second);
  }

  template <typename OtherKey, typename Mapped>
  std::pair<iterator, bool> insertOrAssignImpl(OtherKey &&key, Mapped &&obj) {
    ROBIN_HOOD_TRACE(this)
    auto idxAndState = insertKeyPrepareEmptySpot(key);
    switch (idxAndState.second) {
      case InsertionState::key_found:
        mKeyVals[idxAndState.first].getSecond() = std::forward<Mapped>(obj);
        break;

      case InsertionState::new_node:
        ::new (static_cast<void *>(&mKeyVals[idxAndState.first]))
            Node(*this,
                 std::piecewise_construct,
                 std::forward_as_tuple(std::forward<OtherKey>(key)),
                 std::forward_as_tuple(std::forward<Mapped>(obj)));
        break;

      case InsertionState::overwrite_node:
        mKeyVals[idxAndState.first] = Node(*this,
                                           std::piecewise_construct,
                                           std::forward_as_tuple(std::forward<OtherKey>(key)),
                                           std::forward_as_tuple(std::forward<Mapped>(obj)));
        break;

      case InsertionState::overflow_error:
        throwOverflowError();
        break;
    }

    return std::make_pair(iterator(mKeyVals + idxAndState.first, mInfo + idxAndState.first),
                          InsertionState::key_found != idxAndState.second);
  }

  void initData(size_t max_elements) {
    mNumElements = 0;
    mMask = max_elements - 1;
    mMaxNumElementsAllowed = calcMaxNumElementsAllowed(max_elements);

    auto const numElementsWithBuffer = calcNumElementsWithBuffer(max_elements);

    // malloc & zero mInfo. Faster than calloc everything.
    auto const numBytesTotal = calcNumBytesTotal(numElementsWithBuffer);
    ROBIN_HOOD_LOG("std::calloc " << numBytesTotal << " = calcNumBytesTotal(" << numElementsWithBuffer << ")")
    mKeyVals = reinterpret_cast<Node *>(detail::assertNotNull<std::bad_alloc>(hf3fs::memory::allocate(numBytesTotal)));
    mInfo = reinterpret_cast<uint8_t *>(mKeyVals + numElementsWithBuffer);
    std::memset(mInfo, 0, numBytesTotal - numElementsWithBuffer * sizeof(Node));

    // set sentinel
    mInfo[numElementsWithBuffer] = 1;

    mInfoInc = InitialInfoInc;
    mInfoHashShift = InitialInfoHashShift;
  }

  enum class InsertionState { overflow_error, key_found, new_node, overwrite_node };

  // Finds key, and if not already present prepares a spot where to pot the key & value.
  // This potentially shifts nodes out of the way, updates mInfo and number of inserted
  // elements, so the only operation left to do is create/assign a new node at that spot.
  template <typename OtherKey>
  std::pair<size_t, InsertionState> insertKeyPrepareEmptySpot(OtherKey &&key) {
    for (int i = 0; i < 256; ++i) {
      size_t idx{};
      InfoType info{};
      keyToIdx(key, &idx, &info);
      nextWhileLess(&info, &idx);

      // while we potentially have a match
      while (info == mInfo[idx]) {
        if (WKeyEqual::operator()(key, mKeyVals[idx].getFirst())) {
          // key already exists, do NOT insert.
          // see http://en.cppreference.com/w/cpp/container/unordered_map/insert
          return std::make_pair(idx, InsertionState::key_found);
        }
        next(&info, &idx);
      }

      // unlikely that this evaluates to true
      if (ROBIN_HOOD_UNLIKELY(mNumElements >= mMaxNumElementsAllowed)) {
        if (!increase_size()) {
          return std::make_pair(size_t(0), InsertionState::overflow_error);
        }
        continue;
      }

      // key not found, so we are now exactly where we want to insert it.
      auto const insertion_idx = idx;
      auto const insertion_info = info;
      if (ROBIN_HOOD_UNLIKELY(insertion_info + mInfoInc > 0xFF)) {
        mMaxNumElementsAllowed = 0;
      }

      // find an empty spot
      while (0 != mInfo[idx]) {
        next(&info, &idx);
      }

      if (idx != insertion_idx) {
        shiftUp(idx, insertion_idx);
      }
      // put at empty spot
      mInfo[insertion_idx] = static_cast<uint8_t>(insertion_info);
      ++mNumElements;
      return std::make_pair(insertion_idx,
                            idx == insertion_idx ? InsertionState::new_node : InsertionState::overwrite_node);
    }

    // enough attempts failed, so finally give up.
    return std::make_pair(size_t(0), InsertionState::overflow_error);
  }

  bool try_increase_info() {
    ROBIN_HOOD_LOG("mInfoInc=" << mInfoInc << ", numElements=" << mNumElements
                               << ", maxNumElementsAllowed=" << calcMaxNumElementsAllowed(mMask + 1))
    if (mInfoInc <= 2) {
      // need to be > 2 so that shift works (otherwise undefined behavior!)
      return false;
    }
    // we got space left, try to make info smaller
    mInfoInc = static_cast<uint8_t>(mInfoInc >> 1U);

    // remove one bit of the hash, leaving more space for the distance info.
    // This is extremely fast because we can operate on 8 bytes at once.
    ++mInfoHashShift;
    auto const numElementsWithBuffer = calcNumElementsWithBuffer(mMask + 1);

    for (size_t i = 0; i < numElementsWithBuffer; i += 8) {
      auto val = unaligned_load<uint64_t>(mInfo + i);
      val = (val >> 1U) & UINT64_C(0x7f7f7f7f7f7f7f7f);
      std::memcpy(mInfo + i, &val, sizeof(val));
    }
    // update sentinel, which might have been cleared out!
    mInfo[numElementsWithBuffer] = 1;

    mMaxNumElementsAllowed = calcMaxNumElementsAllowed(mMask + 1);
    return true;
  }

  // True if resize was possible, false otherwise
  bool increase_size() {
    // nothing allocated yet? just allocate InitialNumElements
    if (0 == mMask) {
      initData(InitialNumElements);
      return true;
    }

    auto const maxNumElementsAllowed = calcMaxNumElementsAllowed(mMask + 1);
    if (mNumElements < maxNumElementsAllowed && try_increase_info()) {
      return true;
    }

    ROBIN_HOOD_LOG("mNumElements=" << mNumElements << ", maxNumElementsAllowed=" << maxNumElementsAllowed << ", load="
                                   << (static_cast<double>(mNumElements) * 100.0 / (static_cast<double>(mMask) + 1)))

    if (mNumElements * 2 < calcMaxNumElementsAllowed(mMask + 1)) {
      // we have to resize, even though there would still be plenty of space left!
      // Try to rehash instead. Delete freed memory so we don't steadyily increase mem in case
      // we have to rehash a few times
      nextHashMultiplier();
      rehashPowerOfTwo(mMask + 1, true);
    } else {
      // we've reached the capacity of the map, so the hash seems to work nice. Keep using it.
      rehashPowerOfTwo((mMask + 1) * 2, false);
    }
    return true;
  }

  void nextHashMultiplier() {
    // adding an *even* number, so that the multiplier will always stay odd. This is necessary
    // so that the hash stays a mixing function (and thus doesn't have any information loss).
    mHashMultiplier += UINT64_C(0xc4ceb9fe1a85ec54);
  }

  void destroy() {
    if (0 == mMask) {
      // don't deallocate!
      return;
    }

    Destroyer<Self, IsFlat && std::is_trivially_destructible<Node>::value>{}.nodesDoNotDeallocate(*this);

    // This protection against not deleting mMask shouldn't be needed as it's sufficiently
    // protected with the 0==mMask check, but I have this anyways because g++ 7 otherwise
    // reports a compile error: attempt to free a non-heap object 'fm'
    // [-Werror=free-nonheap-object]
    if (mKeyVals != reinterpret_cast_no_cast_align_warning<Node *>(&mMask)) {
      ROBIN_HOOD_LOG("hf3fs::memory::deallocate")
      hf3fs::memory::deallocate(mKeyVals);
    }
  }

  void init() noexcept {
    mKeyVals = reinterpret_cast_no_cast_align_warning<Node *>(&mMask);
    mInfo = reinterpret_cast<uint8_t *>(&mMask);
    mNumElements = 0;
    mMask = 0;
    mMaxNumElementsAllowed = 0;
    mInfoInc = InitialInfoInc;
    mInfoHashShift = InitialInfoHashShift;
  }

  // members are sorted so no padding occurs
  uint64_t mHashMultiplier = UINT64_C(0xc4ceb9fe1a85ec53);                  // 8 byte  8
  Node *mKeyVals = reinterpret_cast_no_cast_align_warning<Node *>(&mMask);  // 8 byte 16
  uint8_t *mInfo = reinterpret_cast<uint8_t *>(&mMask);                     // 8 byte 24
  size_t mNumElements = 0;                                                  // 8 byte 32
  size_t mMask = 0;                                                         // 8 byte 40
  size_t mMaxNumElementsAllowed = 0;                                        // 8 byte 48
  InfoType mInfoInc = InitialInfoInc;                                       // 4 byte 52
  InfoType mInfoHashShift = InitialInfoHashShift;                           // 4 byte 56
                                                                            // 16 byte 56 if NodeAllocator
};

}  // namespace detail

// map

template <typename Key,
          typename T,
          typename Hash = hash<Key>,
          typename KeyEqual = std::equal_to<Key>,
          size_t MaxLoadFactor100 = 80>
using unordered_flat_map = detail::Table<true, MaxLoadFactor100, Key, T, Hash, KeyEqual>;

template <typename Key,
          typename T,
          typename Hash = hash<Key>,
          typename KeyEqual = std::equal_to<Key>,
          size_t MaxLoadFactor100 = 80>
using unordered_node_map = detail::Table<false, MaxLoadFactor100, Key, T, Hash, KeyEqual>;

template <typename Key,
          typename T,
          typename Hash = hash<Key>,
          typename KeyEqual = std::equal_to<Key>,
          size_t MaxLoadFactor100 = 80>
using unordered_map = detail::Table<sizeof(robin_hood::pair<Key, T>) <= sizeof(size_t) * 6 &&
                                        std::is_nothrow_move_constructible<robin_hood::pair<Key, T>>::value &&
                                        std::is_nothrow_move_assignable<robin_hood::pair<Key, T>>::value,
                                    MaxLoadFactor100,
                                    Key,
                                    T,
                                    Hash,
                                    KeyEqual>;

// set

template <typename Key, typename Hash = hash<Key>, typename KeyEqual = std::equal_to<Key>, size_t MaxLoadFactor100 = 80>
using unordered_flat_set = detail::Table<true, MaxLoadFactor100, Key, void, Hash, KeyEqual>;

template <typename Key, typename Hash = hash<Key>, typename KeyEqual = std::equal_to<Key>, size_t MaxLoadFactor100 = 80>
using unordered_node_set = detail::Table<false, MaxLoadFactor100, Key, void, Hash, KeyEqual>;

template <typename Key, typename Hash = hash<Key>, typename KeyEqual = std::equal_to<Key>, size_t MaxLoadFactor100 = 80>
using unordered_set =
    detail::Table<sizeof(Key) <= sizeof(size_t) * 6 && std::is_nothrow_move_constructible<Key>::value &&
                      std::is_nothrow_move_assignable<Key>::value,
                  MaxLoadFactor100,
                  Key,
                  void,
                  Hash,
                  KeyEqual>;

}  // namespace robin_hood

#endif

#pragma once

#include <atomic>
#include <fmt/format.h>
#include <folly/Likely.h>
#include <folly/ThreadLocal.h>
#include <folly/concurrency/AtomicSharedPtr.h>
#include <functional>
#include <iomanip>
#include <map>
#include <memory>
#include <mutex>
#include <optional>
#include <set>
#include <span>
#include <sstream>
#include <type_traits>
#include <utility>
#include <vector>

#include "common/utils/AtomicValue.h"
#include "common/utils/MagicEnum.hpp"
#include "common/utils/Path.h"
#include "common/utils/Result.h"
#include "common/utils/RobinHood.h"
#include "common/utils/Toml.hpp"
#include "common/utils/TypeTraits.h"

DECLARE_string(cfg);

namespace hf3fs {

/*
 * ConfigBase class with macro.
 * The supported value types:
 *  - std::string
 *  - int64_t
 *  - double
 *  - bool
 *  - enum
 *  - std::vector of above types
 */

#define CONFIG_OBJ(name, cls, ...) /* optional parameter: initializer */                                     \
 public:                                                                                \
  cls &name() { return name##_; }                                                                            \
  const cls &name() const { return name##_; }                                                                \
                                                                                \
 private:                                                                                \
  cls name##_;                                                                                \
  [[maybe_unused]] bool name##Insert_ = [this]() {                                                           \
    using Self = std::decay_t<decltype(*this)>;                                                              \
    ConfigBase<Self>::sections_[#name] = reinterpret_cast<::hf3fs::config::IConfig Self::*>(&Self::name##_); \
    __VA_OPT__(__VA_ARGS__(name##_);)                                                                        \
    return true;                                                                                \
  }()

#define CONFIG_OBJ_ARRAY(name, cls, cap, ...) /* optional parameter: initializer */             \
 public:                                                                                \
  cls &name(size_t idx) { return name##_[idx]; }                                                \
  const cls &name(size_t idx) const { return name##_[idx]; }                                    \
  size_t name##_length() const {                                                                \
    auto lock = std::unique_lock(mutex_);                                                       \
    return name##Length_;                                                                       \
  }                                                                                \
  void set_##name##_length(size_t len) {                                                        \
    auto lock = std::unique_lock(mutex_);                                                       \
    if (len < k_capacity_##name) {                                                              \
      name##Length_ = len;                                                                      \
    }                                                                                \
  }                                                                                \
                                                                                \
 private:                                                                                \
  static constexpr size_t k_capacity_##name{cap};                                               \
  static_assert(k_capacity_##name <= 4096u, "array cap should less than 4096");                 \
  std::array<cls, k_capacity_##name> name##_;                                                   \
  size_t name##Length_ = [this]() {                                                             \
    using Self = std::decay_t<decltype(*this)>;                                                 \
    ConfigBase<Self>::lengths_[#name] = reinterpret_cast<size_t Self::*>(&Self::name##Length_); \
    auto base = reinterpret_cast<::hf3fs::config::IConfig Self::*>(&Self::name##_);             \
    static_assert(sizeof(base) == sizeof(uintptr_t), "sizeof(base) != sizeof(uintptr_t)");      \
    for (auto i = 0ul; i < k_capacity_##name; ++i) {                                            \
      ConfigBase<Self>::sections_[fmt::format(#name "#{}", i)] = base;                          \
      *reinterpret_cast<uintptr_t *>(&base) += sizeof(cls);                                     \
    }                                                                                \
    auto length = 1ul;                                                                          \
    __VA_OPT__(length = __VA_ARGS__(name##_);)                                                  \
    return length;                                                                              \
  }()

#define CONFIG_SECT(name, section)                     \
 protected:                                            \
  struct T##name : public ConfigBase<T##name> section; \
  CONFIG_OBJ(name, T##name)

#define CONFIG_ADD_ITEM(name, defaultValue, supportHotUpdated, ...) /* optional parameter: checker */   \
 private:                                                                                \
  using T##name = ::hf3fs::config::ValueType<std::decay_t<decltype(defaultValue)>>;                     \
  using R##name = ::hf3fs::config::ReturnType<T##name>;                                                 \
                                                                                \
 public:                                                                                \
  auto name##_getter() const {                                                                          \
    return [this] { return name(); };                                                                   \
  }                                                                                \
  R##name name() const { return name##_.value(); }                                                      \
  bool set_##name(R##name value) { return name##_.checkAndSet(value); }                                 \
                                                                                \
 private:                                                                                \
  ::hf3fs::config::Item<T##name> name##_ = ::hf3fs::config::Item<T##name>(#name, defaultValue, [this] { \
    using Self = std::decay_t<decltype(*this)>;                                                         \
    ConfigBase<Self>::items_[#name] = reinterpret_cast<::hf3fs::config::IItem Self::*>(&Self::name##_); \
    return supportHotUpdated;                                                                           \
  }() __VA_OPT__(, ) __VA_ARGS__)

#define CONFIG_ITEM(name, defaultValue, ...) CONFIG_ADD_ITEM(name, defaultValue, false, __VA_ARGS__)
#define CONFIG_HOT_UPDATED_ITEM(name, defaultValue, ...) CONFIG_ADD_ITEM(name, defaultValue, true, __VA_ARGS__)

#define CONFIG_VARIANT_TYPE(defaultType)                                        \
  CONFIG_ITEM(type, std::string{defaultType}, [this](const std::string &name) { \
    using Self = std::decay_t<decltype(*this)>;                                 \
    return ConfigBase<Self>::sections_.count(name);                             \
  });                                                                           \
                                                                                \
 public:                                                                        \
  constexpr static inline bool is_variant_type() { return true; }

namespace config {

struct IItem {
  virtual ~IItem() = default;
  virtual Result<Void> validate(const std::string &path) const = 0;
  virtual Result<Void> update(const toml::node &node, bool isHotUpdate, const std::string &path) = 0;
  virtual void toToml(toml::table &table) const = 0;
  virtual bool isParsedFromString() const = 0;
  virtual bool operator==(const IItem &other) const = 0;
  virtual String toString() const = 0;
};

class ConfigCallbackGuard;

using KeyValue = std::pair<std::string, std::string>;

inline std::string tomlToString(const toml::node &node) {
  std::stringstream ss;
  ss << toml::toml_formatter(node, toml::toml_formatter::default_flags & ~toml::format_flags::indentation);
  return ss.str();
}

struct IConfig {
  virtual ~IConfig() = default;

  virtual std::unique_ptr<IConfig> clonePtr() const = 0;

  // return a default constructed object
  virtual std::unique_ptr<IConfig> defaultPtr() const = 0;

  // validate configuration items one by one.
  virtual Result<Void> validate(const std::string &path = {}) const = 0;

  // validate configuration items as a whole.
  virtual Result<Void> overallValidate() const { return Void{}; }

  // update configuration. [thread safe]
  virtual Result<Void> update(const toml::table &table, bool isHotUpdate = true, const std::string &path = {}) = 0;

  // update configuration from a string. [thread safe]
  Result<Void> update(std::string_view str, bool isHotUpdate);

  // update configuration from a file. [thread safe]
  Result<Void> update(const Path &path, bool isHotUpdate);

  // update configuration from a series of key-values. [thread safe]
  Result<Void> update(const std::vector<KeyValue> &updates, bool isHotUpdate);

  // remove callback guard.
  virtual void removeCallbackGuard(ConfigCallbackGuard *guard) const = 0;

  // convert configuration to TOML. [thread safe]
  virtual toml::table toToml() const = 0;

  // convert configuration of specific section/item to TOML. [thread safe]
  virtual Result<toml::table> toToml(std::string_view key) const = 0;

  // find item by a key.
  virtual Result<IItem *> find(std::string_view key) = 0;

  virtual bool operator==(const IConfig &other) const = 0;

  struct ItemDiff {
    String key;
    String left;
    String right;
  };

  size_t diffWith(const IConfig &other, std::span<ItemDiff> diffs, size_t pos = 0) const {
    return diffWith(other, {}, diffs, pos);
  }

  virtual size_t diffWith(const IConfig &other, String path, std::span<ItemDiff> diffs, size_t pos) const = 0;

  virtual Result<Void> atomicallyUpdate(std::string_view str, bool isHotUpdate) = 0;
  Result<Void> atomicallyUpdate(std::string_view str) { return atomicallyUpdate(str, /*isHotUpdate=*/true); }

  virtual Result<Void> atomicallyUpdate(const Path &path, bool isHotUpdate) = 0;
  Result<Void> atomicallyUpdate(const Path &path) { return atomicallyUpdate(path, /*isHotUpdate=*/true); }

  virtual Result<Void> atomicallyUpdate(const std::vector<KeyValue> &updates, bool isHotUpdate) = 0;
  Result<Void> atomicallyUpdate(const std::vector<KeyValue> &updates) {
    return atomicallyUpdate(updates, /*isHotUpdate=*/true);
  }

  virtual Result<Void> validateUpdate(std::string_view str, bool isHotUpdate) = 0;
  Result<Void> validateUpdate(std::string_view str) { return validateUpdate(str, /*isHotUpdate=*/true); }

  // convert configuration to string. [thread safe]
  std::string toString() const { return tomlToString(toToml()); }

  // initialize config from command line and config files.
  Result<Void> init(int *argc, char ***argv, bool follyInit = true);
};

template <class T>
struct ConfigValueToTomlNode {
  template <class V>
  auto operator()(V &&v) {
    if constexpr (std::is_enum_v<T>) {
      return magic_enum::enum_name(std::forward<V>(v));
    } else if constexpr (std::is_same_v<T, uint64_t>) {
      return static_cast<int64_t>(v);
    } else if constexpr (std::is_same_v<T, Path>) {
      return v.string();
    } else if constexpr (std::derived_from<T, IConfig>) {
      return v.toToml();
    } else if constexpr (requires { std::string(v.toString()); }) {
      return v.toString();
    } else {
      return std::forward<V>(v);
    }
  }
};

Result<std::vector<KeyValue>> parseFlags(std::string_view prefix, int &argc, char *argv[]);

class ConfigCallbackGuard {
 public:
  explicit ConfigCallbackGuard(const config::IConfig *cfg, auto &&...f)
      : cfg_(cfg),
        func_(std::forward<decltype(f)>(f)...) {}
  ~ConfigCallbackGuard() { dismiss(); }

  void setCallback(auto &&f) { func_ = std::forward<decltype(f)>(f); }
  void callCallback() { func_ ? func_() : void(); }
  void dismiss() { cfg_ ? std::exchange(cfg_, nullptr)->removeCallbackGuard(this) : void(); }

 private:
  const config::IConfig *cfg_;
  std::function<void()> func_;
};

template <class T>
class TLSStore {
 public:
  TLSStore(T &&value)
      : ptr_(std::make_shared<T>(std::move(value))) {}
  TLSStore(const TLSStore &o)
      : ptr_(o.ptr_.load(std::memory_order_acquire)) {}

  const T &value() const {
    auto &cache = *tlsCache_;
    size_t latest = version_.load(std::memory_order_acquire);
    if (UNLIKELY(cache.version != latest)) {
      cache.ptr = ptr_.load(std::memory_order_acquire);
      cache.version = latest;
    }
    return *cache.ptr;
  }

  template <class V>
  void setValue(V &&value) {
    ptr_.store(std::make_shared<T>(std::forward<V>(value)));
    ++version_;
  }

  TLSStore &operator=(const TLSStore &o) {
    ptr_.store(o.ptr_.load(std::memory_order_acquire));
    ++version_;
    return *this;
  }

 private:
  std::atomic<uint64_t> version_ = 1;
  folly::atomic_shared_ptr<T> ptr_{std::make_shared<T>()};
  struct Cache {
    std::shared_ptr<const T> ptr;
    uint64_t version = 0;
  };
  folly::ThreadLocal<Cache> tlsCache_;
};

template <class T>
using RemoveOptional = typename std::conditional_t<is_optional_v<T>, T, std::optional<T>>::value_type;
template <class T>
inline constexpr bool IsPrimitive = std::is_trivially_copyable_v<T> && sizeof(T) <= 8;
template <class T>
using ReturnType = std::conditional_t<IsPrimitive<T>, T, const T &>;
template <class T>
using StoreType = std::conditional_t<IsPrimitive<T>, AtomicValue<T>, TLSStore<T>>;
template <class T>
using ValueType = std::conditional_t<std::is_same_v<T, const char *>, std::string, T>;

template <typename T>
inline Result<T> tomlNodeToValue(const toml::node &node) {
  return node.visit([&](auto &&el) -> Result<T> {
    using TE = std::decay_t<decltype(el)>;
    if constexpr (!toml::is_value<TE>) {
      return makeError(StatusCode::kConfigInvalidType, fmt::format("{} isn't value", typeid(TE).name()));
    } else {
      using E = typename TE::value_type;

      if constexpr (std::is_same_v<T, E>) {
        return *el;
      } else if constexpr (std::is_same_v<T, bool>) {
        // do not allow any implicit conversion to bool
        return makeError(StatusCode::kConfigInvalidType,
                         fmt::format("implicit conversion from {} to bool is not allowed", typeid(E).name()));
      } else if constexpr (std::is_floating_point_v<T>) {
        if constexpr (std::is_same_v<E, int64_t> || std::is_floating_point_v<E>) {
          return *el;
        } else {
          return makeError(StatusCode::kConfigInvalidType,
                           fmt::format("{} is not int64_t or floating types", typeid(E).name()));
        }
      } else if constexpr (std::is_enum_v<T>) {
        if constexpr (std::is_same_v<E, std::string>) {
          auto opt = magic_enum::enum_cast<T>(*el);
          if (opt) {
            return opt.value();
          } else {
            return makeError(StatusCode::kConfigInvalidValue, fmt::format("Value {}", *el));
          }
        } else {
          return makeError(StatusCode::kConfigInvalidType,
                           fmt::format("{} is enum but {} is not string", typeid(T).name(), typeid(E).name()));
        }
      } else if constexpr (requires { Result<T>{T::from(*el)}; }) {
        return T::from(*el);
      } else if constexpr (std::is_constructible_v<T, E>) {
        try {
          return T(*el);
        } catch (const std::exception &e) {
          return makeError(
              StatusCode::kConfigInvalidType,
              fmt::format("Throws when constructing T ({}), E is {}: ", typeid(T).name(), typeid(E).name(), e.what()));
        }
      } else {
        return makeError(StatusCode::kConfigInvalidType,
                         fmt::format("T is {}, E is {}", typeid(T).name(), typeid(E).name()));
      }
    }
  });
}

template <class T>
class Item : public IItem {
 public:
  Item(std::string name, T defaultValue, bool supportHotUpdate, std::function<bool(ReturnType<T>)> checker = nullptr)
      : value_(std::move(defaultValue)),
        name_(std::move(name)),
        supportHotUpdate_(supportHotUpdate),
        checker_(checker ? std::move(checker) : [](ReturnType<T>) { return true; }) {}

  ReturnType<T> value() const { return value_.value(); }

  template <class V>
  void setValue(V &&value) {
    value_.setValue(std::forward<V>(value));
  }

  bool operator==(const IItem &other) const final {
    if (typeid(*this) != typeid(other)) {
      return false;
    }
    return *this == *reinterpret_cast<const Item *>(&other);
  }

  bool operator==(const Item &other) const { return this == std::addressof(other) || value() == other.value(); }

  bool operator==(const T &val) const { return value() == val; }

  String toString() const final {
    toml::table table;
    toToml(table);
    auto *view = table.get(name_);
    if constexpr (is_optional_v<T>) {
      return view ? tomlToString(*view) : "nullopt";
    } else {
      return tomlToString(*view);
    }
  }

  bool checkAndSet(ReturnType<T> value) {
    if (checker_(value)) {
      setValue(value);
      return true;
    }
    return false;
  }

  Result<Void> update(const toml::node &node, bool isHotUpdate, const std::string &path) final {
    try {
      bool enableUpdate = supportHotUpdate_ || !isHotUpdate;
      if constexpr (is_vector_v<T> || is_set_v<T>) {
        return updateVectorOrSet(node, enableUpdate, path);
      } else if constexpr (is_map_v<T>) {
        return updateMap(node, enableUpdate, path);
      } else {
        return updateNormal(node, enableUpdate, path);
      }
    } catch (const std::exception &e) {
      return makeError(StatusCode::kConfigUpdateFailed, fmt::format("name: {}, error: {}", path, e.what()));
    }
  }

  Result<Void> validate(const std::string &path) const final {
    if (!checker_(value())) {
      return makeError(StatusCode::kConfigValidateFailed, fmt::format("Check failed: {}", path));
    }
    return Void{};
  }

  void toToml(toml::table &table) const override {
    if constexpr (is_vector_v<T> || is_set_v<T>) {
      toml::array array;
      for (const auto &item : value()) {
        array.emplace_back(ConfigValueToTomlNode<typename T::value_type>{}(item));
      }
      table.insert_or_assign(name_, std::move(array));
    } else if constexpr (is_map_v<T>) {
      toml::table inlineTable;
      for (const auto &pair : value()) {
        inlineTable.emplace(pair.first, ConfigValueToTomlNode<typename T::mapped_type>{}(pair.second));
      }
      inlineTable.is_inline(true);
      table.insert_or_assign(name_, std::move(inlineTable));
    } else if constexpr (is_optional_v<T>) {
      if (value().has_value()) {
        table.insert_or_assign(name_, ConfigValueToTomlNode<RemoveOptional<T>>{}(value().value()));
      }
    } else {
      table.insert_or_assign(name_, ConfigValueToTomlNode<T>{}(value()));
    }
  }

  bool isParsedFromString() const override {
    return std::is_constructible_v<RemoveOptional<T>, std::string> || std::is_enum_v<RemoveOptional<T>>;
  }

 protected:
  Result<Void> updateNormal(const toml::node &node, bool enableUpdate, const std::string &path) {
    try {
      auto res = tomlNodeToValue<RemoveOptional<T>>(node);
      if (res.hasError()) {
        return makeError(StatusCode::kConfigUpdateFailed, fmt::format("name: {}, error: {}", path, res.error()));
      }
      if (!checker_(res.value())) {
        return makeError(StatusCode::kConfigValidateFailed, fmt::format("Check failed: {}", path));
      }
      if (res.value() != value()) {
        if (enableUpdate) {
          setValue(std::move(res.value()));
        } else {
          return makeError(StatusCode::kConfigUpdateFailed, fmt::format("Not support hot update: {}", path));
        }
      }
      return Void{};
    } catch (const std::exception &e) {
      return makeError(StatusCode::kConfigUpdateFailed, fmt::format("throws when update {}: {}", path, e.what()));
    }
  }

  Result<Void> updateVectorOrSet(const toml::node &node, bool enableUpdate, const std::string &path) {
    using I = typename T::value_type;

    if (!node.is_array()) {
      return makeError(StatusCode::kConfigInvalidType, fmt::format("Not array: {}", path));
    }
    auto arr = node.as_array();
    if (arr == nullptr) {
      return makeError(StatusCode::kConfigInvalidValue, fmt::format("Empty array: {}", path));
    }

    T tmp;
    for (size_t i = 0; i < arr->size(); ++i) {
      const auto &e = (*arr)[i];
      auto res = [&]() -> Result<I> {
        if constexpr (std::derived_from<I, IConfig>) {
          if (!e.is_table()) {
            return makeError(StatusCode::kConfigInvalidType, fmt::format("Not table: {}[{}]", path, i));
          }
          I v;
          RETURN_ON_ERROR(v.update(*e.as_table(), /*isHotUpdate=*/false));
          return v;
        } else {
          return tomlNodeToValue<RemoveOptional<I>>(e);
        }
      }();
      if (res.hasError()) {
        return makeError(StatusCode::kConfigUpdateFailed, fmt::format("name: {}, error: {}", path, res.error()));
      }
      if constexpr (is_set_v<T>) {
        auto [it, succ] = tmp.emplace(std::move(res.value()));
        if (!succ) {
          return makeError(StatusCode::kConfigUpdateFailed, fmt::format("name: {}, set value repeats", path));
        }
      } else {
        tmp.push_back(std::move(res.value()));
      }
    }
    if (!checker_(tmp)) {
      return makeError(StatusCode::kConfigValidateFailed, fmt::format("Array check failed: {}", path));
    }
    if (tmp != value()) {
      if (enableUpdate) {
        value_.setValue(std::move(tmp));
      } else {
        return makeError(StatusCode::kConfigUpdateFailed, fmt::format("Not support hot update: {}", path));
      }
    }
    return Void{};
  }

  Result<Void> updateMap(const toml::node &node, bool enableUpdate, const std::string &path) {
    using I = typename T::mapped_type;

    if (!node.is_table()) {
      return makeError(StatusCode::kConfigInvalidType, fmt::format("Not map: {}", path));
    }
    auto table = node.as_table();
    if (table == nullptr) {
      return makeError(StatusCode::kConfigInvalidValue, fmt::format("Empty map: {}", path));
    }

    T tmp;
    for (const auto &e : *table) {
      auto res = tomlNodeToValue<I>(e.second);
      if (res.hasError()) {
        return makeError(StatusCode::kConfigUpdateFailed, fmt::format("name: {}, error: {}", path, res.error()));
      }
      auto [it, succ] = tmp.emplace(std::string{e.first.str()}, std::move(res.value()));
      if (!succ) {
        return makeError(StatusCode::kConfigRedundantKey,
                         fmt::format("name: {}, redundant key: {}", path, e.first.str()));
      }
    }
    if (!checker_(tmp)) {
      return makeError(StatusCode::kConfigValidateFailed, fmt::format("Array check failed: {}", path));
    }
    if (tmp != value()) {
      if (enableUpdate) {
        value_.setValue(std::move(tmp));
      } else {
        return makeError(StatusCode::kConfigUpdateFailed, fmt::format("Not support hot update: {}", path));
      }
    }
    return Void{};
  }

 private:
  StoreType<T> value_;
  std::string name_;
  bool supportHotUpdate_ = false;
  std::function<bool(ReturnType<T>)> checker_;
};

inline std::string concat(const std::string &a, const std::string &b) { return a.empty() ? b : a + "." + b; }

}  // namespace config

using config::ConfigCallbackGuard;

template <class Parent>
class ConfigBase : public config::IConfig {
 protected:
  ConfigBase() = default;
  ConfigBase(const ConfigBase &o)
      : sections_(o.sections_),
        items_(o.items_),
        lengths_(o.lengths_) {}
  ConfigBase &operator=(const ConfigBase &) { return *this; }

 public:
  using config::IConfig::update;
  Result<Void> update(const toml::table &table, bool isHotUpdate = true, const std::string &path = {}) final {
    auto self = reinterpret_cast<Parent *>(this);
    auto lock = std::unique_lock(mutex_);

    for (auto &pair : table) {
      auto name = std::string(pair.first.str());
      auto &node = pair.second;

      if (lengths().count(name)) {
        if (!(node.is_array_of_tables() || (node.is_array() && node.as_array()->empty()))) {
          return makeError(StatusCode::kConfigInvalidType, fmt::format("Not array of table: {}", name));
        }
        auto array = node.as_array();
        auto size = array->size();
        for (auto i = 0ul; i < size; ++i) {
          auto itemName = fmt::format("{}#{}", name, i);
          auto itemTable = array->get_as<toml::table>(i);
          if (itemTable == nullptr) {
            return makeError(StatusCode::kConfigInvalidType, fmt::format("Not a table: {}", itemName));
          }
          if (!sections().count(itemName)) {
            return makeError(StatusCode::kConfigInvalidType, fmt::format("Array length exceed: {}", itemName));
          }
          RETURN_ON_ERROR(
              (self->*(sections().at(itemName))).update(*itemTable, isHotUpdate, config::concat(path, itemName)));
        }
        self->*(lengths().at(name)) = size;
      } else if (sections().count(name)) {
        if (!node.is_table()) {
          return makeError(StatusCode::kConfigInvalidType, fmt::format("Not table: {}", name));
        }
        RETURN_ON_ERROR(
            (self->*(sections().at(name))).update(*node.as_table(), isHotUpdate, config::concat(path, name)));
      } else if (items().count(name)) {
        RETURN_ON_ERROR((self->*(items().at(name))).update(node, isHotUpdate, config::concat(path, name)));
      } else {
        return makeError(StatusCode::kConfigRedundantKey, fmt::format("Invalid key: {}", config::concat(path, name)));
      }
    }

    // call callbacks after update.
    for (auto &callback : callbacks_) {
      callback->callCallback();
    }
    return overallValidate();
  }

  toml::table toToml() const final {
    auto lock = std::unique_lock(mutex_);

    auto self = reinterpret_cast<const Parent *>(this);
    toml::table table;
    for (auto &pair : items()) {
      (self->*pair.second).toToml(table);
    }
    if constexpr (requires { Parent::is_variant_type; }) {
      auto it = sections().find(self->type());
      if (it != sections().end()) {
        table.insert_or_assign(it->first, (self->*it->second).toToml());
      }
      return table;
    }
    for (auto &pair : sections()) {
      if (pair.first.find('#') != std::string::npos) {
        continue;
      }
      table.insert_or_assign(pair.first, (self->*pair.second).toToml());
    }
    for (auto &pair : lengths()) {
      toml::array array;
      for (auto i = 0ul; i < self->*pair.second; ++i) {
        array.emplace_back((self->*sections().at(fmt::format("{}#{}", pair.first, i))).toToml());
      }
      table.insert_or_assign(pair.first, std::move(array));
    }
    return table;
  }

  Result<toml::table> toToml(std::string_view key) const final {
    if (key.empty()) return toToml();

    auto self = reinterpret_cast<const Parent *>(this);

    auto pos = key.find('.');
    if (pos == std::string_view::npos) {
      auto iit = items().find(key);
      if (iit != items().end()) {
        toml::table table;
        (self->*(iit->second)).toToml(table);
        return table;
      }
      auto sit = sections().find(key);
      if (sit != sections().end()) {
        return (self->*(sit->second)).toToml();
      }
      return makeError(StatusCode::kConfigKeyNotFound, key);
    } else {
      auto section = key.substr(0, pos);
      key.remove_prefix(pos + 1);
      auto it = sections().find(section);
      if (it != sections().end()) {
        return (self->*(it->second)).toToml(key);
      }
      return makeError(StatusCode::kConfigKeyNotFound, section);
    }
  }

  Result<config::IItem *> find(std::string_view key) final {
    auto self = reinterpret_cast<Parent *>(this);

    auto pos = key.find('.');
    if (pos == std::string_view::npos) {
      auto it = items().find(std::string{key});
      if (it == items().end()) {
        return makeError(StatusCode::kConfigValidateFailed);
      }
      return &(self->*(it->second));
    } else {
      auto section = key.substr(0, pos);
      key.remove_prefix(pos + 1);
      auto it = sections().find(std::string{section});
      if (it == sections().end()) {
        return makeError(StatusCode::kConfigValidateFailed);
      }
      return (self->*(it->second)).find(key);
    }
  }

  std::unique_ptr<config::IConfig> clonePtr() const final { return std::make_unique<Parent>(clone()); }

  std::unique_ptr<config::IConfig> defaultPtr() const final { return std::make_unique<Parent>(); }

  // clone current configuration. [thread safe]
  Parent clone() const {
    auto lock = std::unique_lock(mutex_);
    return reinterpret_cast<const Parent &>(*this);
  }

  // copy from another configuration. [thread safe]
  void copy(const Parent &o) {
    std::scoped_lock lock(mutex_, o.mutex_);
    reinterpret_cast<Parent &>(*this) = o;
  }

  // atomically update configuration from a string.
  using IConfig::atomicallyUpdate;
  Result<Void> atomicallyUpdate(std::string_view str, bool isHotUpdate) final {
    Parent newConfig = clone();
    RETURN_ON_ERROR(newConfig.update(str, isHotUpdate));
    auto res = update(str, isHotUpdate);
    XLOGF_IF(FATAL, !res, "Unexpected update error: {}", res.error());
    return Void{};
  }

  // atomically update configuration from a file.
  Result<Void> atomicallyUpdate(const Path &path, bool isHotUpdate) final {
    Parent newConfig = clone();
    RETURN_ON_ERROR(newConfig.IConfig::update(path, isHotUpdate));
    auto res = update(path, isHotUpdate);
    XLOGF_IF(FATAL, !res, "Unexpected update error: {}", res.error());
    return Void{};
  }

  Result<Void> atomicallyUpdate(const std::vector<config::KeyValue> &updates, bool isHotUpdate) final {
    Parent newConfig = clone();
    RETURN_ON_ERROR(newConfig.IConfig::update(updates, isHotUpdate));
    auto res = update(updates, isHotUpdate);
    XLOGF_IF(FATAL, !res, "Unexpected update error: {}", res.error());
    return Void{};
  }

  Result<Void> validateUpdate(std::string_view str, bool isHotUpdate) final {
    Parent newConfig = clone();
    RETURN_ON_ERROR(newConfig.IConfig::update(str, isHotUpdate));
    return Void{};
  }

  Result<Void> validate(const std::string &path = {}) const final {
    auto self = reinterpret_cast<const Parent *>(this);
    for (auto &pair : sections()) {
      RETURN_ON_ERROR((self->*pair.second).validate(config::concat(path, pair.first)));
    }
    for (auto &pair : items()) {
      RETURN_ON_ERROR((self->*pair.second).validate(config::concat(path, pair.first)));
    }
    return overallValidate();
  }

  // add callback guard.
  std::unique_ptr<ConfigCallbackGuard> addCallbackGuard(auto &&...func) const {
    auto guard = std::make_unique<ConfigCallbackGuard>(this, std::forward<decltype(func)>(func)...);
    auto lock = std::unique_lock(mutex_);
    callbacks_.insert(guard.get());
    return guard;
  }

  bool operator==(const config::IConfig &other) const final {
    if (typeid(*this) != typeid(other)) {
      return false;
    }
    if (this == std::addressof(other)) {
      return true;
    }
    return *this == *reinterpret_cast<const Parent *>(&other);
  }

  bool operator==(const Parent &other) const {
    ItemDiff diffs[1];
    return diffWith(other, std::span(diffs)) == 0;
  }

  using config::IConfig::diffWith;
  size_t diffWith(const config::IConfig &other, String path, std::span<ItemDiff> diffs, size_t pos) const final {
    XLOGF_IF(FATAL,
             typeid(*this) != typeid(other),
             "Call diffWith on different config types: {} and {}",
             typeid(*this).name(),
             typeid(other).name());
    if (this == std::addressof(other)) {
      return pos;
    }
    return diffWith(*reinterpret_cast<const Parent *>(&other), std::move(path), diffs, pos);
  }

  size_t diffWith(const Parent &other, String path, std::span<ItemDiff> diffs, size_t pos) const {
    auto &self = *reinterpret_cast<const Parent *>(this);
    auto makePath = [&path](const String &name) { return path.empty() ? name : fmt::format("{}.{}", path, name); };

    for (const auto &[name, item] : items()) {
      auto &selfItem = self.*item;
      auto &otherItem = other.*item;
      if (!(selfItem == otherItem)) {
        diffs[pos++] = ItemDiff{makePath(name), selfItem.toString(), otherItem.toString()};
        if (pos >= diffs.size()) return pos;
      }
    }
    for (const auto &[name, sec] : sections()) {
      auto &selfSec = self.*sec;
      auto &otherSec = other.*sec;
      pos = selfSec.diffWith(otherSec, makePath(name), diffs, pos);
      if (pos >= diffs.size()) return pos;
    }
    return pos;
  }

 protected:
  // remove callback guard.
  void removeCallbackGuard(ConfigCallbackGuard *guard) const final {
    auto lock = std::unique_lock(mutex_);
    callbacks_.erase(guard);
  }

  const auto &sections() const noexcept { return sections_; }
  const auto &items() const noexcept { return items_; }
  const auto &lengths() const noexcept { return lengths_; }

 protected:
  mutable std::mutex mutex_;
  // offsets of sections.
  std::map<std::string, config::IConfig Parent::*, std::less<>> sections_;
  // offsets of items.
  std::map<std::string, config::IItem Parent::*, std::less<>> items_;
  // length of section arrays.
  std::map<std::string, size_t Parent::*, std::less<>> lengths_;
  // callbacks after update.
  mutable std::set<ConfigCallbackGuard *, std::less<>> callbacks_;
};

namespace ConfigCheckers {
template <typename T>
concept Arithmetic = std::is_arithmetic_v<T>;

template <typename T>
concept Container = requires(T t) {
  t.empty();
  t.size();
};

template <Arithmetic T, T threshold>
inline bool checkGE(T val) {
  return val >= threshold;
}

struct CheckPositive {
  template <Arithmetic T>
  bool operator()(T val) const {
    return val > 0;
  }
};

inline constexpr CheckPositive checkPositive;

template <Arithmetic T>
inline bool isPositivePrime(T n) {
  if (n < 2) return false;
  for (T f = 2; f * f <= n; f++)
    if (n % f == 0) return false;
  return true;
}

template <Arithmetic T>
inline bool checkNotNegative(T val) {
  return val >= 0;
}

struct CheckNotEmpty {
  template <Container C>
  bool operator()(const C &c) const {
    return !c.empty();
  }
};

inline constexpr CheckNotEmpty checkNotEmpty;

template <Container C>
inline bool checkEmpty(const C &c) {
  return c.empty();
}
}  // namespace ConfigCheckers

}  // namespace hf3fs

FMT_BEGIN_NAMESPACE

template <>
struct formatter<hf3fs::config::IConfig> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(const hf3fs::config::IConfig &config, FormatContext &ctx) const {
    return formatter<std::string_view>::format(config.toString(), ctx);
  }
};

FMT_END_NAMESPACE

#pragma once

#include <boost/algorithm/string.hpp>
#include <boost/core/ignore_unused.hpp>
#include <boost/filesystem/string_file.hpp>
#include <common/utils/UtcTime.h>
#include <folly/experimental/coro/Collect.h>
#include <folly/futures/Barrier.h>
#include <folly/stats/TDigest.h>
#include <numeric>
#include <optional>
#include <random>
#include <vector>
#include <fstream>

#include "common/logging/LogInit.h"
#include "common/net/ib/IBDevice.h"
#include "common/utils/Duration.h"
#include "common/utils/SysResource.h"
#include "tests/lib/UnitTestFabric.h"

namespace hf3fs::storage::benchmark {

using namespace hf3fs::storage::client;

class StorageBench : public test::UnitTestFabric {
 public:
  struct Options {
    const size_t numChunks;
    const size_t readSize;
    const size_t writeSize;
    const size_t batchSize;
    const uint64_t numReadSecs;
    const uint64_t numWriteSecs;
    const uint64_t clientTimeoutMS;
    const size_t numCoroutines;
    const size_t numTestThreads;
    const uint32_t randSeed = 0;
    const uint16_t chunkIdPrefix = 0xFFFF;
    const bool benchmarkNetwork = false;
    const bool benchmarkStorage = false;
    const bool ignoreIOError = false;
    const bool injectRandomServerError = false;
    const bool injectRandomClientError = false;
    const bool retryPermanentError = false;
    const bool verifyReadData = false;
    const bool verifyReadChecksum = false;
    const bool verifyWriteChecksum = true;
    const bool randomShuffleChunkIds = true;
    const bool generateTestData = true;
    const bool sparseChunkIds = true;
    const std::string statsFilePath = "./perfstats.csv";
    const std::vector<std::string> ibvDevices = {};
    const std::vector<std::string> ibnetZones = {};
    const std::vector<net::Address> mgmtdEndpoints = {};
    const std::string clusterId = kClusterId;
    const uint32_t chainTableId = 0;
    const uint32_t chainTableVersion = 0;
    const std::vector<uint32_t> chainIds = {};
    const std::vector<uint32_t> storageNodeIds = {};
    const size_t memoryAlignment = 1;
    const size_t readOffAlignment = 0;
    const size_t defaultPKeyIndex = 1;
    size_t readBatchSize = 0;
    size_t writeBatchSize = 0;
    size_t removeBatchSize = 0;
  };

 private:
  static constexpr uint32_t kTDigestMaxSize = 1000;

  struct ChunkInfo {
    ChainId chainId;
    ChunkId chunkId;
    size_t size;
  };

  Options benchOptions_;
  std::vector<folly::TDigest> writeLatencyDigests_;
  std::vector<folly::TDigest> readLatencyDigests_;
  folly::CPUThreadPoolExecutor testExecutor_;
  std::atomic_uint64_t numWriteBytes_;
  std::atomic_uint64_t numReadBytes_;
  folly::Random::DefaultGenerator randGen_;
  std::vector<std::vector<ChunkInfo>> chunkInfos_;
  std::vector<size_t> numCreatedChunks_;
  size_t totalNumChunks_;
  double totalChunkGiB_;

 public:
  StorageBench(const test::SystemSetupConfig &setupConfig, const Options &options)
      : UnitTestFabric(setupConfig),
        benchOptions_(options),
        writeLatencyDigests_(benchOptions_.numCoroutines, folly::TDigest(kTDigestMaxSize)),
        readLatencyDigests_(benchOptions_.numCoroutines, folly::TDigest(kTDigestMaxSize)),
        testExecutor_(benchOptions_.numTestThreads),
        numWriteBytes_(0),
        numReadBytes_(0),
        randGen_(folly::Random::create()),
        chunkInfos_(benchOptions_.numCoroutines),
        numCreatedChunks_(benchOptions_.numCoroutines) {
    if (benchOptions_.readBatchSize == 0) benchOptions_.readBatchSize = benchOptions_.batchSize;
    if (benchOptions_.writeBatchSize == 0) benchOptions_.writeBatchSize = benchOptions_.batchSize;
    if (benchOptions_.removeBatchSize == 0) benchOptions_.removeBatchSize = benchOptions_.batchSize;
  }

  void generateChunkIds() {
    static_assert(sizeof(benchOptions_.chunkIdPrefix) == 2);
    uint64_t chunkIdPrefix64 = ((uint64_t)benchOptions_.chunkIdPrefix) << (UINT64_WIDTH - UINT16_WIDTH);
    std::sort(chainIds_.begin(), chainIds_.end());
    static thread_local std::mt19937 generator;
    randGen_.seed(benchOptions_.randSeed);

    XLOGF(WARN,
          "Generating {} chunk ids with prefix {:08X} and random seed {}...",
          totalNumChunks_,
          chunkIdPrefix64,
          benchOptions_.randSeed);

    for (auto &chunkInfos : chunkInfos_) {
      uint64_t instancePrefix = chunkIdPrefix64 | folly::Random::rand64(randGen_);
      XLOGF(DBG3, "Random chunk id prefix {:08X}", instancePrefix);

      chunkInfos.reserve(chainIds_.size() * benchOptions_.numChunks);

      for (auto chainId : chainIds_) {
        for (size_t chunkIndex = 0; chunkIndex < benchOptions_.numChunks; chunkIndex++) {
          if (benchOptions_.sparseChunkIds) {
            uint64_t chunkIdHigh = chunkIdPrefix64 | (folly::Random::rand64(randGen_) & 0x000000FFFFFFFFFF);
            uint64_t chunkIdLow = (folly::Random::rand64(randGen_) << UINT32_WIDTH) + chunkIndex;
            chunkInfos.push_back({chainId, ChunkId(chunkIdHigh, chunkIdLow), 0});
          } else {
            chunkInfos.push_back({chainId, ChunkId(instancePrefix, chunkIndex), 0});
          }
        }
      }

      if (benchOptions_.randomShuffleChunkIds) std::shuffle(chunkInfos.begin(), chunkInfos.end(), generator);
    }
  }

  bool connect() {
    XLOGF(INFO, "Start to connect...");

    if (!setupIBSock()) {
      return false;
    }

    mgmtdClientConfig_.set_mgmtd_server_addresses(benchOptions_.mgmtdEndpoints);
    mgmtdClientConfig_.set_enable_auto_refresh(true);
    mgmtdClientConfig_.set_enable_auto_heartbeat(false);
    mgmtdClientConfig_.set_enable_auto_extend_client_session(true);
    mgmtdClientConfig_.set_auto_refresh_interval(3_s);
    mgmtdClientConfig_.set_auto_heartbeat_interval(3_s);
    mgmtdClientConfig_.set_auto_extend_client_session_interval(3_s);
    mgmtdClientConfig_.set_accept_incomplete_routing_info_during_mgmtd_bootstrapping(false);

    if (!client_.start()) {
      XLOGF(ERR, "Failed to start net client for mgmtd client");
      return false;
    }

    XLOGF(INFO, "Creating mgmtd client...");

    auto stubFactory = std::make_unique<hf3fs::stubs::RealStubFactory<hf3fs::mgmtd::MgmtdServiceStub>>(
        stubs::ClientContextCreator{[this](net::Address addr) { return client_.serdeCtx(addr); }});
    auto mgmtdClient = std::make_unique<hf3fs::client::MgmtdClientForClient>(benchOptions_.clusterId,
                                                                             std::move(stubFactory),
                                                                             mgmtdClientConfig_);

    auto physicalHostnameRes = SysResource::hostname(/*physicalMachineName=*/true);
    if (!physicalHostnameRes) {
      XLOGF(ERR, "getHostname(true) failed: {}", physicalHostnameRes.error());
      return false;
    }

    auto containerHostnameRes = SysResource::hostname(/*physicalMachineName=*/false);
    if (!containerHostnameRes) {
      XLOGF(ERR, "getHostname(false) failed: {}", containerHostnameRes.error());
      return false;
    }

    mgmtdClient->setClientSessionPayload({clientId_.uuid.toHexString(),
                                          flat::NodeType::CLIENT,
                                          flat::ClientSessionData::create(
                                              /*universalId=*/*physicalHostnameRes,
                                              /*description=*/fmt::format("StorageBench: {}", *containerHostnameRes),
                                              /*serviceGroups=*/std::vector<flat::ServiceGroupInfo>{},
                                              flat::ReleaseVersion::fromVersionInfo()),
                                          flat::UserInfo{}});
    folly::coro::blockingWait(mgmtdClient->start(&client_.tpg().bgThreadPool().randomPick()));
    mgmtdForClient_ = std::move(mgmtdClient);

    // get routing info

    for (size_t retry = 0; retry < 15; retry++) {
      auto routingInfo = mgmtdForClient_->getRoutingInfo();

      if (routingInfo == nullptr || routingInfo->raw()->chains.empty()) {
        XLOGF(WARN, "Empty routing info, #{} retry...", retry + 1);
        std::this_thread::sleep_for(std::chrono::milliseconds(1000));
      } else {
        for (const auto &[tableId, tableVersions] : routingInfo->raw()->chainTables) {
          if (tableId == benchOptions_.chainTableId) {
            if (tableVersions.empty()) {
              XLOGF(WARN, "No version found for chain table with id {}", tableId);
              return false;
            }

            XLOGF(INFO, "Found {} version(s) of chain table {}", tableVersions.size(), benchOptions_.chainTableId);

            flat::ChainTable chainTable;

            if (benchOptions_.chainTableVersion > 0) {
              flat::ChainTableVersion tableVersion(benchOptions_.chainTableVersion);
              auto tableIter = tableVersions.find(tableVersion);

              if (tableIter == tableVersions.end()) {
                XLOGF(WARN, "Version {} not found in chain table with id {}", tableVersion, tableId);
                return false;
              }

              chainTable = tableIter->second;
              XLOGF(INFO,
                    "Found version {} of chain table {}: {}",
                    benchOptions_.chainTableVersion,
                    benchOptions_.chainTableId,
                    chainTable.chainTableVersion);
            } else {
              const auto iter = --tableVersions.cend();
              const auto &latestTable = iter->second;
              chainTable = latestTable;
              XLOGF(INFO,
                    "Found latest version of chain table {}: {}",
                    benchOptions_.chainTableId,
                    chainTable.chainTableVersion);
            }

            XLOGF(WARN,
                  "Selected chain table: {}@{} [{}] {} chains",
                  chainTable.chainTableId,
                  chainTable.chainTableVersion,
                  chainTable.desc,
                  chainTable.chains.size());

            if (!benchOptions_.storageNodeIds.empty()) {
              for (const auto &chainId : chainTable.chains) {
                const auto chainInfo = routingInfo->raw()->getChain(chainId);
                for (const auto &target : chainInfo->targets) {
                  const auto targetInfo = routingInfo->raw()->getTarget(target.targetId);
                  auto nodeIter = std::find(benchOptions_.storageNodeIds.begin(),
                                            benchOptions_.storageNodeIds.end(),
                                            *targetInfo->nodeId);
                  if (nodeIter != benchOptions_.storageNodeIds.end()) {
                    chainIds_.push_back(chainId);
                    break;
                  }
                }
              }
            } else if (!benchOptions_.chainIds.empty()) {
              for (const auto &chainId : chainTable.chains) {
                auto chainIter = std::find(benchOptions_.chainIds.begin(), benchOptions_.chainIds.end(), chainId);
                if (chainIter != benchOptions_.chainIds.end()) {
                  chainIds_.push_back(chainId);
                }
              }
            } else {
              chainIds_ = chainTable.chains;
            }

            break;
          }
        }

        if (!chainIds_.empty()) break;
      }
    }

    if (chainIds_.empty()) {
      XLOGF(ERR, "Failed to get chain table with id {}", benchOptions_.chainTableId);
      return false;
    } else {
      XLOGF(WARN, "Selected {} replication chains for benchmark", chainIds_.size());
    }

    // create storage client

    if (setupConfig_.client_config().empty()) {
      XLOGF(ERR, "Storage client config not specified");
      return false;
    }

    auto configRes = clientConfig_.atomicallyUpdate(setupConfig_.client_config(), false /*isHotUpdate*/);
    if (!configRes) {
      XLOGF(ERR, "Cannot load client config from {}, error: {}", setupConfig_.client_config(), configRes.error());
      return false;
    }

    totalNumChunks_ = chainIds_.size() * benchOptions_.numCoroutines * benchOptions_.numChunks;
    totalChunkGiB_ = (double)totalNumChunks_ * setupConfig_.chunk_size() / 1_GB;
    clientConfig_.retry().set_max_retry_time(Duration(std::chrono::milliseconds(benchOptions_.clientTimeoutMS)));
    clientConfig_.net_client().io_worker().ibsocket().set_sl(setupConfig_.service_level());

    XLOGF(INFO, "Creating storage client...");
    storageClient_ = client::StorageClient::create(clientId_, clientConfig_, *mgmtdForClient_);

    return true;
  }

  bool setupIBSock() {
    XLOGF(WARN, "Setting up IB socket...");

    std::vector<net::IBConfig::Subnet> subnets;

    for (const auto &ibnetZoneStr : benchOptions_.ibnetZones) {
      std::vector<std::string> ibnetZoneSubnet;
      boost::split(ibnetZoneSubnet, ibnetZoneStr, boost::is_any_of(":"));

      if (ibnetZoneSubnet.size() != 2) {
        XLOGF(CRITICAL, "Invalid IB zone subnet: {}", ibnetZoneStr);
        return false;
      }

      auto zone = boost::trim_copy(ibnetZoneSubnet[0]);
      auto subnet = boost::trim_copy(ibnetZoneSubnet[1]);

      if (zone.empty() || subnet.empty()) {
        XLOGF(CRITICAL, "Invalid IB zone subnet: {}", ibnetZoneStr);
        return false;
      }

      subnets.emplace_back();
      subnets.back().set_network_zones({zone});
      subnets.back().set_subnet(*net::IBConfig::Network::from(subnet));
      XLOGF(WARN, "Add IB network zone: {} -- {}", zone, subnet);
    }

    net::IBConfig ibConfig;
    ibConfig.set_subnets(subnets);
    ibConfig.set_allow_unknown_zone(false);
    ibConfig.set_default_network_zone("$HF3FS_NETWORK_ZONE");
    ibConfig.set_device_filter(benchOptions_.ibvDevices);
    ibConfig.set_default_pkey_index(benchOptions_.defaultPKeyIndex);

    auto ibResult = net::IBManager::start(ibConfig);
    if (ibResult.hasError()) {
      XLOGF(CRITICAL, "Cannot initialize IB device: {}", ibResult.error());
      return false;
    }

    return true;
  }

  bool setup() {
    XLOGF(WARN, "Setting up benchmark...");

    if (!setupIBSock()) {
      return false;
    }

    bool ok = setUpStorageSystem();

    totalNumChunks_ = chainIds_.size() * benchOptions_.numCoroutines * benchOptions_.numChunks;
    totalChunkGiB_ = (double)totalNumChunks_ * setupConfig_.chunk_size() / 1_GB;
    clientConfig_.retry().set_max_retry_time(Duration(std::chrono::milliseconds(benchOptions_.clientTimeoutMS)));

    return ok;
  }

  void teardown() {
    tearDownStorageSystem();
    net::IBManager::stop();
  }

  void printThroughput(hf3fs::SteadyClock::duration elapsedMicro, double totalGiB) {
    auto elapsedMilli = std::chrono::duration_cast<std::chrono::milliseconds>(elapsedMicro);
    double throughput = totalGiB / (elapsedMilli.count() / 1000.0);
    XLOGF(WARN, "Average throughput: {:.3f}GiB/s, total {:.3f} GiB", throughput, totalGiB);
  }

  void printLatencyDigest(const folly::TDigest &digest) {
    XLOGF(WARN, "latency summary ({} samples)", digest.count());
    XLOGF(WARN, "min: {:10.1f}us", digest.min());
    XLOGF(WARN, "max: {:10.1f}us", digest.max());
    XLOGF(WARN, "avg: {:10.1f}us", digest.mean());
    for (double p : {0.1, 0.2, 0.5, 0.9, 0.95, 0.99}) {
      XLOGF(WARN, "{}%: {:10.1f}us", p * 100.0, digest.estimateQuantile(p));
    }
  }

  void dumpPerfStats(const std::string &testName,
                     const folly::TDigest &digest,
                     hf3fs::SteadyClock::duration elapsedTime,
                     double totalGiB,
                     bool readIO) {
    if (benchOptions_.statsFilePath.empty()) return;

    boost::filesystem::path outFilePath(benchOptions_.statsFilePath);

    if (!boost::filesystem::exists(outFilePath) || boost::filesystem::is_empty(outFilePath)) {
      XLOGF(INFO, "Create a file for perfermance stats at {}", outFilePath);
      std::ofstream outFile(outFilePath);
      if (!outFile) {
          throw std::runtime_error("Failed to open file: " + outFilePath.string());
      }

      outFile << "test name,#storages,#chains,#replicas,concurrency,batch size,"
                 "io size (bytes),effective batch size (batch size / #replicas),elapsed time (us),"
                 "QPS,IOPS,bandwidth (MB/s),latency samples,min latency (us),max latency (us),avg latency (us),"
                 "latency P50 (us),latency P75 (us),latency P90 (us),latency P95 (us),latency P99 (us)\n";

      if (!outFile) {
          throw std::runtime_error("Failed to write to file: " + outFilePath.string());
      }

      outFile.close();
    }

    auto elapsedMicro = std::chrono::duration_cast<std::chrono::microseconds>(elapsedTime);
    double bandwidthMBps = totalGiB * 1024.0 / (elapsedMicro.count() / 1'000'000.0);
    size_t ioSize = readIO ? benchOptions_.readSize : benchOptions_.writeSize;
    size_t batchSize = readIO ? benchOptions_.readBatchSize : benchOptions_.writeBatchSize;
    double iops = bandwidthMBps * 1024.0 * 1024.0 / ioSize;
    double qps = bandwidthMBps * 1024.0 * 1024.0 / (batchSize * ioSize);

    boost::filesystem::ofstream fout(outFilePath, std::ios_base::app);

    fout << fmt::format("{},{},{},{},{},{},{},{:.1f},{},{:.1f},{:.1f},{:.3f},{},{:.1f},{:.1f},{:.1f}",
                        testName,
                        setupConfig_.num_storage_nodes(),
                        setupConfig_.num_chains(),
                        setupConfig_.num_replicas(),
                        benchOptions_.numCoroutines,
                        batchSize,
                        ioSize,
                        double(batchSize) / setupConfig_.num_storage_nodes(),
                        elapsedMicro.count(),
                        qps,
                        iops,
                        bandwidthMBps,
                        digest.count(),
                        digest.min(),
                        digest.max(),
                        digest.mean());

    for (double p : {0.5, 0.75, 0.9, 0.95, 0.99}) {
      fout << fmt::format(",{:.1f}", digest.estimateQuantile(p));
    }

    fout << "\n";
    fout.close();
  }

  CoTask<uint32_t> batchWrite(uint32_t instanceId, size_t writeBatchSize, size_t writeSize, uint32_t numWriteSecs) {
    // create an aligned memory block
    size_t memoryBlockSize = ALIGN_UPPER(setupConfig_.chunk_size(), benchOptions_.memoryAlignment);
    auto memoryBlock = (uint8_t *)folly::aligned_malloc(memoryBlockSize, sysconf(_SC_PAGESIZE));
    auto deleter = [](uint8_t *ptr) { folly::aligned_free(ptr); };
    std::unique_ptr<uint8_t, decltype(deleter)> memoryBlockPtr(memoryBlock, deleter);
    std::memset(memoryBlock, 0xFF, memoryBlockSize);

    if (benchOptions_.verifyReadData) {
      for (size_t byteIndex = 0; byteIndex < memoryBlockSize; byteIndex++) {
        memoryBlock[byteIndex] = byteIndex;
      }
    }

    // register a block of memory
    auto regRes = storageClient_->registerIOBuffer(memoryBlock, memoryBlockSize);

    if (regRes.hasError()) {
      co_return regRes.error().code();
    }

    // create write IOs

    auto ioBuffer = std::move(*regRes);

    WriteOptions options;
    options.set_enableChecksum(benchOptions_.verifyWriteChecksum);
    options.debug().set_bypass_disk_io(benchOptions_.benchmarkNetwork);
    options.debug().set_bypass_rdma_xmit(benchOptions_.benchmarkStorage);
    options.debug().set_inject_random_server_error(benchOptions_.injectRandomServerError);
    options.debug().set_inject_random_client_error(benchOptions_.injectRandomClientError);
    options.retry().set_retry_permanent_error(benchOptions_.retryPermanentError);

    std::vector<double> elapsedMicroSecs;
    uint64_t numWriteBytes = 0;

    std::vector<WriteIO> writeIOs;
    writeIOs.reserve(writeBatchSize);

    auto benchStart = hf3fs::SteadyClock::now();
    std::vector<ChunkInfo> &chunkInfos = chunkInfos_[instanceId];
    size_t &numCreatedChunks = numCreatedChunks_[instanceId];
    size_t seqChunkIndex = 0;

    while (true) {
      if (numWriteSecs) {
        auto accumElapsedSecs =
            std::chrono::duration_cast<std::chrono::seconds>(hf3fs::SteadyClock::now() - benchStart);
        if (accumElapsedSecs >= std::chrono::seconds(numWriteSecs)) break;
      } else {
        if (numCreatedChunks >= chunkInfos.size()) break;
      }

      writeIOs.clear();

      for (size_t writeIndex = 0; writeIndex < writeBatchSize; writeIndex++) {
        auto &[chainId, chunkId, chunkSize] = chunkInfos[seqChunkIndex++ % chunkInfos.size()];
        size_t writeOffset = 0;
        size_t writeLength = 0;

        if (chunkSize < setupConfig_.chunk_size()) {
          writeOffset = chunkSize;
          writeLength = std::min(writeSize, setupConfig_.chunk_size() - writeOffset);
          chunkSize += writeLength;
          numCreatedChunks += chunkSize == setupConfig_.chunk_size();
        } else {
          writeOffset = folly::Random::rand32(0, setupConfig_.chunk_size() - writeSize);
          writeLength = writeSize;
        }

        auto writeIO = storageClient_->createWriteIO(chainId,
                                                     chunkId,
                                                     writeOffset,
                                                     writeLength,
                                                     setupConfig_.chunk_size(),
                                                     &memoryBlock[writeOffset],
                                                     &ioBuffer);
        writeIOs.push_back(std::move(writeIO));
        numWriteBytes += writeLength;
      }

      auto rpcStart = hf3fs::SteadyClock::now();

      co_await storageClient_->batchWrite(writeIOs, flat::UserInfo(), options);

      auto elapsedMicro = std::chrono::duration_cast<std::chrono::microseconds>(hf3fs::SteadyClock::now() - rpcStart);
      elapsedMicroSecs.push_back(elapsedMicro.count());

      if (!benchOptions_.ignoreIOError) {
        for (const auto &writeIO : writeIOs) {
          if (writeIO.result.lengthInfo.hasError()) {
            XLOGF(ERR, "Error in write result: {}", writeIO.result);
            co_return writeIO.result.lengthInfo.error().code();
          }
          if (writeIO.length != *writeIO.result.lengthInfo) {
            XLOGF(ERR, "Unexpected write length: {} != {}", *writeIO.result.lengthInfo, writeIO.length);
            co_return StorageClientCode::kRemoteIOError;
          }
        }
      }
    }

    folly::TDigest digest;
    writeLatencyDigests_[instanceId] = digest.merge(elapsedMicroSecs);
    numWriteBytes_ += numWriteBytes;

    co_return StatusCode::kOK;
  }

  CoTask<uint32_t> batchRead(uint32_t instanceId) {
    // create an aligned memory block
    size_t alignedBufSize = ALIGN_UPPER(std::max(size_t(1), benchOptions_.readSize), benchOptions_.memoryAlignment);
    size_t memoryBlockSize = alignedBufSize * benchOptions_.readBatchSize;
    auto memoryBlock = (uint8_t *)folly::aligned_malloc(memoryBlockSize, sysconf(_SC_PAGESIZE));
    auto deleter = [](uint8_t *ptr) { folly::aligned_free(ptr); };
    std::unique_ptr<uint8_t, decltype(deleter)> memoryBlockPtr(memoryBlock, deleter);
    std::memset(memoryBlock, 0, memoryBlockSize);

    // register a block of memory
    auto regRes = storageClient_->registerIOBuffer(memoryBlock, memoryBlockSize);

    if (regRes.hasError()) {
      co_return regRes.error().code();
    }

    std::vector<uint8_t> expectedChunkData(setupConfig_.chunk_size());

    if (benchOptions_.verifyReadData) {
      for (size_t byteIndex = 0; byteIndex < expectedChunkData.size(); byteIndex++) {
        expectedChunkData[byteIndex] = byteIndex;
      }
    }

    // create read IOs

    auto ioBuffer = std::move(*regRes);

    ReadOptions options;
    options.set_enableChecksum(benchOptions_.verifyReadChecksum);
    options.debug().set_bypass_disk_io(benchOptions_.benchmarkNetwork);
    options.debug().set_bypass_rdma_xmit(benchOptions_.benchmarkStorage);
    options.debug().set_inject_random_server_error(benchOptions_.injectRandomServerError);
    options.debug().set_inject_random_client_error(benchOptions_.injectRandomClientError);
    options.retry().set_retry_permanent_error(benchOptions_.retryPermanentError);

    std::vector<double> elapsedMicroSecs;
    uint64_t numReadBytes = 0;
    size_t offsetAlignment =
        benchOptions_.readOffAlignment ? benchOptions_.readOffAlignment : std::max(size_t(1), benchOptions_.readSize);

    std::vector<client::ReadIO> readIOs;
    readIOs.reserve(benchOptions_.readBatchSize);

    auto benchStart = hf3fs::SteadyClock::now();
    std::vector<ChunkInfo> &chunkInfos = chunkInfos_[instanceId];

    while (true) {
      auto accumElapsedSecs = std::chrono::duration_cast<std::chrono::seconds>(hf3fs::SteadyClock::now() - benchStart);
      if (accumElapsedSecs >= std::chrono::seconds(benchOptions_.numReadSecs)) break;

      readIOs.clear();

      for (size_t readIndex = 0; readIndex < benchOptions_.readBatchSize; readIndex++) {
        uint64_t randChunkIndex = folly::Random::rand64(0, chunkInfos.size());
        const auto &[chainId, chunkId, chunkSize] = chunkInfos[randChunkIndex];
        uint32_t offset = folly::Random::rand32(0, setupConfig_.chunk_size() - benchOptions_.readSize);
        uint32_t alignedOffset = ALIGN_LOWER(offset, offsetAlignment);
        auto readIO = storageClient_->createReadIO(chainId,
                                                   chunkId,
                                                   alignedOffset /*offset*/,
                                                   benchOptions_.readSize /*length*/,
                                                   &memoryBlock[readIndex * alignedBufSize],
                                                   &ioBuffer);
        readIOs.push_back(std::move(readIO));
        numReadBytes += benchOptions_.readSize;
      }

      auto rpcStart = hf3fs::SteadyClock::now();

      co_await storageClient_->batchRead(readIOs, flat::UserInfo(), options);

      auto elapsedMicro = std::chrono::duration_cast<std::chrono::microseconds>(hf3fs::SteadyClock::now() - rpcStart);
      elapsedMicroSecs.push_back(elapsedMicro.count());

      if (!benchOptions_.ignoreIOError) {
        for (const auto &readIO : readIOs) {
          if (readIO.result.lengthInfo.hasError()) {
            XLOGF(ERR, "Error in read result: {}", readIO.result);
            co_return readIO.result.lengthInfo.error().code();
          }
          if (readIO.length != *readIO.result.lengthInfo) {
            XLOGF(ERR, "Unexpected read length: {} != {}", *readIO.result.lengthInfo, readIO.length);
            co_return StorageClientCode::kRemoteIOError;
          }
        }
      }

      if (benchOptions_.verifyReadData) {
        for (const auto &readIO : readIOs) {
          auto diffPos = std::mismatch(&readIO.data[0], &readIO.data[readIO.length], &expectedChunkData[readIO.offset]);
          uint32_t byteIndex = diffPos.first - &readIO.data[0];
          if (byteIndex < readIO.length) {
            XLOGF(ERR,
                  "Wrong data at bytes index {} and chunk offset {}: data {:#x} != expected {:#x}",
                  byteIndex,
                  readIO.offset + byteIndex,
                  *diffPos.first,
                  *diffPos.second);
            co_return StorageClientCode::kFoundBug;
          }
        }
      }
    }

    folly::TDigest digest;
    readLatencyDigests_[instanceId] = digest.merge(elapsedMicroSecs);
    numReadBytes_ += numReadBytes;

    co_return StatusCode::kOK;
  }

  uint32_t generateChunks() {
    XLOGF(WARN, "Generating {} test chunks ({:.3f} GiB)...", totalNumChunks_, totalChunkGiB_);

    auto testStart = hf3fs::SteadyClock::now();
    std::vector<folly::SemiFuture<uint32_t>> writeTasks;
    numWriteBytes_ = 0;

    size_t writeBatchSize =
        std::max(benchOptions_.writeBatchSize,
                 clientConfig_.traffic_control().write().max_concurrent_requests() / benchOptions_.numCoroutines);

    for (size_t instanceId = 0; instanceId < benchOptions_.numCoroutines; instanceId++) {
      writeTasks.push_back(batchWrite(instanceId, writeBatchSize, setupConfig_.chunk_size(), 0 /*numWriteSecs*/)
                               .scheduleOn(folly::Executor::getKeepAliveToken(testExecutor_))
                               .start());
    }

    auto results = folly::coro::blockingWait(folly::coro::collectAllRange(std::move(writeTasks)));

    for (auto res : results) {
      if (res != StatusCode::kOK) {
        XLOGF(WARN, "Test task failed with status code: {}", res);
        return res;
      }
    }

    auto elapsedTime = hf3fs::SteadyClock::now() - testStart;
    double totalGiB = (double)numWriteBytes_ / 1_GB;
    printThroughput(elapsedTime, totalGiB);

    auto mergedDigest = folly::TDigest::merge(writeLatencyDigests_);
    printLatencyDigest(mergedDigest);

    return StatusCode::kOK;
  }

  uint32_t runWriteBench() {
    XLOGF(WARN,
          "Running write benchmark ({} secs, {} chunks, {:.3f} GiB)...",
          benchOptions_.numWriteSecs,
          totalNumChunks_,
          totalChunkGiB_);

    auto testStart = hf3fs::SteadyClock::now();
    std::vector<folly::SemiFuture<uint32_t>> writeTasks;
    numWriteBytes_ = 0;

    for (size_t instanceId = 0; instanceId < benchOptions_.numCoroutines; instanceId++) {
      writeTasks.push_back(
          batchWrite(instanceId, benchOptions_.writeBatchSize, benchOptions_.writeSize, benchOptions_.numWriteSecs)
              .scheduleOn(folly::Executor::getKeepAliveToken(testExecutor_))
              .start());
    }

    auto results = folly::coro::blockingWait(folly::coro::collectAllRange(std::move(writeTasks)));

    for (auto res : results) {
      if (res != StatusCode::kOK) {
        XLOGF(WARN, "Test task failed with status code: {}", res);
        return res;
      }
    }

    auto elapsedTime = hf3fs::SteadyClock::now() - testStart;
    double totalGiB = (double)numWriteBytes_ / 1_GB;
    printThroughput(elapsedTime, totalGiB);

    auto mergedDigest = folly::TDigest::merge(writeLatencyDigests_);
    printLatencyDigest(mergedDigest);

    dumpPerfStats("batch write", mergedDigest, elapsedTime, totalGiB, false /*readIO*/);

    return StatusCode::kOK;
  }

  uint32_t runReadBench() {
    XLOGF(WARN, "Running read benchmark ({} secs)...", benchOptions_.numReadSecs);

    auto testStart = hf3fs::SteadyClock::now();
    std::vector<folly::SemiFuture<uint32_t>> readTasks;
    numReadBytes_ = 0;

    for (size_t instanceId = 0; instanceId < benchOptions_.numCoroutines; instanceId++) {
      readTasks.push_back(batchRead(instanceId).scheduleOn(folly::Executor::getKeepAliveToken(testExecutor_)).start());
    }

    auto results = folly::coro::blockingWait(folly::coro::collectAllRange(std::move(readTasks)));

    for (auto res : results) {
      if (res != StatusCode::kOK) {
        XLOGF(WARN, "Test task failed with status code: {}", res);
        return res;
      }
    }

    auto elapsedTime = hf3fs::SteadyClock::now() - testStart;
    double totalGiB = (double)numReadBytes_ / 1_GB;
    printThroughput(elapsedTime, totalGiB);

    auto mergedDigest = folly::TDigest::merge(readLatencyDigests_);
    printLatencyDigest(mergedDigest);

    dumpPerfStats("batch read", mergedDigest, elapsedTime, totalGiB, false /*readIO*/);

    return StatusCode::kOK;
  }

  uint32_t cleanup() {
    XLOGF(WARN, "Clean up chunks...");

    std::vector<folly::SemiFuture<uint32_t>> removeTasks;

    for (size_t instanceId = 0; instanceId < benchOptions_.numCoroutines; instanceId++) {
      auto batchRemove = [this](size_t instanceId) -> folly::coro::Task<uint32_t> {
        std::vector<client::RemoveChunksOp> removeOps;
        size_t totalNumChunksRemoved = 0;

        for (const auto &[chainId, chunkId, chunkSize] : chunkInfos_[instanceId]) {
          removeOps.push_back(storageClient_->createRemoveOp(chainId, chunkId, ChunkId(chunkId, 1)));

          if (removeOps.size() >= benchOptions_.removeBatchSize) {
            WriteOptions options;
            options.debug().set_inject_random_server_error(benchOptions_.injectRandomServerError);
            options.debug().set_inject_random_client_error(benchOptions_.injectRandomClientError);
            options.retry().set_retry_permanent_error(benchOptions_.retryPermanentError);

            co_await storageClient_->removeChunks(removeOps, flat::UserInfo(), options);

            for (const auto &removeOp : removeOps) {
              if (removeOp.result.statusCode.hasError()) {
                XLOGF(WARN, "Remove operation failed with error: {}", removeOp.result.statusCode.error());
                co_return removeOp.result.statusCode.error().code();
              }

              XLOGF_IF(DBG5,
                       removeOp.result.numChunksRemoved != 1,
                       "{} chunks removed in range {}",
                       removeOp.result.numChunksRemoved,
                       removeOp.chunkRange());
              totalNumChunksRemoved += removeOp.result.numChunksRemoved;
            }

            removeOps.clear();
          }
        }

        XLOGF(WARN, "{} chunks removed by instance #{}", totalNumChunksRemoved, instanceId);
        co_return StatusCode::kOK;
      };

      removeTasks.push_back(
          batchRemove(instanceId).scheduleOn(folly::Executor::getKeepAliveToken(testExecutor_)).start());
    }

    auto results = folly::coro::blockingWait(folly::coro::collectAllRange(std::move(removeTasks)));

    for (auto res : results) {
      if (res != StatusCode::kOK) {
        XLOGF(WARN, "Test task failed with status code: {}", res);
        return res;
      }
    }

    return StatusCode::kOK;
  };

  uint32_t truncate() {
    XLOGF(WARN, "Truncate chunks...");

    std::vector<folly::SemiFuture<uint32_t>> truncateTasks;

    for (size_t instanceId = 0; instanceId < benchOptions_.numCoroutines; instanceId++) {
      auto batchTruncate = [this](size_t instanceId) -> folly::coro::Task<uint32_t> {
        std::vector<client::TruncateChunkOp> truncateOps;

        for (const auto &[chainId, chunkId, chunkSize] : chunkInfos_[instanceId]) {
          truncateOps.push_back(storageClient_->createTruncateOp(chainId, chunkId, 0, setupConfig_.chunk_size()));

          if (truncateOps.size() >= benchOptions_.writeBatchSize) {
            WriteOptions options;
            options.debug().set_inject_random_server_error(benchOptions_.injectRandomServerError);
            options.debug().set_inject_random_client_error(benchOptions_.injectRandomClientError);
            options.retry().set_retry_permanent_error(benchOptions_.retryPermanentError);

            co_await storageClient_->truncateChunks(truncateOps, flat::UserInfo(), options);

            for (const auto &truncateOp : truncateOps) {
              if (truncateOp.result.lengthInfo.hasError()) {
                XLOGF(WARN, "Truncate operation failed with error: {}", truncateOp.result.lengthInfo.error());
                co_return truncateOp.result.lengthInfo.error().code();
              }
            }

            truncateOps.clear();
          }
        }

        co_return StatusCode::kOK;
      };

      truncateTasks.push_back(
          batchTruncate(instanceId).scheduleOn(folly::Executor::getKeepAliveToken(testExecutor_)).start());
    }

    auto results = folly::coro::blockingWait(folly::coro::collectAllRange(std::move(truncateTasks)));

    for (auto res : results) {
      if (res != StatusCode::kOK) {
        XLOGF(WARN, "Test task failed with status code: {}", res);
        return res;
      }
    }

    return StatusCode::kOK;
  };

  bool run() {
    if (benchOptions_.numWriteSecs > 0)
      if (runWriteBench() != StatusCode::kOK) return false;
    if (benchOptions_.generateTestData)
      if (generateChunks() != StatusCode::kOK) return false;
    if (benchOptions_.numReadSecs > 0)
      if (runReadBench() != StatusCode::kOK) return false;
    return true;
  }

  uint64_t getWriteBytes() { return numWriteBytes_; }

  uint64_t getReadBytes() { return numReadBytes_; }
};

}  // namespace hf3fs::storage::benchmark

#pragma once

#include <span>
#include <vector>

#include "TargetSelection.h"
#include "UpdateChannelAllocator.h"
#include "client/mgmtd/ICommonMgmtdClient.h"
#include "common/net/Client.h"
#include "common/utils/Address.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Result.h"
#include "common/utils/Semaphore.h"
#include "fbs/mgmtd/RoutingInfo.h"
#include "fbs/storage/Common.h"

namespace hf3fs::storage::client {

/* Dynamic routing info for accessing a storage target */

class RoutingTarget {
 public:
  RoutingTarget(ChainId chainId)
      : chainId(chainId) {}

  ~RoutingTarget() {
    XLOGF_IF(DFATAL,
             channel.id != ChannelId{0},
             "Leaked update channel, routing target: {}, stack trace: {}",
             *this,
             folly::symbolizer::getStackTraceStr());
  }

  const hf3fs::storage::VersionedChainId getVersionedChainId() const { return {chainId, chainVer}; };

 public:
  ChainId chainId;
  ChainVer chainVer;
  flat::RoutingInfoVersion routingInfoVer;
  SlimTargetInfo targetInfo;
  UpdateChannel channel;
};

/* Read/write IOs */

class IOBuffer : public folly::MoveOnly {
 public:
  uint8_t *data() const { return const_cast<uint8_t *>(rdmabuf.ptr()); }

  size_t size() const { return rdmabuf.size(); }

  bool contains(const uint8_t *data, uint32_t len) const { return rdmabuf.contains(data, len); }

  net::RDMABuf subrange(size_t offset, size_t length) const { return rdmabuf.subrange(offset, length); }

  IOBuffer(hf3fs::net::RDMABuf rdmabuf)
      : rdmabuf(rdmabuf) {}

 private:
  const hf3fs::net::RDMABuf rdmabuf;

  friend class IOBase;
  friend class StorageClient;
  friend class StorageClientImpl;
  friend class StorageClientInMem;
};

class IOBase : public folly::MoveOnly {
 private:
  IOBase(ChainId chainId,
         const ChunkId &chunkId,
         uint32_t offset,
         uint32_t length,
         uint32_t chunkSize,
         uint8_t *data,
         IOBuffer *buffer,
         void *userCtx)
      : routingTarget(chainId),
        chunkId(chunkId),
        offset(offset),
        length(length),
        chunkSize(chunkSize),
        data(data),
        buffer(buffer),
        userCtx(userCtx) {}

 public:
  Status status() const { return result.lengthInfo ? Status(StatusCode::kOK) : result.lengthInfo.error(); }
  status_code_t statusCode() const { return hf3fs::getStatusCode(result.lengthInfo); }
  uint32_t resultLen() const { return result.lengthInfo ? *result.lengthInfo : 0; }
  uint32_t dataLen() const { return length; }
  uint8_t *dataEnd() const { return data + length; }
  ChunkIdRange chunkRange() const { return {chunkId, chunkId, 1}; }
  uint32_t numProcessedChunks() const { return bool(result.lengthInfo); }
  void resetResult() { result = IOResult{}; }

 public:
  RoutingTarget routingTarget;
  ChunkId chunkId;
  const uint32_t offset;
  const uint32_t length;
  const uint32_t chunkSize;
  uint8_t *const data;
  IOBuffer *const buffer;
  void *const userCtx;
  IOResult result;

  friend class ReadIO;
  friend class WriteIO;
};

class ReadIO : public IOBase {
 private:
  ReadIO(ChainId chainId,
         const ChunkId &chunkId,
         uint32_t offset,
         uint32_t length,
         uint8_t *data,
         IOBuffer *buffer,
         void *userCtx)
      : IOBase(chainId, chunkId, offset, length, 0 /*chunkSize*/, data, buffer, userCtx) {}

  friend class StorageClient;
  friend class StorageClientImpl;
  friend class StorageClientInMem;

 public:
  RequestId requestId;
  std::vector<ReadIO> splittedIOs;
};

class WriteIO : public IOBase {
 private:
  WriteIO(RequestId requestId,
          ChainId chainId,
          const ChunkId &chunkId,
          uint32_t offset,
          uint32_t length,
          uint32_t chunkSize,
          uint8_t *data,
          IOBuffer *buffer,
          void *userCtx)
      : IOBase(chainId, chunkId, offset, length, chunkSize, data, buffer, userCtx),
        requestId(requestId) {}

 public:
  const ChecksumInfo &localChecksum() const { return checksum; }

 private:
  friend class StorageClient;
  friend class StorageClientImpl;
  friend class StorageClientInMem;

 public:
  const RequestId requestId;
  ChecksumInfo checksum;
};

/* Read/write options */

class DebugOptions : public hf3fs::ConfigBase<DebugOptions> {
  CONFIG_HOT_UPDATED_ITEM(bypass_disk_io, false);
  CONFIG_HOT_UPDATED_ITEM(bypass_rdma_xmit, false);
  CONFIG_HOT_UPDATED_ITEM(inject_random_server_error, false);
  CONFIG_HOT_UPDATED_ITEM(inject_random_client_error, false);
  CONFIG_HOT_UPDATED_ITEM(max_num_of_injection_points, 100);

 public:
  DebugFlags toDebugFlags() const {
#ifndef NDEBUG
    return DebugFlags{
        .injectRandomServerError = inject_random_server_error(),
        .injectRandomClientError = inject_random_client_error(),
        .numOfInjectPtsBeforeFail = (uint16_t)folly::Random::rand32(1, max_num_of_injection_points() + 1)};
#else
    return DebugFlags{};
#endif
  }
};

class RetryOptions : public hf3fs::ConfigBase<RetryOptions> {
  CONFIG_HOT_UPDATED_ITEM(init_wait_time, Duration::zero());  // if set to zero, use the value from client config
  CONFIG_HOT_UPDATED_ITEM(max_wait_time, Duration::zero());   // if set to zero, use the value from client config
  CONFIG_HOT_UPDATED_ITEM(max_retry_time, Duration::zero());  // if set to zero, use the value from client config
  CONFIG_HOT_UPDATED_ITEM(retry_permanent_error, false);
};

class ReadOptions : public hf3fs::ConfigBase<ReadOptions> {
  CONFIG_OBJ(debug, DebugOptions);
  CONFIG_OBJ(retry, RetryOptions);
  CONFIG_OBJ(targetSelection, TargetSelectionOptions);
  CONFIG_HOT_UPDATED_ITEM(enableChecksum, false);
  CONFIG_HOT_UPDATED_ITEM(allowReadUncommitted, false);

 public:
  bool verifyChecksum() const {
#ifndef NDEBUG
    bool enabled = true;
#else
    bool enabled = enableChecksum();
#endif

    return enabled && !debug().bypass_disk_io() && !debug().bypass_rdma_xmit();
  }
};

class WriteOptions : public hf3fs::ConfigBase<WriteOptions> {
  CONFIG_OBJ(debug, DebugOptions);
  CONFIG_OBJ(retry, RetryOptions);
  CONFIG_OBJ(targetSelection, TargetSelectionOptions);  // for test only
  CONFIG_HOT_UPDATED_ITEM(enableChecksum, true);

 public:
  bool verifyChecksum() const {
#ifndef NDEBUG
    bool enabled = true;
#else
    bool enabled = enableChecksum();
#endif

    return enabled && !debug().bypass_disk_io() && !debug().bypass_rdma_xmit();
  }
};

class IoOptions : public ConfigBase<IoOptions> {
  CONFIG_OBJ(read, ReadOptions);
  CONFIG_OBJ(write, WriteOptions);
};

/* queryLastChunk */

class QueryLastChunkOp : public folly::MoveOnly {
 private:
  QueryLastChunkOp(ChainId chainId, ChunkIdRange range, void *userCtx)
      : requestId(0),
        routingTarget(chainId),
        range(range),
        userCtx(userCtx) {}

 public:
  Status status() const { return result.statusCode ? Status(StatusCode::kOK) : result.statusCode.error(); }
  status_code_t statusCode() const { return hf3fs::getStatusCode(result.statusCode); }
  uint32_t resultLen() const { return 0; }
  uint32_t dataLen() const { return 0; }
  ChunkIdRange chunkRange() const { return range; }
  uint32_t numProcessedChunks() const { return result.statusCode ? result.totalNumChunks : 0; }
  void resetResult() { result = QueryLastChunkResult{}; }

 public:
  RequestId requestId;
  RoutingTarget routingTarget;
  const ChunkIdRange range;
  void *const userCtx;
  QueryLastChunkResult result;

 public:
  friend class StorageClient;
  friend class StorageClientImpl;
  friend class StorageClientInMem;
};

/* removeChunks */

class RemoveChunksOp : public folly::MoveOnly {
 private:
  RemoveChunksOp(RequestId requestId, ChainId chainId, ChunkIdRange range, void *userCtx)
      : requestId(requestId),
        routingTarget(chainId),
        range(range),
        userCtx(userCtx) {}

 public:
  Status status() const { return result.statusCode ? Status(StatusCode::kOK) : result.statusCode.error(); }
  status_code_t statusCode() const { return hf3fs::getStatusCode(result.statusCode); }
  uint32_t resultLen() const { return 0; }
  uint32_t dataLen() const { return 0; }
  ChunkIdRange chunkRange() const { return range; }
  uint32_t numProcessedChunks() const { return result.statusCode ? result.numChunksRemoved : 0; }
  void resetResult() { result = RemoveChunksResult{}; }

 public:
  const RequestId requestId;
  RoutingTarget routingTarget;
  const ChunkIdRange range;
  void *const userCtx;
  RemoveChunksResult result;

  friend class StorageClient;
  friend class StorageClientImpl;
  friend class StorageClientInMem;
};

/* truncateChunks */

class TruncateChunkOp : public folly::MoveOnly {
 private:
  TruncateChunkOp(RequestId requestId,
                  ChainId chainId,
                  const ChunkId &chunkId,
                  uint32_t chunkLen,
                  uint32_t chunkSize,
                  bool onlyExtendChunk,
                  void *userCtx)
      : requestId(requestId),
        routingTarget(chainId),
        chunkId(chunkId),
        chunkLen(chunkLen),
        chunkSize(chunkSize),
        onlyExtendChunk(onlyExtendChunk),
        userCtx(userCtx) {}

 public:
  Status status() const { return result.lengthInfo ? Status(StatusCode::kOK) : result.lengthInfo.error(); }
  status_code_t statusCode() const { return hf3fs::getStatusCode(result.lengthInfo); }
  uint32_t resultLen() const { return 0; }
  uint32_t dataLen() const { return 0; }
  ChunkIdRange chunkRange() const { return {chunkId, chunkId, 1}; }
  uint32_t numProcessedChunks() const { return bool(result.lengthInfo); }
  void resetResult() { result = IOResult{}; }

 public:
  const RequestId requestId;
  RoutingTarget routingTarget;
  const ChunkId chunkId;
  const uint32_t chunkLen;
  const uint32_t chunkSize;
  bool onlyExtendChunk;
  void *const userCtx;
  IOResult result;  // result.lengthInfo == chunkLen if the op succeeds

  friend class StorageClient;
  friend class StorageClientImpl;
  friend class StorageClientInMem;
};

/* Storage client */

class StorageClient : public folly::MoveOnly {
 public:
  enum class ImplementationType {
    RPC,
    InMem,
  };

  enum class MethodType {
    batchRead = 1,
    batchWrite,
    read,
    write,
    queryLastChunk,
    removeChunks,
    truncateChunks,
    querySpaceInfo,
    createTarget,
    offlineTarget,
    removeTarget,
    queryChunk,
    getAllChunkMetadata,
  };

  class RetryConfig : public hf3fs::ConfigBase<RetryConfig> {
   public:
    CONFIG_HOT_UPDATED_ITEM(init_wait_time, 10_s);
    CONFIG_HOT_UPDATED_ITEM(max_wait_time, 30_s);
    CONFIG_HOT_UPDATED_ITEM(max_retry_time, 60_s);
    CONFIG_HOT_UPDATED_ITEM(max_failures_before_failover,
                            1U);  // the max number of failed retries before switching to alternative targets

   public:
    RetryOptions mergeWith(RetryOptions options) const {
      if (options.init_wait_time() == Duration::zero()) options.set_init_wait_time(this->init_wait_time());
      if (options.max_wait_time() == Duration::zero()) options.set_max_wait_time(this->max_wait_time());
      if (options.max_retry_time() == Duration::zero()) options.set_max_retry_time(this->max_retry_time());
      return options;
    }
  };

  class OperationConcurrency : public hf3fs::ConfigBase<OperationConcurrency> {
    CONFIG_ITEM(max_batch_size, 128U);
    CONFIG_ITEM(max_batch_bytes, 4_MB);
    CONFIG_ITEM(max_concurrent_requests, 32U);
    CONFIG_ITEM(max_concurrent_requests_per_server, 8U);
    CONFIG_HOT_UPDATED_ITEM(random_shuffle_requests, true);
    CONFIG_HOT_UPDATED_ITEM(process_batches_in_parallel, true);
  };

  class HotLoadOperationConcurrency : public hf3fs::ConfigBase<HotLoadOperationConcurrency> {
    CONFIG_HOT_UPDATED_ITEM(max_batch_size, 128U);
    CONFIG_HOT_UPDATED_ITEM(max_batch_bytes, 4_MB);
    CONFIG_HOT_UPDATED_ITEM(max_concurrent_requests, 32U);
    CONFIG_HOT_UPDATED_ITEM(max_concurrent_requests_per_server, 8U);
    CONFIG_HOT_UPDATED_ITEM(random_shuffle_requests, true);
    CONFIG_HOT_UPDATED_ITEM(process_batches_in_parallel, true);
  };

  class TrafficControlConfig : public hf3fs::ConfigBase<TrafficControlConfig> {
    CONFIG_OBJ(read, HotLoadOperationConcurrency);
    CONFIG_OBJ(write, OperationConcurrency);
    CONFIG_OBJ(query, HotLoadOperationConcurrency);
    CONFIG_OBJ(remove, OperationConcurrency);
    CONFIG_OBJ(truncate, OperationConcurrency);

   public:
    size_t max_concurrent_updates() const {
      return write().max_concurrent_requests() * write().max_batch_size() +
             remove().max_concurrent_requests() * remove().max_batch_size() +
             truncate().max_concurrent_requests() * truncate().max_batch_size();
    }
  };

  class Config : public hf3fs::ConfigBase<Config> {
   public:
    CONFIG_OBJ(net_client, hf3fs::net::Client::Config);
    CONFIG_OBJ(net_client_for_updates, hf3fs::net::Client::Config);
    CONFIG_OBJ(retry, RetryConfig);
    CONFIG_OBJ(traffic_control, TrafficControlConfig);
    CONFIG_ITEM(implementation_type, ImplementationType::RPC);
    CONFIG_ITEM(chunk_checksum_type, ChecksumType::CRC32C);
    CONFIG_ITEM(create_net_client_for_updates, false);
    CONFIG_HOT_UPDATED_ITEM(check_overlapping_read_buffers, true);
    CONFIG_HOT_UPDATED_ITEM(check_overlapping_write_buffers, false);
    CONFIG_HOT_UPDATED_ITEM(max_inline_read_bytes, Size{0});
    CONFIG_HOT_UPDATED_ITEM(max_inline_write_bytes, Size{0});
    CONFIG_HOT_UPDATED_ITEM(max_read_io_bytes, Size{0});
  };

 public:
  StorageClient(const ClientId &clientId, const Config &config)
      : clientId_(clientId),
        config_(config) {}

  StorageClient()
      : StorageClient(ClientId::random(), kDefaultConfig) {}

  virtual ~StorageClient() = default;

  static std::shared_ptr<StorageClient> create(ClientId clientId,
                                               const Config &config,
                                               hf3fs::client::ICommonMgmtdClient &mgmtdClient);

  virtual hf3fs::client::ICommonMgmtdClient &getMgmtdClient() = 0;

  virtual Result<Void> start() { return Void{}; }

  // If the user does not call `stop()', the client is stopped in destructor.
  virtual void stop() {}

  /* Read `length' bytes from `offset' of the chunk.
     The memory pointed by `data' should be large enough to store the data, fall in the range of
     the registered `buffer' and does not overlap with other IOs in the same batch (can be disable by
     setting `check_overlapping_read_buffers').
   */
  virtual ReadIO createReadIO(ChainId chainId,
                              const ChunkId &chunkId,
                              uint32_t offset,
                              uint32_t length,
                              uint8_t *data,
                              IOBuffer *buffer,
                              void *userCtx = nullptr);

  /* Write `length' bytes of data at `offset' of the chunk.
     The memory pointed by `data' should be large enough to store the data, fall in the range of
     the registered `buffer' and does not overlap with other IOs in the same batch (can be disable by
     setting `check_overlapping_write_buffers').
     If option `chunk_checksum_type' is not none, a checksum will be calculated for the write buffer.
   */
  virtual WriteIO createWriteIO(ChainId chainId,
                                const ChunkId &chunkId,
                                uint32_t offset,
                                uint32_t length,
                                uint32_t chunkSize,
                                uint8_t *data,
                                IOBuffer *buffer,
                                void *userCtx = nullptr);

  /* Query the chunk with largest lexicographical id in range [chunkIdBegin, chunkIdEnd).
     `totalChunkLen' and `totalNumChunks' of chunks in the range are calculated and included in
     `QueryLastChunkResult'.
     If `moreChunksInRange' in `QueryLastChunkResult' is true, there exist more than
     `maxNumChunkIdsToProcess' chunks in range [chunkIdBegin, chunkIdEnd).
  */
  virtual QueryLastChunkOp createQueryOp(ChainId chainId,
                                         ChunkId chunkIdBegin,
                                         ChunkId chunkIdEnd,
                                         uint32_t maxNumChunkIdsToProcess = 1,
                                         void *userCtx = nullptr);

  /* Remove chunks in the range [chunkIdBegin, chunkIdEnd).
     Note that the `numChunksRemoved' in `RemoveChunksResult' might be less or equal to
     the number of chunks actually removed by storage service if the request fails and is
     automatically retried until it succeeds.
     If `moreChunksInRange' in `RemoveChunksResult' is true, there exist more than
     `maxNumChunkIdsToProcess' chunks in range [chunkIdBegin, chunkIdEnd).
   */
  virtual RemoveChunksOp createRemoveOp(ChainId chainId,
                                        ChunkId chunkIdBegin,
                                        ChunkId chunkIdEnd,
                                        uint32_t maxNumChunkIdsToProcess = 1,
                                        void *userCtx = nullptr);

  /* Truncate the chunk to `chunkLen' and create the chunk if it does not exist.
     `chunkSize' must equal to the size when the chunk was created if it already exists.
     A chunk of size `chunkSize' is created if does not exist.
     If `onlyExtendChunk' = true, extend the chunk if its length is less than `chunkLen';
     noop if its length is already greater or equal to `chunkLen'.
     The truncated/extended chunk size is returned as `lengthInfo' in the IO result;
     the user should check if the chunk size is expected.
  */
  virtual TruncateChunkOp createTruncateOp(ChainId chainId,
                                           const ChunkId &chunkId,
                                           uint32_t chunkLen,
                                           uint32_t chunkSize,
                                           bool onlyExtendChunk = false,
                                           void *userCtx = nullptr);

  // delete the returned IOBuffer object to deregister the buffer
  virtual Result<IOBuffer> registerIOBuffer(uint8_t *buf, size_t len);

  virtual CoTryTask<void> batchRead(std::span<ReadIO> readIOs,
                                    const flat::UserInfo &userInfo,
                                    const ReadOptions &options = ReadOptions(),
                                    std::vector<ReadIO *> *failedIOs = nullptr) = 0;

  virtual CoTryTask<void> batchWrite(std::span<WriteIO> writeIOs,
                                     const flat::UserInfo &userInfo,
                                     const WriteOptions &options = WriteOptions(),
                                     std::vector<WriteIO *> *failedIOs = nullptr) = 0;

  virtual CoTryTask<void> read(ReadIO &readIO,
                               const flat::UserInfo &userInfo,
                               const ReadOptions &options = ReadOptions()) = 0;

  virtual CoTryTask<void> write(WriteIO &writeIO,
                                const flat::UserInfo &userInfo,
                                const WriteOptions &options = WriteOptions()) = 0;

  // the following interfaces are assumed to be used at server-side (e.g. in meta service)

  virtual CoTryTask<void> queryLastChunk(std::span<QueryLastChunkOp> ops,
                                         const flat::UserInfo &userInfo,
                                         const ReadOptions &options = ReadOptions(),
                                         std::vector<QueryLastChunkOp *> *failedOps = nullptr) = 0;

  virtual CoTryTask<void> removeChunks(std::span<RemoveChunksOp> ops,
                                       const flat::UserInfo &userInfo,
                                       const WriteOptions &options = WriteOptions(),
                                       std::vector<RemoveChunksOp *> *failedOps = nullptr) = 0;

  virtual CoTryTask<void> truncateChunks(std::span<TruncateChunkOp> ops,
                                         const flat::UserInfo &userInfo,
                                         const WriteOptions &options = WriteOptions(),
                                         std::vector<TruncateChunkOp *> *failedOps = nullptr) = 0;

  virtual CoTryTask<SpaceInfoRsp> querySpaceInfo(NodeId nodeId) = 0;

  virtual CoTryTask<CreateTargetRsp> createTarget(NodeId nodeId, const CreateTargetReq &req) = 0;

  virtual CoTryTask<OfflineTargetRsp> offlineTarget(NodeId nodeId, const OfflineTargetReq &req) = 0;

  virtual CoTryTask<RemoveTargetRsp> removeTarget(NodeId nodeId, const RemoveTargetReq &req) = 0;

  virtual CoTryTask<std::vector<Result<QueryChunkRsp>>> queryChunk(const QueryChunkReq &req) = 0;

  virtual CoTryTask<ChunkMetaVector> getAllChunkMetadata(const ChainId &chainId, const TargetId &targetId) = 0;

 protected:
  static const Config kDefaultConfig;
  const ClientId clientId_;
  const Config &config_;
  std::atomic_uint64_t nextRequestId_ = 1;
};

}  // namespace hf3fs::storage::client

FMT_BEGIN_NAMESPACE

template <>
struct formatter<hf3fs::storage::client::RoutingTarget> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(const hf3fs::storage::client::RoutingTarget &routingTarget, FormatContext &ctx) const {
    return fmt::format_to(ctx.out(),
                          "{}@{}@{}:{}@{}:{}#{}",
                          routingTarget.chainId,
                          routingTarget.chainVer,
                          routingTarget.routingInfoVer,
                          routingTarget.targetInfo.targetId,
                          routingTarget.targetInfo.nodeId,
                          routingTarget.channel.id,
                          routingTarget.channel.seqnum);
  }
};

FMT_END_NAMESPACE

#pragma once

#include <algorithm>
#include <array>
#include <atomic>
#include <bits/types/struct_iovec.h>
#include <cassert>
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <fmt/core.h>
#include <folly/CancellationToken.h>
#include <folly/Conv.h>
#include <folly/IPAddressV4.h>
#include <folly/Likely.h>
#include <folly/MPMCQueue.h>
#include <folly/Memory.h>
#include <folly/Optional.h>
#include <folly/Range.h>
#include <folly/String.h>
#include <folly/Synchronized.h>
#include <folly/Utility.h>
#include <folly/concurrency/ConcurrentHashMap.h>
#include <folly/experimental/StampedPtr.h>
#include <folly/experimental/coro/AsyncGenerator.h>
#include <folly/experimental/coro/Baton.h>
#include <folly/experimental/coro/Promise.h>
#include <folly/experimental/coro/SharedMutex.h>
#include <folly/experimental/coro/ViaIfAsync.h>
#include <folly/fibers/BatchSemaphore.h>
#include <folly/io/coro/Transport.h>
#include <folly/lang/Bits.h>
#include <folly/logging/xlog.h>
#include <folly/net/NetworkSocket.h>
#include <functional>
#include <gtest/gtest_prod.h>
#include <infiniband/verbs.h>
#include <limits>
#include <map>
#include <memory>
#include <mutex>
#include <optional>
#include <queue>
#include <set>
#include <span>
#include <string>
#include <tuple>
#include <type_traits>
#include <unistd.h>
#include <unordered_map>
#include <utility>
#include <vector>

#include "common/monitor/Sample.h"
#include "common/net/EventLoop.h"
#include "common/net/Network.h"
#include "common/net/Socket.h"
#include "common/net/WriteItem.h"
#include "common/net/ib/IBConnect.h"
#include "common/net/ib/IBDevice.h"
#include "common/net/ib/RDMABuf.h"
#include "common/utils/Address.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Duration.h"
#include "common/utils/FdWrapper.h"
#include "common/utils/Result.h"
#include "common/utils/StatusCode.h"
#include "common/utils/UtcTime.h"

namespace hf3fs {

namespace serde {
class ClientContext;
};

namespace net {
class IBConnectService;
class IBSocketManager;

class IBSocket : public Socket, folly::MoveOnly {
 public:
  struct Config : public ConfigBase<Config> {
    // for recorder
    CONFIG_HOT_UPDATED_ITEM(record_bytes_per_peer, false);
    CONFIG_HOT_UPDATED_ITEM(record_latency_per_peer, false);

    // connection param
    CONFIG_HOT_UPDATED_ITEM(pkey_index, std::optional<uint16_t>());
    CONFIG_HOT_UPDATED_ITEM(roce_pkey_index, std::optional<uint16_t>());  // RoCE pkey should be 0
    CONFIG_HOT_UPDATED_ITEM(sl, (uint8_t)0);
    CONFIG_HOT_UPDATED_ITEM(traffic_class, std::optional<uint8_t>());  // for RoCE
    // CONFIG_HOT_UPDATED_ITEM(gid_index, std::optional<uint8_t>());      // for RoCE

    CONFIG_HOT_UPDATED_ITEM(start_psn, (uint32_t)0);
    CONFIG_HOT_UPDATED_ITEM(min_rnr_timer, (uint8_t)1);
    CONFIG_HOT_UPDATED_ITEM(timeout, (uint8_t)14);  // ibv_qp_attr.timeout
    CONFIG_HOT_UPDATED_ITEM(retry_cnt, (uint8_t)7);
    CONFIG_HOT_UPDATED_ITEM(rnr_retry, (uint8_t)0);

    CONFIG_HOT_UPDATED_ITEM(max_sge, 16u, ConfigCheckers::checkPositive);
    CONFIG_HOT_UPDATED_ITEM(max_rdma_wr, 128u, ConfigCheckers::checkPositive);  // max number of RDMA WRs in send queue
    CONFIG_HOT_UPDATED_ITEM(max_rdma_wr_per_post,
                            32u,
                            ConfigCheckers::checkPositive);  // max number of RDMA WRs per ibv_post_send
    CONFIG_HOT_UPDATED_ITEM(max_rd_atomic, 16u, ConfigCheckers::checkPositive);  // ibv_qp_attr.max_rd_atomic

    CONFIG_HOT_UPDATED_ITEM(buf_size, (16u * 1024), ConfigCheckers::checkPositive);
    CONFIG_HOT_UPDATED_ITEM(send_buf_cnt, 32u, ConfigCheckers::checkPositive);
    CONFIG_HOT_UPDATED_ITEM(buf_ack_batch, 8u, ConfigCheckers::checkPositive);
    CONFIG_HOT_UPDATED_ITEM(buf_signal_batch, 8u, ConfigCheckers::checkPositive);
    CONFIG_HOT_UPDATED_ITEM(event_ack_batch, 128u, ConfigCheckers::checkPositive);

    CONFIG_HOT_UPDATED_ITEM(drain_timeout, 5_s);
    CONFIG_HOT_UPDATED_ITEM(drop_connections, 0u);

   public:
    mutable folly::Synchronized<std::unordered_map<Address, uint64_t>> roundRobin_;
    IBConnectConfig toIBConnectConfig(bool roce) const;
  };

  using Ptr = std::unique_ptr<IBSocket>;

  IBSocket(const Config &config, IBPort port = {});
  ~IBSocket() override;

  CoTryTask<void> connect(serde::ClientContext &ctx, Duration timeout);
  Result<Void> accept(folly::IPAddressV4 ip, const IBConnectReq &req, Duration acceptTimeout = 15_s);

  std::string describe() override { return *describe_.rlock(); };
  std::string describe() const { return *describe_.rlock(); }
  folly::IPAddressV4 peerIP() override { return peerIP_; };

  int fd() const override { return channel_->fd; }

  Result<Events> poll(uint32_t events) override;

  Result<size_t> send(folly::ByteRange buf);
  Result<size_t> send(struct iovec *iov, uint32_t cnt) override;
  Result<Void> flush() override;
  Result<size_t> recv(folly::MutableByteRange buf) override;

  CoTask<void> close();

  // Send a empty message to check the liveness of the socket.
  // - if socket is known to be broken, return corresponding error.
  // - else send check message and return immediately
  //   - if the message cannot be delivered, an error will be returned in future during poll.
  Result<Void> check() override;

  // Socket should already turn from ACCEPTED state to READY or ERROR state
  bool checkConnectFinished() const;

  /** RDMA operations. */
  CoTryTask<void> rdmaRead(const RDMARemoteBuf &remoteBuf, RDMABuf &localBuf) {
    co_return co_await rdmaRead(remoteBuf, std::span(&localBuf, 1));
  }
  CoTryTask<void> rdmaRead(const RDMARemoteBuf &remoteBuf, std::span<RDMABuf> localBufs) {
    co_return co_await rdma(IBV_WR_RDMA_READ, remoteBuf, localBufs);
  }

  CoTryTask<void> rdmaWrite(const RDMARemoteBuf &remoteBuf, RDMABuf &localBuf) {
    co_return co_await rdmaWrite(remoteBuf, std::span(&localBuf, 1));
  }
  CoTryTask<void> rdmaWrite(const RDMARemoteBuf &remoteBuf, std::span<RDMABuf> localBufs) {
    co_return co_await rdma(IBV_WR_RDMA_WRITE, remoteBuf, localBufs);
  }

 private:
  struct RDMAReq {
    uint64_t raddr;
    uint32_t rkey;
    uint32_t localBufFirst;
    uint32_t localBufCnt;

    RDMAReq()
        : RDMAReq(0, 0, 0, 0) {}
    RDMAReq(uint64_t raddr, uint32_t rkey, uint32_t localBufFirst, uint32_t localBufCnt)
        : raddr(raddr),
          rkey(rkey),
          localBufFirst(localBufFirst),
          localBufCnt(localBufCnt) {}
  };

 public:
  class RDMAReqBatch {
   public:
    RDMAReqBatch()
        : RDMAReqBatch(nullptr, ibv_wr_opcode(-1)) {}
    RDMAReqBatch(IBSocket *socket, ibv_wr_opcode opcode)
        : socket_(socket),
          opcode_(opcode),
          reqs_(),
          localBufs_() {}

    Result<Void> add(const RDMARemoteBuf &remoteBuf, RDMABuf localBuf);
    Result<Void> add(RDMARemoteBuf remoteBuf, std::span<RDMABuf> localBufs);

    void reserve(size_t numReqs, size_t numLocalBufs) {
      reqs_.reserve(numReqs);
      localBufs_.reserve(numLocalBufs);
    }

    void clear() {
      reqs_.clear();
      localBufs_.clear();
      waitLatency_ = std::chrono::nanoseconds(0);
      transferLatency_ = std::chrono::nanoseconds(0);
    }

    ibv_wr_opcode opcode() const { return opcode_; }
    CoTryTask<void> post() {
      co_return co_await socket_->rdmaBatch(opcode_, reqs_, localBufs_, waitLatency_, transferLatency_);
    }

    std::chrono::nanoseconds waitLatency() const { return waitLatency_; };
    std::chrono::nanoseconds transferLatency() const { return transferLatency_; };

   private:
    IBSocket *socket_;
    ibv_wr_opcode opcode_;
    std::vector<RDMAReq> reqs_;
    std::vector<RDMABuf> localBufs_;
    std::chrono::nanoseconds waitLatency_;
    std::chrono::nanoseconds transferLatency_;
  };

  RDMAReqBatch rdmaReadBatch() { return RDMAReqBatch(this, IBV_WR_RDMA_READ); }
  RDMAReqBatch rdmaWriteBatch() { return RDMAReqBatch(this, IBV_WR_RDMA_WRITE); }

  const IBDevice *device() const { return port_.dev(); }

 private:
  static constexpr size_t kRDMAPostBatch = 8;

  enum class State {
    INIT,
    CONNECTING,
    ACCEPTED,
    READY,
    CLOSE,
    ERROR,
  };

  struct RDMAPostCtx {
    std::optional<std::reference_wrapper<folly::fibers::BatchSemaphore>> sem;
    std::optional<folly::fibers::BatchSemaphore::Waiter> waiter;

    ibv_wr_opcode opcode;
    std::span<const RDMAReq> reqs;
    std::span<RDMABuf> localBufs;
    size_t bytes = 0;

    folly::coro::Baton baton;
    ibv_wc_status status = IBV_WC_SUCCESS;
    std::chrono::steady_clock::time_point postBegin;
    std::chrono::steady_clock::time_point postEnd;

    CoTask<void> waitSem() {
      if (waiter.has_value()) {
        co_await waiter->baton;
      }
    }

    bool setError(ibv_wc_status error) {
      if (status == IBV_WC_SUCCESS) {
        status = error;
        return true;
      }
      return false;
    }

    __attribute__((no_sanitize("thread"))) void finish() {
      postEnd = std::chrono::steady_clock::now();
      baton.post();
    }
  };

  enum class WRType : uint16_t { SEND, RECV, ACK, RDMA, RDMA_LAST, CLOSE, CHECK };

 public:  // note: public to formatter
  struct WRId : private folly::StampedPtr<void> {
    using Base = folly::StampedPtr<void>;

    static uint64_t send(uint32_t signalCount) { return WRId::pack(signalCount, WRType::SEND); }
    static uint64_t recv(uint32_t bufIndex) { return WRId::pack(bufIndex, WRType::RECV); }
    static uint64_t ack() { return WRId::pack(nullptr, WRType::ACK); }
    static uint64_t rdma(RDMAPostCtx *ptr, bool last) {
      return WRId::pack(ptr, last ? WRType::RDMA_LAST : WRType::RDMA);
    }
    static uint64_t close() { return WRId::pack(nullptr, WRType::CLOSE); }
    static uint64_t check() { return WRId::pack(nullptr, WRType::CHECK); }

    static uint64_t pack(uint32_t val, WRType type) { return Base::pack((void *)(uint64_t)val, (uint16_t)type); }
    static uint64_t pack(void *ptr, WRType type) { return Base::pack(ptr, (uint16_t)type); }

    WRId(uint64_t raw)
        : Base{raw} {}

    WRType type() const { return static_cast<WRType>(stamp()); }

    uint32_t sendSignalCount() const {
      assert(type() == WRType::SEND);
      return (uint32_t)(uint64_t)ptr();
    }
    uint32_t recvBufIndex() const {
      assert(type() == WRType::RECV);
      return (uint32_t)(uint64_t)ptr();
    }
    RDMAPostCtx *rdmaPostCtx() const {
      assert(type() == WRType::RDMA || type() == WRType::RDMA_LAST);
      return (RDMAPostCtx *)ptr();
    }

    std::string describe() const {
      switch (type()) {
        case WRType::SEND:
          return fmt::format("[WRType::SEND, signal {}]", sendSignalCount());
        case WRType::RECV:
          return fmt::format("[WRType::RECV, bufIndex {}]", recvBufIndex());
        case WRType::ACK:
          return fmt::format("[WRType::ACK]");
        case WRType::RDMA:
          return fmt::format("[WRType::RDMA, ctx {}]", ptr());
        case WRType::RDMA_LAST:
          return fmt::format("[WRTpe::RDMA_LAST, ctx {}]", ptr());
        case WRType::CLOSE:
          return fmt::format("[WRType::CLOSE]");
        case WRType::CHECK:
          return fmt::format("[WRType::CHECK]");
      }
      assert(false);
      return fmt::format("[INVALID 0x{:016x}]", raw);
    }

    bool operator==(const WRId &other) const { return raw == other.raw; }
  };

  class ImmData {
    static constexpr size_t kTypeOffset = 24;
    static constexpr uint32_t kValMax = (1 << kTypeOffset) - 1;
    static constexpr uint32_t kValMask = kValMax;
    uint32_t val;

   public:
    enum class Type : uint8_t {
      ACK,
      CLOSE,
    };

    ImmData(__be32 val = 0)
        : val(folly::Endian::big32(val)) {}

    static ImmData ack(uint32_t count) { return create(Type::ACK, count); }
    static ImmData close() { return create(Type::CLOSE, 0); }
    static ImmData create(Type type, uint32_t val) {
      assert(type == Type::ACK || type == Type::CLOSE);
      assert(val <= kValMax);
      return folly::Endian::big32(((uint8_t)type << kTypeOffset) | val);
    }

    Type type() const { return Type(val >> kTypeOffset); }
    uint32_t data() const { return val & kValMask; }

    std::string describe() const {
      switch (type()) {
        case Type::ACK:
          return fmt::format("[ACK, count {}]", data());
        case Type::CLOSE:
          return fmt::format("[CLOSE]");
        default:
          return fmt::format("[INVALID 0x{:08x}]", val);
      }
    }

    operator __be32() const { return folly::Endian::big(val); }
  };
  static_assert(sizeof(ImmData) == sizeof(__be32));

 private:
  struct BufferMem : folly::MoveOnly {
    const IBDevice *dev = nullptr;
    int64_t total = 0;
    uint8_t *ptr = nullptr;
    ibv_mr *mr = nullptr;

    static Result<BufferMem> create(const IBDevice *dev, size_t bufSize, size_t bufCnt, unsigned int flags);
    BufferMem() = default;
    BufferMem(BufferMem &&o)
        : dev(std::exchange(o.dev, nullptr)),
          total(std::exchange(o.total, 0)),
          ptr(std::exchange(o.ptr, nullptr)),
          mr(std::exchange(o.mr, nullptr)) {}
    ~BufferMem();
  };

  class Buffers {
   public:
    void init(uint8_t *ptr, ibv_mr *mr, size_t bufSize, size_t bufCnt) {
      ptr_ = ptr;
      mr_ = mr;
      bufSize_ = bufSize;
      bufCnt_ = bufCnt;
    }

    uint8_t *getBuf(size_t idx) { return ptr_ + idx * bufSize_; }
    ibv_mr *getMr() { return mr_; }
    size_t getBufCnt() const { return bufCnt_; }
    size_t getBufSize() const { return bufSize_; }

   protected:
    uint8_t *ptr_ = nullptr;
    ibv_mr *mr_ = nullptr;
    size_t bufSize_ = 0;
    size_t bufCnt_ = 0;
  };

  class SendBuffers : public Buffers {
   public:
    void init(uint8_t *ptr, ibv_mr *mr, size_t bufSize, size_t bufCnt);
    void push(size_t cnts);
    bool empty();
    std::pair<size_t, folly::MutableByteRange &> front();
    void pop();

   private:
    alignas(folly::hardware_destructive_interference_size) std::atomic<uint64_t> tailIdx_{0};
    alignas(folly::hardware_destructive_interference_size) std::atomic<uint64_t> frontIdx_{0};
    std::pair<size_t, folly::MutableByteRange> front_;
  };

  class RecvBuffers : public Buffers {
   public:
    void init(uint8_t *ptr, ibv_mr *mr, size_t bufSize, size_t bufCnt);
    void push(uint32_t idx, uint32_t len);
    bool empty();
    std::pair<size_t, folly::ByteRange &> front();
    bool pop();

   private:
    folly::MPMCQueue<std::pair<uint32_t, uint32_t>> queue_;
    std::optional<std::pair<size_t, folly::ByteRange>> front_;
  };

  friend class IBConnectService;

  int checkConfig() const;
  Result<Void> checkPort() const;
  Result<IBConnectInfo> getConnectInfo() const;
  void setPeerInfo(folly::IPAddressV4 ip, const IBConnectInfo &info);

  // ibv_create_qp & ibv_modify_qp
  [[nodiscard]] int qpCreate();
  [[nodiscard]] int qpInit();
  [[nodiscard]] int qpReadyToRecv();
  [[nodiscard]] int qpReadyToSend();
  void qpError();

  Result<Void> checkState();

  void cqPoll(Events &events);
  int cqGetEvent();
  int cqRequestNotify();
  void wcError(const ibv_wc &wc);
  int wcSuccess(const ibv_wc &wc, Events &events);
  int onSended(const ibv_wc &wc, Events &events);
  int onRecved(const ibv_wc &wc, Events &events);
  int onAckSended(const ibv_wc &wc, Events &events);
  int onRDMAFinished(const ibv_wc &wc, Events &events);
  int onImmData(ImmData imm, Events &events);

  int initBufs();
  int postSend(uint32_t idx, size_t len, uint32_t flags = 0);
  int postRecv(uint32_t idx);
  int postAck();

  friend class IBSocketManager;
  class Drainer : public EventLoop::EventHandler, public std::enable_shared_from_this<Drainer> {
   public:
    using Ptr = std::shared_ptr<Drainer>;
    using WeakPtr = std::weak_ptr<Drainer>;

    static Ptr create(IBSocket::Ptr socket, std::weak_ptr<IBSocketManager> manager);

    Drainer(IBSocket::Ptr socket, std::weak_ptr<IBSocketManager> manager);
    ~Drainer() override;

    std::string describe() const { return socket_->describe(); }

    int fd() const override { return socket_->fd(); }
    void handleEvents(uint32_t /* epollEvents */) override;

   private:
    IBSocket::Ptr socket_;
    std::weak_ptr<IBSocketManager> manager_;
  };
  bool closeGracefully();
  bool drain();

  CoTryTask<void> rdma(ibv_wr_opcode opcode, const RDMARemoteBuf &remoteBuf, std::span<RDMABuf> localBufs) {
    RDMAReqBatch batch(this, opcode);
    CO_RETURN_ON_ERROR(batch.add(remoteBuf, localBufs));
    co_return co_await batch.post();
  }

  CoTryTask<void> rdmaBatch(ibv_wr_opcode opcode,
                            const std::span<const RDMAReq> reqs,
                            const std::span<RDMABuf> localBufs,
                            std::chrono::nanoseconds &waitLatency,
                            std::chrono::nanoseconds &transferLatency);
  CoTryTask<void> rdmaPost(RDMAPostCtx &ctx);
  int rdmaPostWR(RDMAPostCtx &ctx);

  const Config &config_;
  IBConnectConfig connectConfig_;

  IBPort port_;
  std::unique_ptr<ibv_comp_channel, IBDevice::Deleter> channel_;
  std::unique_ptr<ibv_cq, IBDevice::Deleter> cq_;
  std::unique_ptr<ibv_qp, IBDevice::Deleter> qp_;

  folly::IPAddressV4 peerIP_;  // IP of TCP socket to setup connection
  IBConnectInfo peerInfo_;
  folly::Synchronized<std::string> describe_;

  monitor::TagSet tag_;
  monitor::TagSet peerTag_;

  SteadyTime acceptTimeout_;
  std::atomic<State> state_ = State::INIT;
  std::atomic<bool> closed_ = false;
  std::atomic<bool> closeMsgSended_ = false;
  size_t unackedEvents_ = 0;

  // memory
  std::optional<BufferMem> mem_;

  // socket send
  SendBuffers sendBufs_;
  size_t sendNotSignaled_ = 0;
  size_t sendSignaled_ = 0;
  size_t sendAcked_ = 0;
  std::atomic<uint64_t> sendWaitBufBegin_ = 0;

  // socket recv
  RecvBuffers recvBufs_;
  uint32_t recvNotAcked_ = 0;
  std::atomic<int32_t> ackBufAvailable_ = 0;

  // RDMA
  folly::fibers::BatchSemaphore rdmaSem_{0};

  // check
  std::atomic<bool> checkMsgSended_ = false;
};

class IBSocketManager : public EventLoop::EventHandler, public std::enable_shared_from_this<IBSocketManager> {
 public:
  static std::shared_ptr<IBSocketManager> create();

  IBSocketManager(int fd)
      : timer_(fd) {}

  void stopAndJoin();

  void close(IBSocket::Ptr socket);

  int fd() const override { return timer_; }
  void handleEvents(uint32_t /* events */) override;

 private:
  friend class IBSocket::Drainer;
  void remove(const IBSocket::Drainer::Ptr &drainer) { drainers_.lock()->erase(drainer); }

  FdWrapper timer_;
  folly::Synchronized<std::set<IBSocket::Drainer::Ptr>, std::mutex> drainers_;
  folly::Synchronized<std::multimap<SteadyTime, IBSocket::Drainer::WeakPtr>, std::mutex> deadlines_;
};

}  // namespace net
}  // namespace hf3fs

#pragma once

#include "common/net/Transport.h"
#include "common/net/ib/IBSocket.h"
#include "common/serde/MessagePacket.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Size.h"
#include "common/utils/Tracing.h"
#include "common/utils/VersionInfo.h"

namespace hf3fs::serde {

class CallContext {
 public:
  using MethodType = CoTask<void> (CallContext::*)();
  using OnErrorType = Status (CallContext::*)(Status);
  struct ServiceWrapper {
    using MethodGetter = MethodType (*)(uint16_t);
    MethodGetter getter = &CallContext::invalidServiceId;
    OnErrorType onError = &CallContext::serviceOnError;
    void *object = nullptr;
    std::shared_ptr<void *> alive = nullptr;
  };

  CallContext(MessagePacket<> &packet, net::TransportPtr tr, ServiceWrapper &service)
      : packet_(packet),
        tr_(std::move(tr)),
        service_(service) {}

  const auto &packet() { return packet_; }
  auto &transport() const { return tr_; }
  auto &responseOptions() { return responseOptions_; }
  std::string peer() { return LIKELY(tr_ != nullptr) ? tr_->peerIP().str() : std::string{"null"}; }

  CoTask<void> handle() {
    auto method = service_.getter(packet_.methodId);
    co_await (this->*method)();
  }

  CoTask<void> invalidId() {
    XLOGF(INFO, "method {}:{} not found!", packet_.serviceId, packet_.methodId);
    onError(makeError(RPCCode::kInvalidMethodID));
    co_return;
  }

  template <class F>
  CoTask<void> call() {
    // deserialize payload.
    if (packet_.timestamp) {
      packet_.timestamp->serverWaked = UtcClock::now().time_since_epoch().count();
    }
    typename F::ReqType req;
    auto deserializeResult = serde::deserialize(req, packet_.payload);
    if (UNLIKELY(!deserializeResult)) {
      onDeserializeFailed();
      co_return;
    }

    // call method.
    auto obj = reinterpret_cast<typename F::Object *>(service_.object);
    auto result = co_await folly::coro::co_awaitTry((obj->*F::method)(*this, req));
    if (UNLIKELY(result.hasException())) {
      XLOGF(FATAL,
            "Processor has exception: {}, request {}:{} {}",
            result.exception().what(),
            packet_.serviceId,
            packet_.methodId,
            serde::toJsonString(req));
      co_return;
    }
    if (packet_.timestamp) {
      packet_.timestamp->serverProcessed = UtcClock::now().time_since_epoch().count();
    }
    makeResponse(result.value());
    co_return;
  }

  void onError(folly::Unexpected<Status> &&status) {
    makeResponse(Result<Void>(makeError((this->*service_.onError)(std::move(status.error())))));
  }

  Status serviceOnError(Status status) { return status; }

  template <class Object, auto Method>
  Status customOnError(Status status) {
    auto obj = reinterpret_cast<Object *>(service_.object);
    return (obj->*Method)(std::move(status));
  }

  void onDeserializeFailed();

  tracing::Points &tracingPoints() { return tracingPoints_; }

  class RDMATransmission {
   public:
    RDMATransmission(CallContext &ctx, ibv_wr_opcode opcode)
        : ctx_(ctx),
          batch_(ctx_.tr_->ibSocket(), opcode) {}

    Result<Void> add(const net::RDMARemoteBuf &remoteBuf, net::RDMABuf localBuf) {
      return batch_.add(remoteBuf, localBuf);
    }

    CoTask<void> applyTransmission(Duration timeout);

    CoTryTask<void> post() { return batch_.post(); }

   private:
    CallContext &ctx_;
    net::IBSocket::RDMAReqBatch batch_;
  };

  RDMATransmission readTransmission() { return RDMATransmission{*this, IBV_WR_RDMA_READ}; }
  RDMATransmission writeTransmission() { return RDMATransmission{*this, IBV_WR_RDMA_WRITE}; }

 private:
  static MethodType invalidServiceId(uint16_t) { return &CallContext::invalidId; }

  void makeResponse(const auto &payload) {
    MessagePacket send(payload);
    send.uuid = packet_.uuid;
    send.serviceId = packet_.serviceId;
    send.methodId = packet_.methodId;
    send.flags = 0;
    send.version = packet_.version;
    send.timestamp = packet_.timestamp;
    tr_->send(net::WriteList(net::WriteItem::createMessage(send, responseOptions_)));
  }

 private:
  MessagePacket<> &packet_;
  net::TransportPtr tr_;
  ServiceWrapper &service_;

  net::CoreRequestOptions responseOptions_;
  tracing::Points tracingPoints_;
};

}  // namespace hf3fs::serde

#pragma once

#include <folly/experimental/coro/Baton.h>
#include <utility>

#include "chunk_engine/src/cxx.rs.h"
#include "common/net/ib/IBSocket.h"
#include "common/serde/CallContext.h"
#include "common/utils/Duration.h"
#include "fbs/storage/Common.h"
#include "storage/store/ChunkMetadata.h"

namespace hf3fs::storage {

class BatchReadJob;
class StorageTarget;

class ChunkEngineReadJob {
 public:
  ChunkEngineReadJob() = default;
  ChunkEngineReadJob(const ChunkEngineReadJob &) = delete;
  ChunkEngineReadJob(ChunkEngineReadJob &&other)
      : engine_(std::exchange(other.engine_, nullptr)),
        chunk_(std::exchange(other.chunk_, nullptr)) {}

  void set(chunk_engine::Engine *engine, const chunk_engine::Chunk *chunk) {
    reset();
    engine_ = engine;
    chunk_ = chunk;
  }

  void reset() {
    if (engine_ && chunk_) {
      std::exchange(engine_, nullptr)->release_raw_chunk(chunk_);
    }
  }

  auto chunk() const { return chunk_; }

  bool has_chunk() const { return chunk_ != nullptr; }

  ~ChunkEngineReadJob() { reset(); }

 private:
  chunk_engine::Engine *engine_{};
  const chunk_engine::Chunk *chunk_{};
};

class AioReadJob {
 public:
  AioReadJob(const ReadIO &readIO, IOResult &result, BatchReadJob &batch);

  auto &readIO() { return readIO_; }
  auto &result() { return result_; }
  auto &batch() { return batch_; }
  auto &state() { return state_; }

  void setResult(Result<uint32_t> lengthInfo);

  uint32_t alignedOffset() const { return readIO_.offset - state_.headLength; }
  uint32_t alignedLength() const { return readIO_.length + state_.headLength + state_.tailLength; }

  auto startTime() const { return startTime_; }
  void resetStartTime() { startTime_ = RelativeTime::now(); }

 private:
  const ReadIO &readIO_;
  IOResult &result_;
  BatchReadJob &batch_;
  struct State {
    net::RDMABuf localbuf{};
    StorageTarget *storageTarget = nullptr;
    ChunkEngineReadJob chunkEngineJob{};
    SERDE_STRUCT_FIELD(headLength, uint32_t{});
    SERDE_STRUCT_FIELD(tailLength, uint32_t{});
    SERDE_STRUCT_FIELD(readLength, uint32_t{});  // after cropping.
    SERDE_STRUCT_FIELD(readFd, int32_t{});
    SERDE_STRUCT_FIELD(readOffset, uint64_t{});
    SERDE_STRUCT_FIELD(chunkLen, uint32_t{});
    SERDE_STRUCT_FIELD(bufferIndex, uint32_t{});
    SERDE_STRUCT_FIELD(fdIndex, std::optional<uint32_t>{});
    SERDE_STRUCT_FIELD(chunkChecksum, ChecksumInfo{});
    SERDE_STRUCT_FIELD(readUncommitted, false);
  } state_;
  static_assert(serde::Serializable<State>);
  RelativeTime startTime_{};
};

class BatchReadJob {
 public:
  BatchReadJob(std::span<const ReadIO> readIOs, std::span<IOResult> results, ChecksumType checksumType);
  BatchReadJob(const ReadIO &readIO, StorageTarget *target, IOResult &result, ChecksumType checksumType)
      : BatchReadJob(std::span(&readIO, 1), std::span(&result, 1), checksumType) {
    jobs_.back().state().storageTarget = target;
  }
  CoTask<void> complete() { co_await baton_; }
  size_t addBufferToBatch(serde::CallContext::RDMATransmission &batch);
  size_t copyToRespBuffer(std::vector<uint8_t> &buffer);
  void finish(AioReadJob *job);
  auto checksumType() const { return checksumType_; }
  bool recalculateChecksum() const { return recalculateChecksum_; }
  void setRecalculateChecksum(bool value = true) { recalculateChecksum_ = value; }
  auto &front() { return jobs_.front(); }
  auto &front() const { return jobs_.front(); }
  auto startTime() const { return startTime_.load(); }
  void resetStartTime() { startTime_ = RelativeTime::now(); }

 private:
  friend class AioReadJobIterator;
  std::vector<AioReadJob> jobs_;
  folly::coro::Baton baton_;
  std::atomic<uint64_t> finishedCount_{};
  std::atomic<RelativeTime> startTime_ = RelativeTime::now();
  const ChecksumType checksumType_;
  bool recalculateChecksum_ = false;
};

class AioReadJobIterator {
 public:
  AioReadJobIterator() = default;
  AioReadJobIterator(BatchReadJob *batch)
      : batch_(batch),
        end_(batch->jobs_.size()) {}
  AioReadJobIterator(BatchReadJob *batch, uint32_t start, uint32_t size)
      : batch_(batch),
        begin_(start),
        end_(std::min((uint32_t)batch->jobs_.size(), start + size)) {}

  operator bool() const { return begin_ < end_; }
  bool isNull() const { return batch_ == nullptr; }
  AioReadJob &operator*() { return batch_->jobs_[begin_]; }
  AioReadJob *operator->() { return &batch_->jobs_[begin_]; }
  AioReadJob *operator++(int) { return &batch_->jobs_[begin_++]; }

  auto startTime() const { return startTime_; }
  auto resetStartTime() { startTime_ = RelativeTime::now(); }

 private:
  BatchReadJob *batch_ = nullptr;
  uint32_t begin_ = 0;
  uint32_t end_ = 0;
  RelativeTime startTime_ = RelativeTime::now();
};

}  // namespace hf3fs::storage

#pragma once

#include <atomic>
#include <folly/Likely.h>
#include <memory>
#include <vector>

#include "common/utils/Status.h"

namespace hf3fs {

template <typename T>
class MPSCQueue {
 public:
  explicit MPSCQueue(uint64_t capacity)
      : capacity_(capacity),
        items_(std::make_unique<std::unique_ptr<T>[]>(capacity)) {}

  uint64_t size() const {
    auto t = tail_.load(std::memory_order_acquire);
    auto h = head_.load(std::memory_order_acquire);
    return h - t;
  }
  bool empty() const { return size() == 0; }
  bool full() const { return size() >= capacity_; }

  Status pop(std::unique_ptr<T> *item) {
    auto t = tail_.load(std::memory_order_acquire);
    auto h = head_.load(std::memory_order_acquire);

    if (h == t) {
      return Status(StatusCode::kQueueEmpty);
    }

    auto &pop = items_[t % capacity_];
    if (UNLIKELY(pop == nullptr)) {
      return Status(StatusCode::kQueueConflict);
    }
    *item = std::move(pop);

    ++tail_;
    return Status::OK;
  }

  Status push(std::unique_ptr<T> *item, int retry_times = 5) {
    if (UNLIKELY((item->get() == nullptr))) {
      return Status(StatusCode::kQueueInvalidItem);
    }

    for (int i = 0; i < retry_times || retry_times == -1; ++i) {
      auto t = tail_.load(std::memory_order_acquire);
      auto h = head_.load(std::memory_order_acquire);

      if (UNLIKELY(h - t >= capacity_)) {
        return Status(StatusCode::kQueueFull);
      }

      // CAS
      if (head_.compare_exchange_strong(h, h + 1)) {
        item->swap(items_[h % capacity_]);
        return Status::OK;
      }
    }

    return Status(StatusCode::kQueueConflict);
  }

 private:
  const uint64_t capacity_;
  std::unique_ptr<std::unique_ptr<T>[]> items_;

  alignas(64) std::atomic<uint64_t> head_{0};
  alignas(64) std::atomic<uint64_t> tail_{0};
};
static_assert(sizeof(MPSCQueue<int>) == 3 * 64, "sizeof(MPSCQueue) != 3* 64");

}  // namespace hf3fs

#pragma once

#include <folly/Random.h>
#include <folly/concurrency/UnboundedQueue.h>
#include <future>

#include "SerdeObjectWriter.h"
#include "common/monitor/Recorder.h"
#include "common/monitor/ScopedMetricsWriter.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/Path.h"
#include "common/utils/SysResource.h"
#include "common/utils/UtcTime.h"

namespace hf3fs::analytics {

template <serde::SerdeType SerdeType>
class StructuredTraceLog : public folly::MoveOnly {
  struct TraceMeta {
    SERDE_STRUCT_FIELD(timestamp, std::time_t{});
    SERDE_STRUCT_FIELD(hostname, String{});
  };

  struct StructuredTrace {
    SERDE_STRUCT_FIELD(trace_meta, TraceMeta{});
    SERDE_STRUCT_FIELD(_, SerdeType{});
  };

  using WriterType = SerdeObjectWriter<StructuredTrace>;
  using WriterPtr = std::shared_ptr<WriterType>;

 public:
  class Config : public hf3fs::ConfigBase<Config> {
   public:
    CONFIG_ITEM(trace_file_dir, Path{"."});
#ifndef NDEBUG
    CONFIG_HOT_UPDATED_ITEM(enabled, false);
    CONFIG_HOT_UPDATED_ITEM(dump_interval, 60_min);
#else
    CONFIG_HOT_UPDATED_ITEM(enabled, true);
    CONFIG_HOT_UPDATED_ITEM(dump_interval, 30_s);
#endif
    CONFIG_HOT_UPDATED_ITEM(max_num_writers, size_t{1}, ConfigCheckers::checkPositive);
    CONFIG_HOT_UPDATED_ITEM(max_row_group_length, size_t{100'000});
  };

 public:
  StructuredTraceLog(const Config &config)
      : config_(config),
        enabled_(config.enabled()),
        typename_(nameof::nameof_short_type<SerdeType>()),
        hostname_(SysResource::hostname().value_or("unknown_host")),
        latencyTagSet_({{"tag", typename_}, {"instance", fmt::to_string(fmt::ptr(this))}}),
        createLatency_("trace_log.create_latency", latencyTagSet_),
        appendLatency_("trace_log.append_latency", latencyTagSet_),
        flushLatency_("trace_log.flush_latency", latencyTagSet_),
        maxNumWriters_(config.max_num_writers()) {
    onConfigUpdated_ = config_.addCallbackGuard([this]() {
      bool enabled = config_.enabled();
      if (enabled_ != enabled) {
        enableTraceLog(enabled);
        if (!enabled) flush(false /*async*/);
      }

      if (maxNumWriters_ != config_.max_num_writers()) {
        updateMaxNumWriters(config_.max_num_writers());
      }
    });

    uint64_t secsUntilFirstDump =
        folly::Random::rand64(config_.dump_interval().asSec().count() / 2, config_.dump_interval().asSec().count());
    nextDumpTime_ = microsecondsSinceEpoch(UtcClock::now() + std::chrono::seconds{secsUntilFirstDump});
  }

  ~StructuredTraceLog() { close(); }

  bool open() {
    auto writer = getOrCreateWriter();
    if (!writer) return false;
    writerPool_.enqueue(writer);
    return true;
  }

  std::shared_ptr<SerdeType> newEntry(const SerdeType &init = SerdeType{}) {
    auto ptr = new SerdeType(init);
    return std::shared_ptr<SerdeType>(ptr, [this](SerdeType *ptr) {
      this->append(*ptr);
      delete ptr;
    });
  }

  void append(const SerdeType &msg) {
    if (!enabled_) return;

    {
      monitor::ScopedLatencyWriter appendLatency(appendLatency_);
      StructuredTrace trace{
          .trace_meta = TraceMeta{.timestamp = UtcClock::secondsSinceEpoch(), .hostname = hostname_},
          ._ = msg,
      };

      WriterPtr writer = getOrCreateWriter();

      if (UNLIKELY(writer == nullptr)) {
        XLOGF(CRITICAL, "Cannot get a writer of {} trace log in directory {}", typename_, config_.trace_file_dir());
        enableTraceLog(false);
        return;
      }

      *writer << trace;
      auto writerOk = writer->ok();
      writerPool_.enqueue(std::move(writer));
      if (UNLIKELY(!writerOk)) enableTraceLog(false);
    }

    auto currentTime = microsecondsSinceEpoch(UtcClock::now());

    if (UNLIKELY(currentTime >= nextDumpTime_)) {
      nextDumpTime_ = currentTime + config_.dump_interval().asUs().count();
      flush(true /*async*/);
    }
  }

  void flush(bool async, bool shutdown = false) {
    auto running = dumpingTrace_.test_and_set();
    if (running) return;

    monitor::ScopedLatencyWriter flushLatency(flushLatency_);
    if (asyncFlush_.valid()) asyncFlush_.wait();

    asyncFlush_ = std::async(
        std::launch::async,
        [this](bool shutdown) {
          size_t numWritersToClose = numWriters_.load();
          auto now = UtcClock::now();

          XLOGF(INFO,
                "Flushing {} {} log writers in directory {}",
                numWritersToClose,
                typename_,
                config_.trace_file_dir());

          for (size_t i = 0; numWritersToClose > 0; i++) {
            // give up flushing old writers after trying for too many loops
            if (i >= 10 * maxNumWriters_) {
              break;
            }

            auto writer = writerPool_.dequeue();
            if (!writer) continue;

            if (writer->createTime() > now) {
              writerPool_.enqueue(writer);
              continue;
            }

            if (writer->ok()) {
              // add an empty trace at the end of log
              *writer << StructuredTrace{
                  .trace_meta = {.timestamp = UtcClock::secondsSinceEpoch(), .hostname = hostname_}};
            }

            try {
              writer.reset();
            } catch (const std::exception &ex) {
              XLOGF(ERR,
                    "Failed to close {} log writer in directory {}, error: {}",
                    typename_,
                    config_.trace_file_dir(),
                    ex.what());
            }

            if (shutdown)
              numWriters_--;
            else
              writerPool_.enqueue(createNewWriter());

            numWritersToClose--;
          }

          if (numWritersToClose > 0) {
            XLOGF(WARN,
                  "Still have {} {} log writers not closed in directory {}",
                  numWritersToClose,
                  typename_,
                  config_.trace_file_dir());
          } else {
            XLOGF(INFO, "Flushed {} trace log in directory {}", typename_, config_.trace_file_dir());
          }
        },
        shutdown);

    if (!async) asyncFlush_.wait();
    dumpingTrace_.clear();
  }

  void close() {
    enableTraceLog(false);
    flush(false /*async*/, true /*shutdown*/);
    XLOGF(INFO, "Closed {} trace log in directory {}", typename_, config_.trace_file_dir());
  }

 private:
  uint64_t microsecondsSinceEpoch(const UtcTime &time) const {
    return std::chrono::duration_cast<std::chrono::microseconds>((time).time_since_epoch()).count();
  }

  WriterPtr getOrCreateWriter() {
    WriterPtr writer;
    if (writerPool_.try_dequeue(writer)) return writer;

    auto currentNumWriters = numWriters_.load();
    if (currentNumWriters < maxNumWriters_) {
      bool create = numWriters_.compare_exchange_strong(currentNumWriters, currentNumWriters + 1);
      if (create) return createNewWriter();
    }

    return writerPool_.dequeue();
  }

  WriterPtr createNewWriter() {
    monitor::ScopedLatencyWriter createLatency(createLatency_);
    auto timestamp = fmt::localtime(UtcClock::to_time_t(UtcClock::now()));
    Path logfilePath =
        config_.trace_file_dir() / Path{fmt::format("{:%Y-%m-%d}", timestamp)} / Path{hostname_} /
        Path{
            fmt::format("{}.{}.{:%Y-%m-%d-%H-%M-%S}.{}.parquet", typename_, hostname_, timestamp, nextLogFileIndex_++)};

    if (!boost::filesystem::exists(logfilePath.parent_path())) {
      boost::system::error_code err{};
      boost::filesystem::create_directories(logfilePath.parent_path(), err);
      if (UNLIKELY(err.failed())) {
        XLOGF(CRITICAL, "Failed to create directory {}, error: {}", logfilePath.parent_path(), err.message());
        return nullptr;
      }
    }

    XLOGF(INFO, "Opening {} trace log: {}", typename_, logfilePath);
    return WriterType::open(logfilePath, false /*append*/, config_.max_row_group_length());
  }

  void enableTraceLog(bool enable) {
    enabled_ = enable;
    XLOGF(INFO,
          "{} {} trace log in directory {}",
          enable ? "Enabled" : "Disabled",
          typename_,
          config_.trace_file_dir());
  }

  void updateMaxNumWriters(size_t newMaxNumWriters) {
    XLOGF(INFO,
          "Update max num of writers from {} to {} for {} trace log in directory {}",
          maxNumWriters_.load(),
          newMaxNumWriters,
          typename_,
          config_.trace_file_dir());
    bool doFlush = maxNumWriters_ > newMaxNumWriters;
    maxNumWriters_ = newMaxNumWriters;
    if (doFlush) flush(false /*async*/);
  }

 private:
  const Config &config_;
  bool enabled_ = false;
  const std::string typename_;
  const std::string hostname_;

  const monitor::TagSet latencyTagSet_;
  monitor::LatencyRecorder createLatency_;
  monitor::LatencyRecorder appendLatency_;
  monitor::LatencyRecorder flushLatency_;

  std::unique_ptr<ConfigCallbackGuard> onConfigUpdated_;
  std::atomic_size_t maxNumWriters_;
  std::atomic_size_t numWriters_ = 0;
  std::atomic_size_t nextLogFileIndex_ = 1;
  folly::UnboundedQueue<WriterPtr, false, false, true> writerPool_;

  std::atomic_uint64_t nextDumpTime_;
  std::atomic_flag dumpingTrace_;
  std::future<void> asyncFlush_;
};

}  // namespace hf3fs::analytics

#pragma once

#include <array>
#include <atomic>
#include <chrono>
#include <condition_variable>
#include <folly/experimental/coro/Baton.h>
#include <mutex>
#include <vector>

#include "common/net/Network.h"
#include "common/net/RDMAControl.h"
#include "common/net/Transport.h"
#include "common/serde/MessagePacket.h"
#include "common/utils/Duration.h"
#include "common/utils/RobinHood.h"
#include "common/utils/Shards.h"
#include "common/utils/Status.h"

namespace hf3fs::net {

// Wake up the waiting caller.
class Waiter {
 public:
  // is singleton.
  static Waiter &instance();
  ~Waiter();

  struct Item {
    folly::coro::Baton baton;
    IOBufPtr buf;
    serde::MessagePacket<> packet;
    Status status = Status::OK;
    TransportPtr transport;
    RDMATransmissionLimiterPtr limiter;
    RelativeTime timestamp = RelativeTime::now();
  };

  size_t bind(Item &item) {
    thread_local size_t start = 0, end = 0;
    if (UNLIKELY(start >= end)) {
      start = uuid_idx_.fetch_add(kShardsSize);
      end = start + kShardsSize;
    }

    auto uuid = start++;
    shards_.withLock([&](Map &map) { map.emplace(uuid, item); }, uuid);
    return uuid;
  }

  bool setTransport(size_t uuid, TransportPtr transport) {
    return shards_.withLock(
        [&](Map &map) {
          auto it = map.find(uuid);
          if (it == map.end()) {
            return false;
          }
          it->second.transport = std::move(transport);
          return true;
        },
        uuid);
  }

  std::optional<Duration> setTransmissionLimiterPtr(size_t uuid,
                                                    const RDMATransmissionLimiterPtr &limiter,
                                                    RelativeTime startTime) {
    return shards_.withLock(
        [&](Map &map) -> std::optional<Duration> {
          auto it = map.find(uuid);
          if (it == map.end() || it->second.limiter) {
            return std::nullopt;
          }
          it->second.limiter = limiter;
          return startTime - std::exchange(it->second.timestamp, RelativeTime::now());
        },
        uuid);
  }

  void post(const serde::MessagePacket<> &packet, IOBufPtr buff) {
    auto item = find(packet.uuid);
    if (item) {
      item->buf = std::move(buff);
      item->packet = packet;
      if (item->limiter) {
        item->limiter->signal(RelativeTime::now() - item->timestamp);
      }
      item->baton.post();
    }
  }

  static void error(Item *item, Status status) {
    if (item) {
      item->status = status;
      if (item->limiter) {
        item->limiter->signal(RelativeTime::now() - item->timestamp);
      }
      item->baton.post();
    }
  }

  bool contains(size_t uuid) {
    return shards_.withLock([&](Map &map) { return map.contains(uuid); }, uuid);
  }

  void timeout(size_t uuid) { error(find(uuid), Status(RPCCode::kTimeout)); }
  void sendFail(size_t uuid) { error(find(uuid), Status(RPCCode::kSendFailed)); }

  void clearPendingRequestsOnTransportFailure(Transport *tr) {
    shards_.iterate([tr](Map &map) {
      for (auto it = map.begin(); it != map.end();) {
        Item *item = &it->second;
        if (item->transport.get() == tr) {
          error(item, Status(RPCCode::kTimeout));
          it = map.erase(it);
        } else {
          ++it;
        }
      }
    });
  }

  void schedule(uint64_t uuid, std::chrono::milliseconds timeout);

 private:
  Waiter();

  Item *find(size_t uuid) {
    return shards_.withLock(
        [&](Map &map) -> Item * {
          auto it = map.find(uuid);
          if (it == map.end()) {
            return nullptr;
          }
          auto item = &it->second;
          map.erase(it);
          return item;
        },
        uuid);
  }

  void run();

 private:
  std::atomic<size_t> uuid_idx_{0};

  constexpr static auto kShardsBit = 8u;
  constexpr static auto kShardsSize = (1u << kShardsBit);
  using Map = robin_hood::unordered_map<size_t, Item &>;
  Shards<Map, kShardsSize> shards_;

  // for timer.
  struct Task {
    uint64_t uuid;
    int64_t runTime;
    bool operator<(const Task &o) const { return runTime > o.runTime; }
  };

  class TaskShard {
   public:
    bool schedule(uint64_t uuid, int64_t runTime);
    void exchangeTasks(std::vector<Task> &out);

   private:
    std::mutex mutex_;
    int64_t nearestRunTime_ = std::numeric_limits<int64_t>::max();
    std::vector<Task> tasks_;
  };

  std::atomic<bool> stop_ = false;
  constexpr static auto kTaskShardNum = 13u;
  std::array<TaskShard, kTaskShardNum> taskShards_;

  std::mutex mutex_;
  std::condition_variable cond_;
  int64_t nearestRunTime_ = std::numeric_limits<int64_t>::max();
  std::vector<Task> reserved_;

  std::jthread thread_;
};

}  // namespace hf3fs::net

#pragma once

#include <atomic>
#include <boost/filesystem/path.hpp>
#include <cstdint>
#include <fmt/core.h>
#include <folly/Overload.h>
#include <folly/Random.h>
#include <folly/lang/Bits.h>
#include <folly/logging/xlog.h>
#include <folly/synchronization/DelayedInit.h>
#include <limits>
#include <linux/fs.h>
#include <optional>
#include <string>
#include <string_view>
#include <utility>
#include <variant>
#include <vector>

#include "common/app/ClientId.h"
#include "common/serde/Serde.h"
#include "common/serde/SerdeComparisons.h"
#include "common/utils/MagicEnum.hpp"
#include "common/utils/Path.h"
#include "common/utils/Result.h"
#include "common/utils/StrongType.h"
#include "common/utils/UtcTimeSerde.h"
#include "common/utils/Uuid.h"
#include "fbs/core/user/User.h"
#include "fbs/meta/Common.h"
#include "fbs/mgmtd/ChainRef.h"
#include "fbs/mgmtd/ChainTable.h"
#include "fbs/mgmtd/MgmtdTypes.h"
#include "fbs/mgmtd/RoutingInfo.h"

#define SETATTR_TIME_NOW hf3fs::UtcTime::fromMicroseconds(-1)

namespace hf3fs::meta {

using flat::ChainId;
using flat::ChainRef;
using flat::ChainTableId;
using flat::ChainTableVersion;

struct Acl {
  SERDE_STRUCT_FIELD(uid, Uid(0));
  SERDE_STRUCT_FIELD(gid, Gid(0));
  SERDE_STRUCT_FIELD(perm, Permission(0));
  SERDE_STRUCT_FIELD(iflags, IFlags(0));

 public:
  Acl() = default;
  Acl(Uid uid, Gid gid, Permission perm, IFlags iflags = IFlags(0))
      : uid(uid),
        gid(gid),
        perm(perm),
        iflags(iflags) {}

  static Acl root() { return Acl(Uid(0), Gid(0), Permission(0755), IFlags(FS_IMMUTABLE_FL)); }
  static Acl gcRoot() { return Acl(Uid(0), Gid(0), Permission(0700), IFlags(FS_IMMUTABLE_FL)); }

  Result<Void> checkPermission(const UserInfo &user, AccessType type) const;

  Result<Void> checkRecursiveRmPerm(const UserInfo &user, bool owner) const;

  bool operator==(const Acl &o) const { return serde::equals(*this, o); }
};

struct Inode;
struct Layout {
  enum class Type {
    Empty,
    ChainRange,
    ChainList,
  };
  struct Empty {
    static constexpr auto kType = Type::Empty;
    using is_serde_copyable = void;
    std::string serdeToReadable() const { return "empty"; }
    static Result<Empty> serdeFromReadable(std::string_view) { return Empty{}; }
    bool operator==(const Empty &) const { return true; }
  };

  struct ChainRange {
    enum Shuffle : uint8_t { NO_SHUFFLE = 0, STD_SHUFFLE_MT19937 };

    SERDE_STRUCT_FIELD(baseIndex, uint32_t(0));
    SERDE_STRUCT_FIELD(shuffle, Shuffle(0));
    SERDE_STRUCT_FIELD(seed, uint64_t(0));
    mutable folly::DelayedInit<std::vector<uint32_t>> chains;

   public:
    static constexpr auto kType = Type::ChainRange;
    ChainRange() = default;
    ChainRange(uint32_t baseIndex, Shuffle shuffle, uint64_t seed)
        : baseIndex(baseIndex),
          shuffle(shuffle),
          seed(seed),
          chains() {}
    ChainRange(const ChainRange &o)
        : ChainRange(o.baseIndex, o.shuffle, o.seed) {}
    ChainRange &operator=(const ChainRange &o) {
      if (&o != this) {
        new (this) ChainRange(o);
      }
      return *this;
    }

    std::span<const uint32_t> getChainIndexList(size_t stripe) const;
    bool operator==(const ChainRange &o) const { return serde::equals(*this, o); }
  };

  struct ChainList {
    SERDE_STRUCT_FIELD(chainIndexes, std::vector<uint32_t>());

   public:
    static constexpr auto kType = Type::ChainList;
    bool operator==(const ChainList &o) const { return serde::equals(*this, o); }
  };

  // make sure uint64_t is used in file length/offset calculate
  class ChunkSize {
   public:
    using is_serde_copyable = void;
    ChunkSize(uint32_t val = 0)
        : val_(val) {}

    uint32_t u32() const { return val_; }
    uint64_t u64() const { return val_; }
    operator uint64_t() const { return val_; }
    constexpr uint32_t serdeToReadable() const { return val_; }
    static Result<ChunkSize> serdeFromReadable(uint32_t val) { return ChunkSize(val); }

   private:
    uint32_t val_;
  };
  static_assert(sizeof(ChunkSize) == sizeof(uint32_t));

  SERDE_STRUCT_FIELD(tableId, ChainTableId());
  SERDE_STRUCT_FIELD(tableVersion, ChainTableVersion());
  SERDE_STRUCT_FIELD(chunkSize, ChunkSize(0));
  SERDE_STRUCT_FIELD(stripeSize, uint32_t(0));
  SERDE_STRUCT_FIELD(chains, (std::variant<Empty, ChainRange, ChainList>{}));

 public:
  static Layout newEmpty(ChainTableId table, uint32_t chunk, uint32_t stripe);
  static Layout newEmpty(ChainTableId table, ChainTableVersion tableVer, uint32_t chunk, uint32_t stripe);
  static Layout newChainList(ChainTableId table,
                             ChainTableVersion tableVer,
                             uint32_t chunk,
                             std::vector<uint32_t> chains);
  static Layout newChainList(uint32_t chunk, std::vector<flat::ChainId> chains);
  static Layout newChainRange(ChainTableId table,
                              ChainTableVersion tableVer,
                              uint32_t chunk,
                              uint32_t stripe,
                              uint32_t baseChainIndex);

  std::span<const uint32_t> getChainIndexList() const;
  ChainRef getChainOfChunk(const Inode &inode, size_t chunkIndex) const;
  bool empty() const { return std::holds_alternative<Empty>(chains); }
  Result<Void> valid(bool allowEmpty) const;
  Type type() const {
    XLOGF_IF(FATAL, chains.valueless_by_exception(), "Chains is valueless");
    return folly::variant_match(chains, [](const auto &v) { return std::decay_t<decltype(v)>::kType; });
  }
  bool operator==(const Layout &o) const { return serde::equals(*this, o); }
};

enum class InodeType : uint8_t {
  File = 0,
  Directory,
  Symlink,
};

class ChunkId {
  /**
   * Use big endian form to keep order.
   * format: [tenent id (0x00)] + [ unused 0x_00] + [ 64bits inode id ] + [ 16bits track id ] + [ 32bits chunk num ]
   */
  FOLLY_MAYBE_UNUSED std::array<uint8_t, 1> tenent_;
  FOLLY_MAYBE_UNUSED std::array<uint8_t, 1> reserved_;
  std::array<uint8_t, 8> inode_;
  FOLLY_MAYBE_UNUSED std::array<uint8_t, 2> track_;  // reserve for multitrack files.
  std::array<uint8_t, 4> chunk_;

 public:
  // ChunkId(InodeId inode = InodeId(-1), uint32_t chunk = 0)
  //     : ChunkId(inode, 0, chunk) {}
  // ChunkId()
  //     : ChunkId(InodeId(-1), 0, 0) {}
  ChunkId(InodeId inode, uint16_t track, uint32_t chunk)
      : tenent_({0}),
        reserved_({0}),
        inode_(folly::bit_cast<std::array<uint8_t, 8>>(folly::Endian::big64(inode.u64()))),
        track_(folly::bit_cast<std::array<uint8_t, 2>>(folly::Endian::big16(track))),
        chunk_(folly::bit_cast<std::array<uint8_t, 4>>(folly::Endian::big32(chunk))) {}

  std::string pack() const { return std::string((char *)this, sizeof(ChunkId)); }
  static ChunkId unpack(std::string_view data) {
    ChunkId id(InodeId(-1), 0, 0);
    if (data.size() == sizeof(ChunkId)) {
      memcpy(&id, data.data(), sizeof(ChunkId));
    }
    return id;
  }
  static Result<std::pair<ChunkId, ChunkId>> range(InodeId inodeId, uint32_t chunkBegin = 0) {
    if (inodeId.u64() == std::numeric_limits<uint64_t>::max()) {
      return makeError(MetaCode::kNotFile, "InodeId is uint64_max");
    }
    auto begin = ChunkId(inodeId, 0, chunkBegin);
    auto end = ChunkId(inodeId, 1, 0);
    return std::pair<ChunkId, ChunkId>{begin, end};
  }

  InodeId inode() const { return InodeId(folly::Endian::big64(folly::bit_cast<uint64_t>(inode_))); }
  uint16_t track() const { return folly::Endian::big16(folly::bit_cast<uint16_t>(track_)); }
  // use uint64_t to prevent error when calculate file offset or length
  uint64_t chunk() const { return folly::Endian::big32(folly::bit_cast<uint32_t>(chunk_)); }

  operator std::string() const { return pack(); }
  explicit operator bool() const { return *this != ChunkId(InodeId(-1), 0, 0); }
  bool operator==(const ChunkId &o) const { return memcmp(this, &o, sizeof(*this)) == 0; }
};
static_assert(sizeof(ChunkId) == 16);

struct VersionedLength {
  SERDE_STRUCT_FIELD(length, uint64_t(0));
  SERDE_STRUCT_FIELD(truncateVer, uint64_t(0));

 public:
  static std::optional<VersionedLength> mergeHint(std::optional<VersionedLength> h1,
                                                  std::optional<VersionedLength> h2) {
    if (!h1 || !h2) {
      return std::nullopt;
    } else {
      return h1->length >= h2->length ? h1 : h2;
    }
  }

  bool operator==(const VersionedLength &o) const { return serde::equals(*this, o); }
};

struct File {
  struct Flags : BitFlags<Flags, uint32_t> {
    using Base = BitFlags<Flags, uint32_t>;
    using Base::Base;
    static constexpr uint32_t kHasHole = 1;
  };
  SERDE_STRUCT_FIELD(length, uint64_t(0));
  SERDE_STRUCT_FIELD(truncateVer, uint64_t(0));
  SERDE_STRUCT_FIELD(layout, Layout());
  SERDE_STRUCT_FIELD(flags, Flags(0));
  SERDE_STRUCT_FIELD(dynStripe, uint32_t(0));  // dynStripe = 0 means dynamic stripe size is not enabled for this file

 public:
  File() = default;
  File(Layout layout, uint32_t dynStripe = 0)
      : layout(std::move(layout)),
        dynStripe(dynStripe) {}
  static constexpr auto kType = InodeType::File;
  Result<Void> valid() const { return layout.valid(false); }
  bool hasHole() const { return flags.contains(File::Flags::kHasHole); }
  Result<ChunkId> getChunkId(InodeId id, uint64_t offset) const;
  Result<ChainId> getChainId(const Inode &inode,
                             size_t offset,
                             const flat::RoutingInfo &routingInfo,
                             uint16_t track = 0) const;
  VersionedLength getVersionedLength() const { return VersionedLength{length, truncateVer}; }
  void setVersionedLength(VersionedLength v) {
    length = v.length;
    truncateVer = v.truncateVer;
  }
  bool operator==(const File &o) const { return serde::equals(*this, o); }
};

struct Directory {
  struct Lock {
    SERDE_STRUCT_FIELD(client, ClientId(Uuid::zero(), ""));

   public:
    bool operator==(const Lock &o) const { return serde::equals(*this, o); }
  };

  SERDE_STRUCT_FIELD(parent, InodeId());
  SERDE_STRUCT_FIELD(layout, Layout());
  SERDE_STRUCT_FIELD(name, std::string());
  SERDE_STRUCT_FIELD(chainAllocCounter, uint32_t(-1));
  SERDE_STRUCT_FIELD(lock, std::optional<Lock>());

 public:
  static constexpr auto kType = InodeType::Directory;
  Result<Void> valid() const { return layout.valid(true); }
  Result<Void> checkLock(const ClientId &client) const {
    if (lock && lock->client.uuid != client.uuid) {
      return makeError(MetaCode::kNoLock, fmt::format("locked by {}", lock->client));
    }
    return Void{};
  }
  bool operator==(const Directory &o) const { return serde::equals(*this, o); }
};

struct Symlink {
  SERDE_STRUCT_FIELD(target, Path());

 public:
  static constexpr auto kType = InodeType::Symlink;
  Result<Void> valid() const {
    if (target.empty()) return INVALID("target is empty");
    return VALID;
  }
  bool operator==(const Symlink &o) const { return serde::equals(*this, o); }
};

struct InodeData {
  SERDE_STRUCT_FIELD(type, (std::variant<File, Directory, Symlink>()));
  SERDE_STRUCT_FIELD(acl, Acl());
  SERDE_STRUCT_FIELD(nlink, uint16_t(1));
  // 2023/6/1 as default inode timestamps
  SERDE_STRUCT_FIELD(atime, UtcTime(std::chrono::microseconds(1685548800ull * 1000 * 1000)));
  SERDE_STRUCT_FIELD(ctime, UtcTime(std::chrono::microseconds(1685548800ull * 1000 * 1000)));
  SERDE_STRUCT_FIELD(mtime, UtcTime(std::chrono::microseconds(1685548800ull * 1000 * 1000)));

 public:
  InodeType getType() const {
    XLOGF_IF(FATAL, type.valueless_by_exception(), "InodeType is valueless");
    return folly::variant_match(type, [](const auto &v) { return std::decay_t<decltype(v)>::kType; });
  }

#define INODE_TYPE_FUNC(type_name)                                               \
  bool is##type_name() const { return std::holds_alternative<type_name>(type); } \
  type_name &as##type_name() {                                                   \
    XLOGF_IF(FATAL, type.valueless_by_exception(), "InodeType is valueless");    \
    XLOGF_IF(FATAL,                                                              \
             !std::holds_alternative<type_name>(type),                           \
             "Inode type {} != " #type_name,                                     \
             magic_enum::enum_name(getType()));                                  \
    return std::get<type_name>(type);                                            \
  }                                                                              \
  const type_name &as##type_name() const {                                       \
    XLOGF_IF(FATAL, type.valueless_by_exception(), "InodeType is valueless");    \
    XLOGF_IF(FATAL,                                                              \
             !std::holds_alternative<type_name>(type),                           \
             "Inode type {} != " #type_name,                                     \
             magic_enum::enum_name(getType()));                                  \
    return std::get<type_name>(type);                                            \
  }
  INODE_TYPE_FUNC(File)
  INODE_TYPE_FUNC(Directory)
  INODE_TYPE_FUNC(Symlink)
#undef INODE_TYPE_FUNC

  Result<Void> valid() const {
    XLOGF_IF(FATAL, type.valueless_by_exception(), "InodeType is valueless");
    return folly::variant_match(type, [](const auto &v) { return v.valid(); });
  }
  bool operator==(const InodeData &o) const { return serde::equals(*this, o); }
};

struct Inode : InodeData {
  SERDE_STRUCT_FIELD(id, InodeId());

 public:
  Inode() = default;
  explicit Inode(InodeId id)
      : Inode(id, {}) {}
  Inode(InodeId id, InodeData data)
      : InodeData(std::move(data)),
        id(id) {}
  InodeData &data() { return *this; }
  const InodeData &data() const { return *this; }
  Result<Void> valid() const { return data().valid(); }
  bool operator==(const Inode &o) const { return serde::equals(*this, o); }
};

struct GcInfo {
  SERDE_STRUCT_FIELD(user, flat::Uid(0));
  SERDE_STRUCT_FIELD(origPath, Path());

 public:
  bool operator==(const GcInfo &o) const { return serde::equals(*this, o); }
};

struct DirEntryData {
  SERDE_STRUCT_FIELD(id, InodeId());
  SERDE_STRUCT_FIELD(type, InodeType::File);
  SERDE_STRUCT_FIELD(dirAcl, std::optional<Acl>());
  SERDE_STRUCT_FIELD(uuid, Uuid::zero());
  SERDE_STRUCT_FIELD(gcInfo, std::optional<GcInfo>());

 public:
#define INODE_TYPE_FUNC(type_name) \
  bool is##type_name() const { return type == InodeType::type_name; }

  INODE_TYPE_FUNC(File)
  INODE_TYPE_FUNC(Directory)
  INODE_TYPE_FUNC(Symlink)
#undef INODE_TYPE_FUNC

  Result<Void> valid() const {
    if (!magic_enum::enum_contains(type)) return INVALID(fmt::format("invalid type {}", (int)type));
    if ((type == InodeType::Directory) != dirAcl.has_value())
      return INVALID(fmt::format("type {}, dirAcl {}", (type == InodeType::Directory), dirAcl.has_value()));
    return VALID;
  }
  bool operator==(const DirEntryData &o) const { return serde::equals(*this, o); }
};

struct DirEntry : DirEntryData {
  SERDE_STRUCT_FIELD(parent, InodeId());
  SERDE_STRUCT_FIELD(name, std::string());

 public:
  DirEntry() = default;
  DirEntry(InodeId parent, std::string name, DirEntryData data = {})
      : DirEntryData(std::move(data)),
        parent(parent),
        name(std::move(name)) {}
  DirEntryData &data() { return *this; }
  const DirEntryData &data() const { return *this; }
  Result<Void> valid() const { return data().valid(); }
  bool operator==(const DirEntry &o) const { return serde::equals(*this, o); }
};

}  // namespace hf3fs::meta

template <>
struct hf3fs::serde::SerdeMethod<hf3fs::meta::InodeType> {
  static constexpr std::string_view serdeToReadable(hf3fs::meta::InodeType t) { return magic_enum::enum_name(t); }
};

FMT_BEGIN_NAMESPACE

template <>
struct formatter<hf3fs::meta::InodeType> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(hf3fs::meta::InodeType type, FormatContext &ctx) const {
    return fmt::format_to(ctx.out(), "{}", magic_enum::enum_name(type));
  }
};

template <>
struct formatter<hf3fs::meta::ChunkId> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(hf3fs::meta::ChunkId chunk, FormatContext &ctx) const {
    return fmt::format_to(ctx.out(), "{}-{}-{}", chunk.inode(), chunk.track(), chunk.chunk());
  }
};

template <>
struct formatter<hf3fs::meta::Layout::ChunkSize> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(hf3fs::meta::Layout::ChunkSize chunk, FormatContext &ctx) const {
    return fmt::format_to(ctx.out(), "{}", (uint64_t)chunk);
  }
};

template <>
struct formatter<hf3fs::meta::VersionedLength> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(hf3fs::meta::VersionedLength v, FormatContext &ctx) const {
    return fmt::format_to(ctx.out(), "{}@{}", v.length, v.truncateVer);
  }
};

FMT_END_NAMESPACE

#pragma once

#include <folly/Executor.h>
#include <folly/IPAddressV4.h>
#include <memory>

#include "common/net/EventLoop.h"
#include "common/net/MessageHeader.h"
#include "common/net/Socket.h"
#include "common/net/WriteItem.h"
#include "common/net/ib/IBSocket.h"
#include "common/net/tcp/TcpSocket.h"
#include "common/utils/Address.h"
#include "common/utils/Duration.h"
#include "common/utils/Size.h"

namespace hf3fs::net {

class IOWorker;

// A wrapper of the underlying socket with read/write states.
class Transport : public EventLoop::EventHandler, public hf3fs::enable_shared_from_this<Transport> {
 protected:
  // not allowed to construct transport directly. use create instead.
  explicit Transport(std::unique_ptr<Socket> socket, IOWorker &io_worker, Address serverAddr);

 public:
  ~Transport() override;

  // create a transport from a socket with ownership.
  static std::shared_ptr<Transport> create(std::unique_ptr<Socket> socket, IOWorker &io_worker, Address::Type addrType);
  static std::shared_ptr<Transport> create(Address addr, IOWorker &io_worker);

  // connect.
  CoTryTask<void> connect(Address addr, Duration timeout);

  // return the address of the server.
  Address serverAddr() const { return serverAddr_; }

  // return IP of peer.
  folly::IPAddressV4 peerIP() const { return socket_->peerIP(); }

  // is TCP or RDMA connection.
  bool isTCP() const { return serverAddr_.isTCP(); }
  bool isRDMA() const { return serverAddr_.isRDMA(); }
  IBSocket *ibSocket() { return dynamic_cast<IBSocket *>(socket_.get()); }

  // return the address of the peer.
  std::string describe() const { return socket_->describe(); }

  // send a list asynchronously. return a list to be retried if send failed. [thread-safe]
  std::optional<WriteList> send(WriteList list);

  // invalidate this transport. [thread-safe]
  void invalidate(bool logError = true);

  // is invalidated or not.
  bool invalidated() const;

  // close IB socket.
  CoTask<void> closeIB();

  // check socket.
  Result<Void> check() { return socket_->check(); }

  // get credentials for UNIX domain socket.
  Result<Void> getPeerCredentials();

  // return credentials for UNIX domain socket.
  auto &credentials() const { return credentials_; }

  // is expired or not.
  bool expired(Duration expiredTime) const {
    return expiredTime != Duration::zero() && RelativeTime::now() >= lastUsedTime_.load() + expiredTime;
  }

 protected:
  enum class Action { OK, Retry, Suspend, Fail };
  template <uint32_t CheckAndRemove, uint32_t WantToRemove, MethodName Name>
  Action tryToSuspend();

  // there is and at most one read task is being executed.
  void doRead(bool error, bool logError = true);

  // there is and at most one write task is being executed.
  void doWrite(bool error, bool logError = true);
  Action writeAll();

  // try to clean up the state of the socket.
  void tryToCleanUp(bool isWrite);

  // file descriptor monitored by epoll. [EventHandler]
  int fd() const final { return socket_->fd(); }
  // handle the events notified by epoll. [EventHandler]
  void handleEvents(uint32_t epollEvents) final;

  // wake up when read/write/poll finished.
  void wakeUpAfterLastReadAndWriteFinished(uint32_t flags);

 private:
  friend class IOWorker;

  // the inner socket, TCP or RDMA.
  std::unique_ptr<Socket> socket_;
  // the object that actually handles the io work.
  IOWorker &ioWorker_;
  // connection executor
  std::weak_ptr<folly::Executor::KeepAlive<>> connExecutor_;
  // address of the remote server.
  Address serverAddr_;

  // protect the read and write for RDMA connection.
  std::mutex mutex_;

  // the inner state that supports atomic changes.
  alignas(folly::hardware_destructive_interference_size) std::atomic<uint32_t> flags_{0};

  // last used time.
  std::atomic<RelativeTime> lastUsedTime_ = RelativeTime::now();

  // for read.
  IOBufPtr readBuff_ = MessageWrapper::allocate(kMessageReadBufferSize);

  // for UNIX domain socket.
  std::optional<struct ucred> credentials_;

  // the linked list in writing.
  WriteListWithProgress inWritingList_;
  // the linked list to be written.
  MPSCWriteList mpscWriteList_;

  // post when last read and write are finished.
  folly::coro::Baton lastReadAndWriteFinished_;
};
using TransportPtr = std::shared_ptr<Transport>;

}  // namespace hf3fs::net

#pragma once

#include <cstddef>
#include <vector>

namespace hf3fs {

template <typename T>
class SimpleRingBuffer {
 public:
  explicit SimpleRingBuffer(size_t capacity)
      : cap_(capacity),
        head_(0),
        tail_(0) {
    buffer_.resize(capacity * sizeof(T));
  }

  template <typename U>
  requires(std::same_as<T, std::decay_t<U>>) bool push(U &&v) {
    if (full()) return false;
    new (addr(head_)) T(std::move(v));
    ++head_;
    return true;
  }

  template <typename... Args>
  bool emplace(Args &&...args) {
    if (full()) return false;
    new (addr(head_)) T(std::forward<Args>(args)...);
    ++head_;
    return true;
  }

  bool pop() {
    if (empty()) return false;
    auto *tail = addr(tail_);
    tail->~T();
    ++tail_;
    return true;
  }

  bool pop(T &v) {
    if (empty()) return false;
    auto *tail = addr(tail_);
    v = std::move(*tail);
    tail->~T();
    ++tail_;
    return true;
  }

  bool empty() const { return head_ == tail_; }

  bool full() const { return head_ == tail_ + cap_; }

  // TODO: should be std::random_access_iterator_tag
  template <bool Const>
  class iterator_base {
   public:
    using BufT = std::conditional_t<Const, const SimpleRingBuffer<T>, SimpleRingBuffer<T>>;
    using ValueT = std::conditional_t<Const, const T, T>;
    using difference_type = std::ptrdiff_t;
    using value_type = ValueT;
    using pointer = ValueT *;
    using reference = ValueT &;
    using iterator_category = std::input_iterator_tag;
    iterator_base(BufT *rb, size_t pos)
        : rb_(rb),
          pos_(pos) {}

    iterator_base &operator++() {
      ++pos_;
      pos_ %= rb_->cap_;
      return *this;
    }

    iterator_base &operator++(int) {
      auto old = *this;
      ++*this;
      return old;
    }

    ValueT *operator->() const { return rb_->addr(pos_); }
    ValueT &operator*() const { return *rb_->addr(pos_); }

    bool operator==(const iterator_base &other) const { return rb_ == other.rb_ && pos_ == other.pos_; }

    bool operator!=(const iterator_base &other) const { return !(*this == other); }

   private:
    BufT *rb_;
    size_t pos_;
  };

  using iterator = iterator_base<false>;
  using const_iterator = iterator_base<true>;

  iterator begin() { return iterator(this, tail_); }
  iterator end() { return iterator(this, head_); }

  const_iterator begin() const { return const_iterator(this, tail_); }
  const_iterator end() const { return const_iterator(this, head_); }

  const_iterator cbegin() const { return begin(); }
  const_iterator cend() const { return end(); }

 private:
  T *addr(size_t pos) {
    pos %= cap_;
    return std::launder(reinterpret_cast<T *>(buffer_.data() + pos * sizeof(T)));
  }

  const size_t cap_;
  size_t head_;
  size_t tail_;
  std::vector<std::byte> buffer_;
};

}  // namespace hf3fs

#pragma once

#include <folly/Likely.h>
#include <memory>
#include <mutex>
#include <new>
#include <type_traits>
#include <vector>

#include "common/utils/Size.h"

namespace hf3fs {

template <class T, size_t NumTLSCachedItem = 64, size_t NumGlobalCachedItem = 64 * 64>
class ObjectPool {
  using Storage = std::aligned_storage_t<sizeof(T), alignof(T)>;
  using Batch = std::vector<std::unique_ptr<Storage>>;

  struct Deleter {
    constexpr Deleter() = default;
    void operator()(T *item) {
      item->~T();
      tls().put(std::unique_ptr<Storage>(reinterpret_cast<Storage *>(item)));
    }
  };

 public:
  using Ptr = std::unique_ptr<T, Deleter>;
  static Ptr get() { return Ptr(new (tls().get().release()) T); }

  template <class... Args>
  static Ptr get(Args &&...args) {
    return Ptr(new (tls().get().release()) T(std::forward<Args>(args)...));
  }

 protected:
  class TLS {
   public:
    TLS(ObjectPool &parent)
        : parent_(parent) {}

    // get a object from thread local cache or global cache.
    std::unique_ptr<Storage> get() {
      // pop from second batch.
      if (!second_.empty()) {
        auto item = std::move(second_.back());
        second_.pop_back();
        return item;
      }

      // allocate a batch when local and global cache is empty.
      if (first_.empty() && !parent_.getBatch(first_)) {
        first_.resize(kThreadLocalMaxNum);
        for (auto &item : first_) {
          item.reset(new Storage);  // DO NOT REPLACE IT WITH std::make_unique<Storage>.
                                    // std::make_unique_for_overwrite is not available currently.
        }
      }

      // pop from the first batch.
      auto item = std::move(first_.back());
      first_.pop_back();
      return item;
    }

    // put a object into thread local cache or global cache.
    void put(std::unique_ptr<Storage> obj) {
      // push to the first batch.
      if (first_.size() < kThreadLocalMaxNum) {
        first_.push_back(std::move(obj));
        return;
      }

      // push to the second batch.
      second_.push_back(std::move(obj));
      if (UNLIKELY(second_.size() >= kThreadLocalMaxNum)) {
        // push to the global cache.
        parent_.putBatch(std::move(second_));
        second_.clear();
        second_.reserve(kThreadLocalMaxNum);
      }
    }

   private:
    ObjectPool &parent_;
    Batch first_;
    Batch second_;
  };

  static auto &tls() {
    static ObjectPool instance;
    thread_local TLS tls{instance};
    return tls;
  }

  bool getBatch(Batch &batch) {
    auto lock = std::unique_lock(mutex_);
    if (global_.empty()) {
      return false;
    }

    // pop a batch from global cache.
    batch = std::move(global_.back());
    global_.pop_back();
    return true;
  }

  void putBatch(Batch batch) {
    auto lock = std::unique_lock(mutex_);
    if (global_.size() < kGlobalMaxBatchNum) {
      global_.push_back(std::move(batch));
    }
  }

 private:
  static constexpr auto kThreadLocalMaxNum = std::max(1ul, NumTLSCachedItem);
  static constexpr auto kGlobalMaxBatchNum = std::max(1ul, NumGlobalCachedItem / NumTLSCachedItem);

  std::mutex mutex_;
  std::vector<Batch> global_;
};

}  // namespace hf3fs

#pragma once

#include <cstdint>
#include <semaphore.h>

#include "IovTable.h"
#include "UserConfig.h"
#include "client/storage/StorageClient.h"
#include "common/utils/AtomicSharedPtrTable.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Uuid.h"
#include "fbs/meta/Schema.h"
#include "lib/common/Shm.h"

namespace hf3fs::fuse {
struct RcInode;
struct IoArgs {
  uint8_t bufId[16];
  size_t bufOff;

  uint64_t fileIid;
  size_t fileOff;

  uint64_t ioLen;

  const void *userdata;
};

struct IoSqe {
  int32_t index;
  const void *userdata;
};

struct IoCqe {
  int32_t index;
  int32_t reserved;
  int64_t result;
  const void *userdata;
};

class IoRing;

struct IoRingJob {
  std::shared_ptr<IoRing> ior;
  int sqeProcTail;
  int toProc;
};

// we allow multiple io workers to process the same ioring, but different ranges
// so 1 ioring can be used to submit ios processed in parallel
// however, we don't allow multiple threads to prepare ios in the same ioring
// or batches may be mixed and things may get ugly
class IoRing : public std::enable_shared_from_this<IoRing> {
 public:
  static int ringMarkerSize() {
    auto n = std::atomic_ref<int32_t>::required_alignment;
    return (4 + n - 1) / n * n;
  }
  // allocate 1 more slot for queue emptiness/fullness checking
  static int ioRingEntries(size_t bufSize) {
    auto n = ringMarkerSize();
    // n * 4 for sqe/cqe head/tail markers
    return (int)std::min((size_t)std::numeric_limits<int>::max(),
                         (bufSize - 4096 - n * 4 - sizeof(sem_t)) / (sizeof(IoArgs) + sizeof(IoCqe) + sizeof(IoSqe))) -
           1;
  }
  static size_t bytesRequired(int entries) {
    auto n = ringMarkerSize();
    // n * 4 for sqe/cqe head/tail markers
    return n * 4 + sizeof(sem_t) + (sizeof(IoArgs) + sizeof(IoCqe) + sizeof(IoSqe)) * (entries + 1) + 4096;
  }

 public:
  using std::enable_shared_from_this<IoRing>::shared_from_this;

  // the shm arg is used to keep it from being destroyed when the iov link is removed
  IoRing(std::shared_ptr<lib::ShmBuf> shm,
         std::string_view nm,
         const meta::UserInfo &ui,
         bool read,
         uint8_t *buf,
         size_t size,
         int iod,
         int prio,
         Duration to,
         uint64_t flags,
         bool owner = true)
      : name(nm),
        entries(ioRingEntries(size) + 1),
        ioDepth(iod),
        priority(prio),
        timeout(to),
        sqeHead_((int32_t *)buf),
        sqeTail_((int32_t *)(buf + ringMarkerSize())),
        cqeHead_((int32_t *)(buf + ringMarkerSize() * 2)),
        cqeTail_((int32_t *)(buf + ringMarkerSize() * 3)),
        sqeHead(*sqeHead_),
        sqeTail(*sqeTail_),
        cqeHead(*cqeHead_),
        cqeTail(*cqeTail_),
        ringSection((IoArgs *)(buf + ringMarkerSize() * 4)),
        cqeSection((IoCqe *)(ringSection + entries)),
        sqeSection((IoSqe *)(cqeSection + entries)),
        slots(entries - 1),
        shm_(std::move(shm)),
        userInfo_(ui),
        forRead_(read),
        flags_(flags) {
    XLOGF_IF(FATAL,
             (uintptr_t)(sqeSection + entries + sizeof(sem_t)) > (uintptr_t)(buf + size),
             "sem has a bad address {}, after whole shm starts at {} with {} bytes",
             (void *)(sqeSection + entries + sizeof(sem_t)),
             (void *)buf,
             size);
    auto sem = (sem_t *)(sqeSection + entries);
    if (owner) {
      sem_init(sem, 1, 0);
    }
    cqeSem.reset(sem);
  }
  std::vector<IoRingJob> jobsToProc(int maxJobs);
  int cqeCount() const { return (cqeHead.load() + entries - cqeTail.load()) % entries; }
  CoTask<void> process(
      int spt,
      int toProc,
      storage::client::StorageClient &storageClient,
      const storage::client::IoOptions &storageIo,
      UserConfig &userConfig,
      std::function<void(std::vector<std::shared_ptr<RcInode>> &, const IoArgs *, const IoSqe *, int)> &&lookupFiles,
      std::function<void(std::vector<Result<lib::ShmBufForIO>> &, const IoArgs *, const IoSqe *, int)> &&lookupBufs);

 public:
  bool addSqe(int idx, const void *userdata) {
    auto h = sqeHead.load();
    if ((h + 1) % entries == sqeTail.load()) {
      return false;
    }

    auto &sqe = sqeSection[h];
    sqe.index = idx;
    sqe.userdata = userdata;

    sqeHead.store((h + 1) % entries);

    return true;
  }
  bool sqeTailAfter(int a, int b) {
    auto h = sqeHead.load();
    if (a == h) {  // caught up with head, must be the last
      return true;
    }
    auto ah = a > h, bh = b > h;
    if (ah == bh) {  // both after or before head, bigger is after
      return a > b;
    } else {  // the one before head is after
      return bh;
    }
  }

 public:
  std::string name;
  std::string mountName;
  int entries;
  int ioDepth;
  int priority;
  Duration timeout;

 private:
  int32_t *sqeHead_;
  int32_t *sqeTail_;
  int32_t *cqeHead_;
  int32_t *cqeTail_;
  std::optional<SteadyTime> lastCheck_;

 public:
  std::atomic_ref<int32_t> sqeHead;
  std::atomic_ref<int32_t> sqeTail;
  std::atomic_ref<int32_t> cqeHead;
  std::atomic_ref<int32_t> cqeTail;
  IoArgs *ringSection;
  IoCqe *cqeSection;
  IoSqe *sqeSection;
  std::unique_ptr<sem_t, std::function<void(sem_t *)>> cqeSem{nullptr, [](sem_t *p) { sem_destroy(p); }};

 public:
  AvailSlots slots;

 private:
  int sqeCount() const { return (sqeHead.load() + entries - sqeProcTail_) % entries; }
  [[nodiscard]] bool addCqe(int idx, ssize_t res, const void *userdata) {
    auto h = cqeHead.load();
    if ((h + 1) % entries == cqeTail.load()) {
      return false;
    }

    auto &cqe = cqeSection[h];
    cqe.index = idx;
    cqe.result = res;
    cqe.userdata = userdata;

    cqeHead.store((h + 1) % entries);
    return true;
  }

 private:  // for fuse
  std::shared_ptr<lib::ShmBuf> shm_;
  meta::UserInfo userInfo_;
  bool forRead_;
  uint64_t flags_;
  std::mutex cqeMtx_;  // when reporting cqes
  int sqeProcTail_{0};
  int processing_{0};
  std::deque<int> sqeProcTails_;  // tails claimed and processing
  std::set<int> sqeDoneTails_;    // tails done processing
};

struct IoRingTable {
  void init(int cap) {
    for (int prio = 0; prio <= 2; ++prio) {
      auto sp = "/" + semOpenPath(prio);
      sems.emplace_back(sem_open(sp.c_str(), O_CREAT, 0666, 0), [sp](sem_t *p) {
        sem_close(p);
        sem_unlink(sp.c_str());
      });
      chmod(semPath(prio).c_str(), 0666);
    }
    ioRings = std::make_unique<AtomicSharedPtrTable<IoRing>>(cap);
  }
  Result<int> addIoRing(const Path &mountName,
                        std::shared_ptr<lib::ShmBuf> shm,
                        std::string_view name,
                        const meta::UserInfo &ui,
                        bool forRead,
                        uint8_t *buf,
                        size_t size,
                        int ioDepth,
                        const hf3fs::lib::IorAttrs &attrs) {
    auto idxRes = ioRings->alloc();
    if (!idxRes) {
      return makeError(ClientAgentCode::kTooManyOpenFiles, "too many io rings");
    }

    auto idx = *idxRes;

    auto ior = std::make_shared<
        IoRing>(std::move(shm), name, ui, forRead, buf, size, ioDepth, attrs.priority, attrs.timeout, attrs.flags);
    ior->mountName = mountName.native();
    ioRings->table[idx].store(ior);

    return idx;
  }
  void rmIoRing(int idx) { ioRings->remove(idx); }
  std::vector<std::unique_ptr<sem_t, std::function<void(sem_t *)>>> sems;
  std::unique_ptr<AtomicSharedPtrTable<IoRing>> ioRings;

 private:
  static std::string semOpenPath(int prio) {
    static std::vector<Uuid> semIds{Uuid::random(), Uuid::random(), Uuid::random()};
    return fmt::format("hf3fs-submit-ios.{}", semIds[prio].toHexString());
  }

 public:
  static std::string semName(int prio) {
    return fmt::format("submit-ios{}", prio == 1 ? "" : prio == 0 ? ".ph" : ".pl");
  }
  static Path semPath(int prio) { return Path("/dev/shm") / ("sem." + semOpenPath(prio)); }
  static meta::Inode lookupSem(int prio) {
    static const std::vector<meta::Inode> inodes{
        {meta::InodeId{meta::InodeId::iovDir().u64() - 1},
         meta::InodeData{meta::Symlink{semPath(0)}, meta::Acl{meta::Uid{0}, meta::Gid{0}, meta::Permission{0666}}}},
        {meta::InodeId{meta::InodeId::iovDir().u64() - 2},
         meta::InodeData{meta::Symlink{semPath(1)}, meta::Acl{meta::Uid{0}, meta::Gid{0}, meta::Permission{0666}}}},
        {meta::InodeId{meta::InodeId::iovDir().u64() - 3},
         meta::InodeData{meta::Symlink{semPath(2)}, meta::Acl{meta::Uid{0}, meta::Gid{0}, meta::Permission{0666}}}}};

    return inodes[prio];
  }
};
}  // namespace hf3fs::fuse

///////////////////////// ankerl::unordered_dense::{map, set} /////////////////////////

// A fast & densely stored hashmap and hashset based on robin-hood backward shift deletion.
// Version 2.0.1
// https://github.com/martinus/unordered_dense
//
// Licensed under the MIT License <http://opensource.org/licenses/MIT>.
// SPDX-License-Identifier: MIT
// Copyright (c) 2022 Martin Leitner-Ankerl <martin.ankerl@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

#ifndef ANKERL_UNORDERED_DENSE_H
#define ANKERL_UNORDERED_DENSE_H

// see https://semver.org/spec/v2.0.0.html
#define ANKERL_UNORDERED_DENSE_VERSION_MAJOR 2  // NOLINT(cppcoreguidelines-macro-usage) incompatible API changes
#define ANKERL_UNORDERED_DENSE_VERSION_MINOR \
  0  // NOLINT(cppcoreguidelines-macro-usage) backwards compatible functionality
#define ANKERL_UNORDERED_DENSE_VERSION_PATCH 1  // NOLINT(cppcoreguidelines-macro-usage) backwards compatible bug fixes

// API versioning with inline namespace, see https://www.foonathan.net/2018/11/inline-namespaces/
#define ANKERL_UNORDERED_DENSE_VERSION_CONCAT1(major, minor, patch) v##major##_##minor##_##patch
#define ANKERL_UNORDERED_DENSE_VERSION_CONCAT(major, minor, patch) \
  ANKERL_UNORDERED_DENSE_VERSION_CONCAT1(major, minor, patch)
#define ANKERL_UNORDERED_DENSE_NAMESPACE                                      \
  ANKERL_UNORDERED_DENSE_VERSION_CONCAT(ANKERL_UNORDERED_DENSE_VERSION_MAJOR, \
                                        ANKERL_UNORDERED_DENSE_VERSION_MINOR, \
                                        ANKERL_UNORDERED_DENSE_VERSION_PATCH)

#if defined(_MSVC_LANG)
#define ANKERL_UNORDERED_DENSE_CPP_VERSION _MSVC_LANG
#else
#define ANKERL_UNORDERED_DENSE_CPP_VERSION __cplusplus
#endif

#if defined(__GNUC__)
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define ANKERL_UNORDERED_DENSE_PACK(decl) decl __attribute__((__packed__))
#elif defined(_MSC_VER)
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define ANKERL_UNORDERED_DENSE_PACK(decl) __pragma(pack(push, 1)) decl __pragma(pack(pop))
#endif

#if ANKERL_UNORDERED_DENSE_CPP_VERSION < 201703L
#error ankerl::unordered_dense requires C++17 or higher
#else
#include <array>             // for array
#include <cstdint>           // for uint64_t, uint32_t, uint8_t, UINT64_C
#include <cstring>           // for size_t, memcpy, memset
#include <functional>        // for equal_to, hash
#include <initializer_list>  // for initializer_list
#include <iterator>          // for pair, distance
#include <limits>            // for numeric_limits
#include <memory>            // for allocator, allocator_traits, shared_ptr
#include <stdexcept>         // for out_of_range
#include <string>            // for basic_string
#include <string_view>       // for basic_string_view, hash
#include <tuple>             // for forward_as_tuple
#include <type_traits>       // for enable_if_t, declval, conditional_t, ena...
#include <utility>           // for forward, exchange, pair, as_const, piece...
#include <vector>            // for vector

#define ANKERL_UNORDERED_DENSE_PMR 0  // NOLINT(cppcoreguidelines-macro-usage)
#if defined(__has_include)
#if __has_include(<memory_resource>)
#undef ANKERL_UNORDERED_DENSE_PMR
#define ANKERL_UNORDERED_DENSE_PMR 1  // NOLINT(cppcoreguidelines-macro-usage)
#include <memory_resource>            // for polymorphic_allocator
#endif
#endif

#if defined(_MSC_VER) && defined(_M_X64)
#include <intrin.h>
#pragma intrinsic(_umul128)
#endif

#if defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__)
#define ANKERL_UNORDERED_DENSE_LIKELY(x) __builtin_expect(x, 1)    // NOLINT(cppcoreguidelines-macro-usage)
#define ANKERL_UNORDERED_DENSE_UNLIKELY(x) __builtin_expect(x, 0)  // NOLINT(cppcoreguidelines-macro-usage)
#else
#define ANKERL_UNORDERED_DENSE_LIKELY(x) (x)    // NOLINT(cppcoreguidelines-macro-usage)
#define ANKERL_UNORDERED_DENSE_UNLIKELY(x) (x)  // NOLINT(cppcoreguidelines-macro-usage)
#endif

namespace ankerl::unordered_dense {
inline namespace ANKERL_UNORDERED_DENSE_NAMESPACE {

// hash ///////////////////////////////////////////////////////////////////////

// This is a stripped-down implementation of wyhash: https://github.com/wangyi-fudan/wyhash
// No big-endian support (because different values on different machines don't matter),
// hardcodes seed and the secret, reformattes the code, and clang-tidy fixes.
namespace detail::wyhash {

static inline void mum(uint64_t *a, uint64_t *b) {
#if defined(__SIZEOF_INT128__)
  __uint128_t r = *a;
  r *= *b;
  *a = static_cast<uint64_t>(r);
  *b = static_cast<uint64_t>(r >> 64U);
#elif defined(_MSC_VER) && defined(_M_X64)
  *a = _umul128(*a, *b, b);
#else
  uint64_t ha = *a >> 32U;
  uint64_t hb = *b >> 32U;
  uint64_t la = static_cast<uint32_t>(*a);
  uint64_t lb = static_cast<uint32_t>(*b);
  uint64_t hi{};
  uint64_t lo{};
  uint64_t rh = ha * hb;
  uint64_t rm0 = ha * lb;
  uint64_t rm1 = hb * la;
  uint64_t rl = la * lb;
  uint64_t t = rl + (rm0 << 32U);
  auto c = static_cast<uint64_t>(t < rl);
  lo = t + (rm1 << 32U);
  c += static_cast<uint64_t>(lo < t);
  hi = rh + (rm0 >> 32U) + (rm1 >> 32U) + c;
  *a = lo;
  *b = hi;
#endif
}

// multiply and xor mix function, aka MUM
[[nodiscard]] static inline auto mix(uint64_t a, uint64_t b) -> uint64_t {
  mum(&a, &b);
  return a ^ b;
}

// read functions. WARNING: we don't care about endianness, so results are different on big endian!
[[nodiscard]] static inline auto r8(const uint8_t *p) -> uint64_t {
  uint64_t v{};
  std::memcpy(&v, p, 8U);
  return v;
}

[[nodiscard]] static inline auto r4(const uint8_t *p) -> uint64_t {
  uint32_t v{};
  std::memcpy(&v, p, 4);
  return v;
}

// reads 1, 2, or 3 bytes
[[nodiscard]] static inline auto r3(const uint8_t *p, size_t k) -> uint64_t {
  return (static_cast<uint64_t>(p[0]) << 16U) | (static_cast<uint64_t>(p[k >> 1U]) << 8U) | p[k - 1];
}

[[maybe_unused]] [[nodiscard]] static inline auto hash(void const *key, size_t len) -> uint64_t {
  static constexpr auto secret = std::array{UINT64_C(0xa0761d6478bd642f),
                                            UINT64_C(0xe7037ed1a0b428db),
                                            UINT64_C(0x8ebc6af09c88c6e3),
                                            UINT64_C(0x589965cc75374cc3)};

  auto const *p = static_cast<uint8_t const *>(key);
  uint64_t seed = secret[0];
  uint64_t a{};
  uint64_t b{};
  if (ANKERL_UNORDERED_DENSE_LIKELY(len <= 16)) {
    if (ANKERL_UNORDERED_DENSE_LIKELY(len >= 4)) {
      a = (r4(p) << 32U) | r4(p + ((len >> 3U) << 2U));
      b = (r4(p + len - 4) << 32U) | r4(p + len - 4 - ((len >> 3U) << 2U));
    } else if (ANKERL_UNORDERED_DENSE_LIKELY(len > 0)) {
      a = r3(p, len);
      b = 0;
    } else {
      a = 0;
      b = 0;
    }
  } else {
    size_t i = len;
    if (ANKERL_UNORDERED_DENSE_UNLIKELY(i > 48)) {
      uint64_t see1 = seed;
      uint64_t see2 = seed;
      do {
        seed = mix(r8(p) ^ secret[1], r8(p + 8) ^ seed);
        see1 = mix(r8(p + 16) ^ secret[2], r8(p + 24) ^ see1);
        see2 = mix(r8(p + 32) ^ secret[3], r8(p + 40) ^ see2);
        p += 48;
        i -= 48;
      } while (ANKERL_UNORDERED_DENSE_LIKELY(i > 48));
      seed ^= see1 ^ see2;
    }
    while (ANKERL_UNORDERED_DENSE_UNLIKELY(i > 16)) {
      seed = mix(r8(p) ^ secret[1], r8(p + 8) ^ seed);
      i -= 16;
      p += 16;
    }
    a = r8(p + i - 16);
    b = r8(p + i - 8);
  }

  return mix(secret[1] ^ len, mix(a ^ secret[1], b ^ seed));
}

[[nodiscard]] static inline auto hash(uint64_t x) -> uint64_t {
  return detail::wyhash::mix(x, UINT64_C(0x9E3779B97F4A7C15));
}

}  // namespace detail::wyhash

template <typename T, typename Enable = void>
struct hash {
  auto operator()(T const &obj) const
      noexcept(noexcept(std::declval<std::hash<T>>().operator()(std::declval<T const &>()))) -> uint64_t {
    return std::hash<T>{}(obj);
  }
};

template <typename CharT>
struct hash<std::basic_string<CharT>> {
  using is_avalanching = void;
  auto operator()(std::basic_string<CharT> const &str) const noexcept -> uint64_t {
    return detail::wyhash::hash(str.data(), sizeof(CharT) * str.size());
  }
};

template <typename CharT>
struct hash<std::basic_string_view<CharT>> {
  using is_avalanching = void;
  auto operator()(std::basic_string_view<CharT> const &sv) const noexcept -> uint64_t {
    return detail::wyhash::hash(sv.data(), sizeof(CharT) * sv.size());
  }
};

template <class T>
struct hash<T *> {
  using is_avalanching = void;
  auto operator()(T *ptr) const noexcept -> uint64_t {
    // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
    return detail::wyhash::hash(reinterpret_cast<uintptr_t>(ptr));
  }
};

template <class T>
struct hash<std::unique_ptr<T>> {
  using is_avalanching = void;
  auto operator()(std::unique_ptr<T> const &ptr) const noexcept -> uint64_t {
    // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
    return detail::wyhash::hash(reinterpret_cast<uintptr_t>(ptr.get()));
  }
};

template <class T>
struct hash<std::shared_ptr<T>> {
  using is_avalanching = void;
  auto operator()(std::shared_ptr<T> const &ptr) const noexcept -> uint64_t {
    // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast)
    return detail::wyhash::hash(reinterpret_cast<uintptr_t>(ptr.get()));
  }
};

template <typename Enum>
struct hash<Enum, typename std::enable_if<std::is_enum<Enum>::value>::type> {
  using is_avalanching = void;
  auto operator()(Enum e) const noexcept -> uint64_t {
    using underlying = typename std::underlying_type_t<Enum>;
    return detail::wyhash::hash(static_cast<underlying>(e));
  }
};

// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define ANKERL_UNORDERED_DENSE_HASH_STATICCAST(T)              \
  template <>                                                  \
  struct hash<T> {                                             \
    using is_avalanching = void;                               \
    auto operator()(T const &obj) const noexcept -> uint64_t { \
      return detail::wyhash::hash(static_cast<uint64_t>(obj)); \
    }                                                          \
  }

#if defined(__GNUC__) && !defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wuseless-cast"
#endif
// see https://en.cppreference.com/w/cpp/utility/hash
ANKERL_UNORDERED_DENSE_HASH_STATICCAST(bool);
ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char);
ANKERL_UNORDERED_DENSE_HASH_STATICCAST(signed char);
ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned char);
#if ANKERL_UNORDERED_DENSE_CPP_VERSION >= 202002L
ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char8_t);
#endif
ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char16_t);
ANKERL_UNORDERED_DENSE_HASH_STATICCAST(char32_t);
ANKERL_UNORDERED_DENSE_HASH_STATICCAST(wchar_t);
ANKERL_UNORDERED_DENSE_HASH_STATICCAST(short);
ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned short);
ANKERL_UNORDERED_DENSE_HASH_STATICCAST(int);
ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned int);
ANKERL_UNORDERED_DENSE_HASH_STATICCAST(long);
ANKERL_UNORDERED_DENSE_HASH_STATICCAST(long long);
ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned long);
ANKERL_UNORDERED_DENSE_HASH_STATICCAST(unsigned long long);

#if defined(__GNUC__) && !defined(__clang__)
#pragma GCC diagnostic pop
#endif

// bucket_type //////////////////////////////////////////////////////////

namespace bucket_type {

struct standard {
  static constexpr uint32_t dist_inc = 1U << 8U;              // skip 1 byte fingerprint
  static constexpr uint32_t fingerprint_mask = dist_inc - 1;  // mask for 1 byte of fingerprint

  uint32_t m_dist_and_fingerprint;  // upper 3 byte: distance to original bucket. lower byte: fingerprint from hash
  uint32_t m_value_idx;             // index into the m_values vector.
};

ANKERL_UNORDERED_DENSE_PACK(struct big {
  static constexpr uint32_t dist_inc = 1U << 8U;              // skip 1 byte fingerprint
  static constexpr uint32_t fingerprint_mask = dist_inc - 1;  // mask for 1 byte of fingerprint

  uint32_t m_dist_and_fingerprint;  // upper 3 byte: distance to original bucket. lower byte: fingerprint from hash
  size_t m_value_idx;               // index into the m_values vector.
});

}  // namespace bucket_type

namespace detail {

struct nonesuch {};

template <class Default, class AlwaysVoid, template <class...> class Op, class... Args>
struct detector {
  using value_t = std::false_type;
  using type = Default;
};

template <class Default, template <class...> class Op, class... Args>
struct detector<Default, std::void_t<Op<Args...>>, Op, Args...> {
  using value_t = std::true_type;
  using type = Op<Args...>;
};

template <template <class...> class Op, class... Args>
using is_detected = typename detail::detector<detail::nonesuch, void, Op, Args...>::value_t;

template <template <class...> class Op, class... Args>
constexpr bool is_detected_v = is_detected<Op, Args...>::value;

template <typename T>
using detect_avalanching = typename T::is_avalanching;

template <typename T>
using detect_is_transparent = typename T::is_transparent;

template <typename T>
using detect_iterator = typename T::iterator;

template <typename T>
using detect_reserve = decltype(std::declval<T &>().reserve(size_t{}));

// enable_if helpers

template <typename Mapped>
constexpr bool is_map_v = !std::is_void_v<Mapped>;

template <typename Hash, typename KeyEqual>
constexpr bool is_transparent_v =
    is_detected_v<detect_is_transparent, Hash> &&is_detected_v<detect_is_transparent, KeyEqual>;

template <typename From, typename To1, typename To2>
constexpr bool is_neither_convertible_v = !std::is_convertible_v<From, To1> && !std::is_convertible_v<From, To2>;

template <typename T>
constexpr bool has_reserve = is_detected_v<detect_reserve, T>;

// This is it, the table. Doubles as map and set, and uses `void` for T when its used as a set.
template <class Key,
          class T,  // when void, treat it as a set.
          class Hash,
          class KeyEqual,
          class AllocatorOrContainer,
          class Bucket>
class table {
 public:
  using value_container_type =
      std::conditional_t<is_detected_v<detect_iterator, AllocatorOrContainer>,
                         AllocatorOrContainer,
                         typename std::vector<typename std::conditional_t<std::is_void_v<T>, Key, std::pair<Key, T>>,
                                              AllocatorOrContainer>>;

 private:
  using bucket_alloc =
      typename std::allocator_traits<typename value_container_type::allocator_type>::template rebind_alloc<Bucket>;
  using bucket_alloc_traits = std::allocator_traits<bucket_alloc>;

  static constexpr uint8_t initial_shifts = 64 - 3;  // 2^(64-m_shift) number of buckets
  static constexpr float default_max_load_factor = 0.8F;

 public:
  using key_type = Key;
  using mapped_type = T;
  using value_type = typename value_container_type::value_type;
  using size_type = typename value_container_type::size_type;
  using difference_type = typename value_container_type::difference_type;
  using hasher = Hash;
  using key_equal = KeyEqual;
  using allocator_type = typename value_container_type::allocator_type;
  using reference = typename value_container_type::reference;
  using const_reference = typename value_container_type::const_reference;
  using pointer = typename value_container_type::pointer;
  using const_pointer = typename value_container_type::const_pointer;
  using iterator = typename value_container_type::iterator;
  using const_iterator = typename value_container_type::const_iterator;
  using bucket_type = Bucket;

 private:
  using value_idx_type = decltype(Bucket::m_value_idx);
  using dist_and_fingerprint_type = decltype(Bucket::m_dist_and_fingerprint);

  static_assert(std::is_trivially_destructible_v<Bucket>, "assert there's no need to call destructor / std::destroy");
  static_assert(std::is_trivially_copyable_v<Bucket>, "assert we can just memset / memcpy");

  value_container_type m_values{};  // Contains all the key-value pairs in one densely stored container. No holes.
  typename std::allocator_traits<bucket_alloc>::pointer m_buckets{};
  size_t m_num_buckets = 0;
  size_t m_max_bucket_capacity = 0;
  float m_max_load_factor = default_max_load_factor;
  Hash m_hash{};
  KeyEqual m_equal{};
  uint8_t m_shifts = initial_shifts;

  [[nodiscard]] auto next(value_idx_type bucket_idx) const -> value_idx_type {
    return ANKERL_UNORDERED_DENSE_UNLIKELY(bucket_idx + 1U == m_num_buckets)
               ? 0
               : static_cast<value_idx_type>(bucket_idx + 1U);
  }

  // Helper to access bucket through pointer types
  [[nodiscard]] static constexpr auto at(typename std::allocator_traits<bucket_alloc>::pointer bucket_ptr,
                                         size_t offset) -> Bucket & {
    return *(bucket_ptr + static_cast<typename std::allocator_traits<bucket_alloc>::difference_type>(offset));
  }

  // use the dist_inc and dist_dec functions so that uint16_t types work without warning
  [[nodiscard]] static constexpr auto dist_inc(dist_and_fingerprint_type x) -> dist_and_fingerprint_type {
    return static_cast<dist_and_fingerprint_type>(x + Bucket::dist_inc);
  }

  [[nodiscard]] static constexpr auto dist_dec(dist_and_fingerprint_type x) -> dist_and_fingerprint_type {
    return static_cast<dist_and_fingerprint_type>(x - Bucket::dist_inc);
  }

  // The goal of mixed_hash is to always produce a high quality 64bit hash.
  template <typename K>
  [[nodiscard]] constexpr auto mixed_hash(K const &key) const -> uint64_t {
    if constexpr (is_detected_v<detect_avalanching, Hash>) {
      // we know that the hash is good because is_avalanching.
      if constexpr (sizeof(decltype(m_hash(key))) < sizeof(uint64_t)) {
        // 32bit hash and is_avalanching => multiply with a constant to avalanche bits upwards
        return m_hash(key) * UINT64_C(0x9ddfea08eb382d69);
      } else {
        // 64bit and is_avalanching => only use the hash itself.
        return m_hash(key);
      }
    } else {
      // not is_avalanching => apply wyhash
      return wyhash::hash(m_hash(key));
    }
  }

  [[nodiscard]] constexpr auto dist_and_fingerprint_from_hash(uint64_t hash) const -> dist_and_fingerprint_type {
    return Bucket::dist_inc | (static_cast<dist_and_fingerprint_type>(hash) & Bucket::fingerprint_mask);
  }

  [[nodiscard]] constexpr auto bucket_idx_from_hash(uint64_t hash) const -> value_idx_type {
    return static_cast<value_idx_type>(hash >> m_shifts);
  }

  [[nodiscard]] static constexpr auto get_key(value_type const &vt) -> key_type const & {
    if constexpr (std::is_void_v<T>) {
      return vt;
    } else {
      return vt.first;
    }
  }

  template <typename K>
  [[nodiscard]] auto next_while_less(K const &key) const -> Bucket {
    auto hash = mixed_hash(key);
    auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
    auto bucket_idx = bucket_idx_from_hash(hash);

    while (dist_and_fingerprint < at(m_buckets, bucket_idx).m_dist_and_fingerprint) {
      dist_and_fingerprint = dist_inc(dist_and_fingerprint);
      bucket_idx = next(bucket_idx);
    }
    return {dist_and_fingerprint, bucket_idx};
  }

  void place_and_shift_up(Bucket bucket, value_idx_type place) {
    while (0 != at(m_buckets, place).m_dist_and_fingerprint) {
      bucket = std::exchange(at(m_buckets, place), bucket);
      bucket.m_dist_and_fingerprint = dist_inc(bucket.m_dist_and_fingerprint);
      place = next(place);
    }
    at(m_buckets, place) = bucket;
  }

  [[nodiscard]] static constexpr auto calc_num_buckets(uint8_t shifts) -> size_t {
    return std::min(max_bucket_count(), size_t{1} << (64U - shifts));
  }

  [[nodiscard]] constexpr auto calc_shifts_for_size(size_t s) const -> uint8_t {
    auto shifts = initial_shifts;
    while (shifts > 0 && static_cast<size_t>(static_cast<float>(calc_num_buckets(shifts)) * max_load_factor()) < s) {
      --shifts;
    }
    return shifts;
  }

  // assumes m_values has data, m_buckets=m_buckets_end=nullptr, m_shifts is INITIAL_SHIFTS
  void copy_buckets(table const &other) {
    if (!empty()) {
      m_shifts = other.m_shifts;
      allocate_buckets_from_shift();
      std::memcpy(m_buckets, other.m_buckets, sizeof(Bucket) * bucket_count());
    }
  }

  /**
   * True when no element can be added any more without increasing the size
   */
  [[nodiscard]] auto is_full() const -> bool { return size() >= m_max_bucket_capacity; }

  void deallocate_buckets() {
    auto ba = bucket_alloc(m_values.get_allocator());
    if (nullptr != m_buckets) {
      bucket_alloc_traits::deallocate(ba, m_buckets, bucket_count());
    }
    m_buckets = nullptr;
    m_num_buckets = 0;
    m_max_bucket_capacity = 0;
  }

  void allocate_buckets_from_shift() {
    auto ba = bucket_alloc(m_values.get_allocator());
    m_num_buckets = calc_num_buckets(m_shifts);
    m_buckets = bucket_alloc_traits::allocate(ba, m_num_buckets);
    if (m_num_buckets == max_bucket_count()) {
      // reached the maximum, make sure we can use each bucket
      m_max_bucket_capacity = max_bucket_count();
    } else {
      m_max_bucket_capacity = static_cast<value_idx_type>(static_cast<float>(m_num_buckets) * max_load_factor());
    }
  }

  void clear_buckets() {
    if (m_buckets != nullptr) {
      std::memset(&*m_buckets, 0, sizeof(Bucket) * bucket_count());
    }
  }

  void clear_and_fill_buckets_from_values() {
    clear_buckets();
    for (value_idx_type value_idx = 0, end_idx = static_cast<value_idx_type>(m_values.size()); value_idx < end_idx;
         ++value_idx) {
      auto const &key = get_key(m_values[value_idx]);
      auto [dist_and_fingerprint, bucket] = next_while_less(key);

      // we know for certain that key has not yet been inserted, so no need to check it.
      place_and_shift_up({dist_and_fingerprint, value_idx}, bucket);
    }
  }

  void increase_size() {
    if (ANKERL_UNORDERED_DENSE_UNLIKELY(m_max_bucket_capacity == max_bucket_count())) {
      throw std::overflow_error("ankerl::unordered_dense: reached max bucket size, cannot increase size");
    }
    --m_shifts;
    deallocate_buckets();
    allocate_buckets_from_shift();
    clear_and_fill_buckets_from_values();
  }

  void do_erase(value_idx_type bucket_idx) {
    auto const value_idx_to_remove = at(m_buckets, bucket_idx).m_value_idx;

    // shift down until either empty or an element with correct spot is found
    auto next_bucket_idx = next(bucket_idx);
    while (at(m_buckets, next_bucket_idx).m_dist_and_fingerprint >= Bucket::dist_inc * 2) {
      at(m_buckets, bucket_idx) = {dist_dec(at(m_buckets, next_bucket_idx).m_dist_and_fingerprint),
                                   at(m_buckets, next_bucket_idx).m_value_idx};
      bucket_idx = std::exchange(next_bucket_idx, next(next_bucket_idx));
    }
    at(m_buckets, bucket_idx) = {};

    // update m_values
    if (value_idx_to_remove != m_values.size() - 1) {
      // no luck, we'll have to replace the value with the last one and update the index accordingly
      auto &val = m_values[value_idx_to_remove];
      val = std::move(m_values.back());

      // update the values_idx of the moved entry. No need to play the info game, just look until we find the values_idx
      auto mh = mixed_hash(get_key(val));
      bucket_idx = bucket_idx_from_hash(mh);

      auto const values_idx_back = static_cast<value_idx_type>(m_values.size() - 1);
      while (values_idx_back != at(m_buckets, bucket_idx).m_value_idx) {
        bucket_idx = next(bucket_idx);
      }
      at(m_buckets, bucket_idx).m_value_idx = value_idx_to_remove;
    }
    m_values.pop_back();
  }

  template <typename K>
  auto do_erase_key(K &&key) -> size_t {
    if (empty()) {
      return 0;
    }

    auto [dist_and_fingerprint, bucket_idx] = next_while_less(key);

    while (dist_and_fingerprint == at(m_buckets, bucket_idx).m_dist_and_fingerprint &&
           !m_equal(key, get_key(m_values[at(m_buckets, bucket_idx).m_value_idx]))) {
      dist_and_fingerprint = dist_inc(dist_and_fingerprint);
      bucket_idx = next(bucket_idx);
    }

    if (dist_and_fingerprint != at(m_buckets, bucket_idx).m_dist_and_fingerprint) {
      return 0;
    }
    do_erase(bucket_idx);
    return 1;
  }

  template <class K, class M>
  auto do_insert_or_assign(K &&key, M &&mapped) -> std::pair<iterator, bool> {
    auto it_isinserted = try_emplace(std::forward<K>(key), std::forward<M>(mapped));
    if (!it_isinserted.second) {
      it_isinserted.first->second = std::forward<M>(mapped);
    }
    return it_isinserted;
  }

  template <typename K, typename... Args>
  auto do_place_element(dist_and_fingerprint_type dist_and_fingerprint,
                        value_idx_type bucket_idx,
                        K &&key,
                        Args &&...args) -> std::pair<iterator, bool> {
    // emplace the new value. If that throws an exception, no harm done; index is still in a valid state
    m_values.emplace_back(std::piecewise_construct,
                          std::forward_as_tuple(std::forward<K>(key)),
                          std::forward_as_tuple(std::forward<Args>(args)...));

    // place element and shift up until we find an empty spot
    auto value_idx = static_cast<value_idx_type>(m_values.size() - 1);
    place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx);
    return {begin() + static_cast<difference_type>(value_idx), true};
  }

  template <typename K, typename... Args>
  auto do_try_emplace(K &&key, Args &&...args) -> std::pair<iterator, bool> {
    if (ANKERL_UNORDERED_DENSE_UNLIKELY(is_full())) {
      increase_size();
    }

    auto hash = mixed_hash(key);
    auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
    auto bucket_idx = bucket_idx_from_hash(hash);

    while (true) {
      auto *bucket = &at(m_buckets, bucket_idx);
      if (dist_and_fingerprint == bucket->m_dist_and_fingerprint) {
        if (m_equal(key, m_values[bucket->m_value_idx].first)) {
          return {begin() + static_cast<difference_type>(bucket->m_value_idx), false};
        }
      } else if (dist_and_fingerprint > bucket->m_dist_and_fingerprint) {
        return do_place_element(dist_and_fingerprint, bucket_idx, std::forward<K>(key), std::forward<Args>(args)...);
      }
      dist_and_fingerprint = dist_inc(dist_and_fingerprint);
      bucket_idx = next(bucket_idx);
    }
  }

  template <typename K>
  auto do_find(K const &key) -> iterator {
    if (ANKERL_UNORDERED_DENSE_UNLIKELY(empty())) {
      return end();
    }

    auto mh = mixed_hash(key);
    auto dist_and_fingerprint = dist_and_fingerprint_from_hash(mh);
    auto bucket_idx = bucket_idx_from_hash(mh);
    auto *bucket = &at(m_buckets, bucket_idx);

    // unrolled loop. *Always* check a few directly, then enter the loop. This is faster.
    if (dist_and_fingerprint == bucket->m_dist_and_fingerprint &&
        m_equal(key, get_key(m_values[bucket->m_value_idx]))) {
      return begin() + static_cast<difference_type>(bucket->m_value_idx);
    }
    dist_and_fingerprint = dist_inc(dist_and_fingerprint);
    bucket_idx = next(bucket_idx);
    bucket = &at(m_buckets, bucket_idx);

    if (dist_and_fingerprint == bucket->m_dist_and_fingerprint &&
        m_equal(key, get_key(m_values[bucket->m_value_idx]))) {
      return begin() + static_cast<difference_type>(bucket->m_value_idx);
    }
    dist_and_fingerprint = dist_inc(dist_and_fingerprint);
    bucket_idx = next(bucket_idx);
    bucket = &at(m_buckets, bucket_idx);

    while (true) {
      if (dist_and_fingerprint == bucket->m_dist_and_fingerprint) {
        if (m_equal(key, get_key(m_values[bucket->m_value_idx]))) {
          return begin() + static_cast<difference_type>(bucket->m_value_idx);
        }
      } else if (dist_and_fingerprint > bucket->m_dist_and_fingerprint) {
        return end();
      }
      dist_and_fingerprint = dist_inc(dist_and_fingerprint);
      bucket_idx = next(bucket_idx);
      bucket = &at(m_buckets, bucket_idx);
    }
  }

  template <typename K>
  auto do_find(K const &key) const -> const_iterator {
    return const_cast<table *>(this)->do_find(key);  // NOLINT(cppcoreguidelines-pro-type-const-cast)
  }

  template <typename K, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
  auto do_at(K const &key) -> Q & {
    if (auto it = find(key); end() != it) {
      return it->second;
    }
    throw std::out_of_range("ankerl::unordered_dense::map::at(): key not found");
  }

  template <typename K, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
  auto do_at(K const &key) const -> Q const & {
    return const_cast<table *>(this)->at(key);  // NOLINT(cppcoreguidelines-pro-type-const-cast)
  }

 public:
  table()
      : table(0) {}

  explicit table(size_t bucket_count,
                 Hash const &hash = Hash(),
                 KeyEqual const &equal = KeyEqual(),
                 allocator_type const &alloc_or_container = allocator_type())
      : m_values(alloc_or_container),
        m_hash(hash),
        m_equal(equal) {
    if (0 != bucket_count) {
      reserve(bucket_count);
    }
  }

  table(size_t bucket_count, allocator_type const &alloc)
      : table(bucket_count, Hash(), KeyEqual(), alloc) {}

  table(size_t bucket_count, Hash const &hash, allocator_type const &alloc)
      : table(bucket_count, hash, KeyEqual(), alloc) {}

  explicit table(allocator_type const &alloc)
      : table(0, Hash(), KeyEqual(), alloc) {}

  template <class InputIt>
  table(InputIt first,
        InputIt last,
        size_type bucket_count = 0,
        Hash const &hash = Hash(),
        KeyEqual const &equal = KeyEqual(),
        allocator_type const &alloc = allocator_type())
      : table(bucket_count, hash, equal, alloc) {
    insert(first, last);
  }

  template <class InputIt>
  table(InputIt first, InputIt last, size_type bucket_count, allocator_type const &alloc)
      : table(first, last, bucket_count, Hash(), KeyEqual(), alloc) {}

  template <class InputIt>
  table(InputIt first, InputIt last, size_type bucket_count, Hash const &hash, allocator_type const &alloc)
      : table(first, last, bucket_count, hash, KeyEqual(), alloc) {}

  table(table const &other)
      : table(other, other.m_values.get_allocator()) {}

  table(table const &other, allocator_type const &alloc)
      : m_values(other.m_values, alloc),
        m_max_load_factor(other.m_max_load_factor),
        m_hash(other.m_hash),
        m_equal(other.m_equal) {
    copy_buckets(other);
  }

  table(table &&other) noexcept
      : table(std::move(other), other.m_values.get_allocator()) {}

  table(table &&other, allocator_type const &alloc) noexcept
      : m_values(std::move(other.m_values), alloc),
        m_buckets(std::exchange(other.m_buckets, nullptr)),
        m_num_buckets(std::exchange(other.m_num_buckets, 0)),
        m_max_bucket_capacity(std::exchange(other.m_max_bucket_capacity, 0)),
        m_max_load_factor(std::exchange(other.m_max_load_factor, default_max_load_factor)),
        m_hash(std::exchange(other.m_hash, {})),
        m_equal(std::exchange(other.m_equal, {})),
        m_shifts(std::exchange(other.m_shifts, initial_shifts)) {
    other.m_values.clear();
  }

  table(std::initializer_list<value_type> ilist,
        size_t bucket_count = 0,
        Hash const &hash = Hash(),
        KeyEqual const &equal = KeyEqual(),
        allocator_type const &alloc = allocator_type())
      : table(bucket_count, hash, equal, alloc) {
    insert(ilist);
  }

  table(std::initializer_list<value_type> ilist, size_type bucket_count, allocator_type const &alloc)
      : table(ilist, bucket_count, Hash(), KeyEqual(), alloc) {}

  table(std::initializer_list<value_type> init, size_type bucket_count, Hash const &hash, allocator_type const &alloc)
      : table(init, bucket_count, hash, KeyEqual(), alloc) {}

  ~table() {
    auto ba = bucket_alloc(m_values.get_allocator());
    bucket_alloc_traits::deallocate(ba, m_buckets, bucket_count());
  }

  auto operator=(table const &other) -> table & {
    if (&other != this) {
      deallocate_buckets();  // deallocate before m_values is set (might have another allocator)
      m_values = other.m_values;
      m_max_load_factor = other.m_max_load_factor;
      m_hash = other.m_hash;
      m_equal = other.m_equal;
      m_shifts = initial_shifts;
      copy_buckets(other);
    }
    return *this;
  }

  auto operator=(table &&other) noexcept(
      noexcept(std::is_nothrow_move_assignable_v<value_container_type> &&std::is_nothrow_move_assignable_v<Hash>
                   &&std::is_nothrow_move_assignable_v<KeyEqual>)) -> table & {
    if (&other != this) {
      deallocate_buckets();  // deallocate before m_values is set (might have another allocator)
      m_values = std::move(other.m_values);
      m_buckets = std::exchange(other.m_buckets, nullptr);
      m_num_buckets = std::exchange(other.m_num_buckets, 0);
      m_max_bucket_capacity = std::exchange(other.m_max_bucket_capacity, 0);
      m_max_load_factor = std::exchange(other.m_max_load_factor, default_max_load_factor);
      m_hash = std::exchange(other.m_hash, {});
      m_equal = std::exchange(other.m_equal, {});
      m_shifts = std::exchange(other.m_shifts, initial_shifts);
      other.m_values.clear();
    }
    return *this;
  }

  auto operator=(std::initializer_list<value_type> ilist) -> table & {
    clear();
    insert(ilist);
    return *this;
  }

  auto get_allocator() const noexcept -> allocator_type { return m_values.get_allocator(); }

  // iterators //////////////////////////////////////////////////////////////

  auto begin() noexcept -> iterator { return m_values.begin(); }

  auto begin() const noexcept -> const_iterator { return m_values.begin(); }

  auto cbegin() const noexcept -> const_iterator { return m_values.cbegin(); }

  auto end() noexcept -> iterator { return m_values.end(); }

  auto cend() const noexcept -> const_iterator { return m_values.cend(); }

  auto end() const noexcept -> const_iterator { return m_values.end(); }

  // capacity ///////////////////////////////////////////////////////////////

  [[nodiscard]] auto empty() const noexcept -> bool { return m_values.empty(); }

  [[nodiscard]] auto size() const noexcept -> size_t { return m_values.size(); }

  [[nodiscard]] static constexpr auto max_size() noexcept -> size_t {
    if constexpr (std::numeric_limits<value_idx_type>::max() == std::numeric_limits<size_t>::max()) {
      return size_t{1} << (sizeof(value_idx_type) * 8 - 1);
    } else {
      return size_t{1} << (sizeof(value_idx_type) * 8);
    }
  }

  // modifiers //////////////////////////////////////////////////////////////

  void clear() {
    m_values.clear();
    clear_buckets();
  }

  auto insert(value_type const &value) -> std::pair<iterator, bool> { return emplace(value); }

  auto insert(value_type &&value) -> std::pair<iterator, bool> { return emplace(std::move(value)); }

  template <class P, std::enable_if_t<std::is_constructible_v<value_type, P &&>, bool> = true>
  auto insert(P &&value) -> std::pair<iterator, bool> {
    return emplace(std::forward<P>(value));
  }

  auto insert(const_iterator /*hint*/, value_type const &value) -> iterator { return insert(value).first; }

  auto insert(const_iterator /*hint*/, value_type &&value) -> iterator { return insert(std::move(value)).first; }

  template <class P, std::enable_if_t<std::is_constructible_v<value_type, P &&>, bool> = true>
  auto insert(const_iterator /*hint*/, P &&value) -> iterator {
    return insert(std::forward<P>(value)).first;
  }

  template <class InputIt>
  void insert(InputIt first, InputIt last) {
    while (first != last) {
      insert(*first);
      ++first;
    }
  }

  void insert(std::initializer_list<value_type> ilist) { insert(ilist.begin(), ilist.end()); }

  // nonstandard API: *this is emptied.
  // Also see "A Standard flat_map" https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2022/p0429r9.pdf
  auto extract() && -> value_container_type { return std::move(m_values); }

  // nonstandard API:
  // Discards the internally held container and replaces it with the one passed. Erases non-unique elements.
  auto replace(value_container_type &&container) {
    if (container.size() > max_size()) {
      throw std::out_of_range("ankerl::unordered_dense::map::replace(): too many elements");
    }

    auto shifts = calc_shifts_for_size(container.size());
    if (0 == m_num_buckets || shifts < m_shifts || container.get_allocator() != m_values.get_allocator()) {
      m_shifts = shifts;
      deallocate_buckets();
      allocate_buckets_from_shift();
    }
    clear_buckets();

    m_values = std::move(container);

    // can't use clear_and_fill_buckets_from_values() because container elements might not be unique
    auto value_idx = value_idx_type{};

    // loop until we reach the end of the container. duplicated entries will be replaced with back().
    while (value_idx != static_cast<value_idx_type>(m_values.size())) {
      auto const &key = get_key(m_values[value_idx]);

      auto hash = mixed_hash(key);
      auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
      auto bucket_idx = bucket_idx_from_hash(hash);

      bool key_found = false;
      while (true) {
        auto const &bucket = at(m_buckets, bucket_idx);
        if (dist_and_fingerprint > bucket.m_dist_and_fingerprint) {
          break;
        }
        if (dist_and_fingerprint == bucket.m_dist_and_fingerprint && m_equal(key, m_values[bucket.m_value_idx].first)) {
          key_found = true;
          break;
        }
        dist_and_fingerprint = dist_inc(dist_and_fingerprint);
        bucket_idx = next(bucket_idx);
      }

      if (key_found) {
        if (value_idx != static_cast<value_idx_type>(m_values.size() - 1)) {
          m_values[value_idx] = std::move(m_values.back());
        }
        m_values.pop_back();
      } else {
        place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx);
        ++value_idx;
      }
    }
  }

  template <class M, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
  auto insert_or_assign(Key const &key, M &&mapped) -> std::pair<iterator, bool> {
    return do_insert_or_assign(key, std::forward<M>(mapped));
  }

  template <class M, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
  auto insert_or_assign(Key &&key, M &&mapped) -> std::pair<iterator, bool> {
    return do_insert_or_assign(std::move(key), std::forward<M>(mapped));
  }

  template <typename K,
            typename M,
            typename Q = T,
            typename H = Hash,
            typename KE = KeyEqual,
            std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
  auto insert_or_assign(K &&key, M &&mapped) -> std::pair<iterator, bool> {
    return do_insert_or_assign(std::forward<K>(key), std::forward<M>(mapped));
  }

  template <class M, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
  auto insert_or_assign(const_iterator /*hint*/, Key const &key, M &&mapped) -> iterator {
    return do_insert_or_assign(key, std::forward<M>(mapped)).first;
  }

  template <class M, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
  auto insert_or_assign(const_iterator /*hint*/, Key &&key, M &&mapped) -> iterator {
    return do_insert_or_assign(std::move(key), std::forward<M>(mapped)).first;
  }

  template <typename K,
            typename M,
            typename Q = T,
            typename H = Hash,
            typename KE = KeyEqual,
            std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
  auto insert_or_assign(const_iterator /*hint*/, K &&key, M &&mapped) -> iterator {
    return do_insert_or_assign(std::forward<K>(key), std::forward<M>(mapped)).first;
  }

  // Single arguments for unordered_set can be used without having to construct the value_type
  template <class K,
            typename Q = T,
            typename H = Hash,
            typename KE = KeyEqual,
            std::enable_if_t<!is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
  auto emplace(K &&key) -> std::pair<iterator, bool> {
    if (is_full()) {
      increase_size();
    }

    auto hash = mixed_hash(key);
    auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
    auto bucket_idx = bucket_idx_from_hash(hash);

    while (dist_and_fingerprint <= at(m_buckets, bucket_idx).m_dist_and_fingerprint) {
      if (dist_and_fingerprint == at(m_buckets, bucket_idx).m_dist_and_fingerprint &&
          m_equal(key, m_values[at(m_buckets, bucket_idx).m_value_idx])) {
        // found it, return without ever actually creating anything
        return {begin() + static_cast<difference_type>(at(m_buckets, bucket_idx).m_value_idx), false};
      }
      dist_and_fingerprint = dist_inc(dist_and_fingerprint);
      bucket_idx = next(bucket_idx);
    }

    // value is new, insert element first, so when exception happens we are in a valid state
    m_values.emplace_back(std::forward<K>(key));
    // now place the bucket and shift up until we find an empty spot
    auto value_idx = static_cast<value_idx_type>(m_values.size() - 1);
    place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx);
    return {begin() + static_cast<difference_type>(value_idx), true};
  }

  template <class... Args>
  auto emplace(Args &&...args) -> std::pair<iterator, bool> {
    if (is_full()) {
      increase_size();
    }

    // we have to instantiate the value_type to be able to access the key.
    // 1. emplace_back the object so it is constructed. 2. If the key is already there, pop it later in the loop.
    auto &key = get_key(m_values.emplace_back(std::forward<Args>(args)...));
    auto hash = mixed_hash(key);
    auto dist_and_fingerprint = dist_and_fingerprint_from_hash(hash);
    auto bucket_idx = bucket_idx_from_hash(hash);

    while (dist_and_fingerprint <= at(m_buckets, bucket_idx).m_dist_and_fingerprint) {
      if (dist_and_fingerprint == at(m_buckets, bucket_idx).m_dist_and_fingerprint &&
          m_equal(key, get_key(m_values[at(m_buckets, bucket_idx).m_value_idx]))) {
        m_values.pop_back();  // value was already there, so get rid of it
        return {begin() + static_cast<difference_type>(at(m_buckets, bucket_idx).m_value_idx), false};
      }
      dist_and_fingerprint = dist_inc(dist_and_fingerprint);
      bucket_idx = next(bucket_idx);
    }

    // value is new, place the bucket and shift up until we find an empty spot
    auto value_idx = static_cast<value_idx_type>(m_values.size() - 1);
    place_and_shift_up({dist_and_fingerprint, value_idx}, bucket_idx);

    return {begin() + static_cast<difference_type>(value_idx), true};
  }

  template <class... Args>
  auto emplace_hint(const_iterator /*hint*/, Args &&...args) -> iterator {
    return emplace(std::forward<Args>(args)...).first;
  }

  template <class... Args, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
  auto try_emplace(Key const &key, Args &&...args) -> std::pair<iterator, bool> {
    return do_try_emplace(key, std::forward<Args>(args)...);
  }

  template <class... Args, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
  auto try_emplace(Key &&key, Args &&...args) -> std::pair<iterator, bool> {
    return do_try_emplace(std::move(key), std::forward<Args>(args)...);
  }

  template <class... Args, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
  auto try_emplace(const_iterator /*hint*/, Key const &key, Args &&...args) -> iterator {
    return do_try_emplace(key, std::forward<Args>(args)...).first;
  }

  template <class... Args, typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
  auto try_emplace(const_iterator /*hint*/, Key &&key, Args &&...args) -> iterator {
    return do_try_emplace(std::move(key), std::forward<Args>(args)...).first;
  }

  template <typename K,
            typename... Args,
            typename Q = T,
            typename H = Hash,
            typename KE = KeyEqual,
            std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE> &&
                                 is_neither_convertible_v<K &&, iterator, const_iterator>,
                             bool> = true>
  auto try_emplace(K &&key, Args &&...args) -> std::pair<iterator, bool> {
    return do_try_emplace(std::forward<K>(key), std::forward<Args>(args)...);
  }

  template <typename K,
            typename... Args,
            typename Q = T,
            typename H = Hash,
            typename KE = KeyEqual,
            std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE> &&
                                 is_neither_convertible_v<K &&, iterator, const_iterator>,
                             bool> = true>
  auto try_emplace(const_iterator /*hint*/, K &&key, Args &&...args) -> iterator {
    return do_try_emplace(std::forward<K>(key), std::forward<Args>(args)...).first;
  }

  auto erase(iterator it) -> iterator {
    auto hash = mixed_hash(get_key(*it));
    auto bucket_idx = bucket_idx_from_hash(hash);

    auto const value_idx_to_remove = static_cast<value_idx_type>(it - cbegin());
    while (at(m_buckets, bucket_idx).m_value_idx != value_idx_to_remove) {
      bucket_idx = next(bucket_idx);
    }

    do_erase(bucket_idx);
    return begin() + static_cast<difference_type>(value_idx_to_remove);
  }

  auto erase(const_iterator it) -> iterator { return erase(begin() + (it - cbegin())); }

  auto erase(const_iterator first, const_iterator last) -> iterator {
    auto const idx_first = first - cbegin();
    auto const idx_last = last - cbegin();
    auto const first_to_last = std::distance(first, last);
    auto const last_to_end = std::distance(last, cend());

    // remove elements from left to right which moves elements from the end back
    auto const mid = idx_first + std::min(first_to_last, last_to_end);
    auto idx = idx_first;
    while (idx != mid) {
      erase(begin() + idx);
      ++idx;
    }

    // all elements from the right are moved, now remove the last element until all done
    idx = idx_last;
    while (idx != mid) {
      --idx;
      erase(begin() + idx);
    }

    return begin() + idx_first;
  }

  auto erase(Key const &key) -> size_t { return do_erase_key(key); }

  template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
  auto erase(K &&key) -> size_t {
    return do_erase_key(std::forward<K>(key));
  }

  void swap(table &other) noexcept(
      noexcept(std::is_nothrow_swappable_v<value_container_type> &&std::is_nothrow_swappable_v<Hash>
                   &&std::is_nothrow_swappable_v<KeyEqual>)) {
    using std::swap;
    swap(other, *this);
  }

  // lookup /////////////////////////////////////////////////////////////////

  template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
  auto at(key_type const &key) -> Q & {
    return do_at(key);
  }

  template <typename K,
            typename Q = T,
            typename H = Hash,
            typename KE = KeyEqual,
            std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
  auto at(K const &key) -> Q & {
    return do_at(key);
  }

  template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
  auto at(key_type const &key) const -> Q const & {
    return do_at(key);
  }

  template <typename K,
            typename Q = T,
            typename H = Hash,
            typename KE = KeyEqual,
            std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
  auto at(K const &key) const -> Q const & {
    return do_at(key);
  }

  template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
  auto operator[](Key const &key) -> Q & {
    return try_emplace(key).first->second;
  }

  template <typename Q = T, std::enable_if_t<is_map_v<Q>, bool> = true>
  auto operator[](Key &&key) -> Q & {
    return try_emplace(std::move(key)).first->second;
  }

  template <typename K,
            typename Q = T,
            typename H = Hash,
            typename KE = KeyEqual,
            std::enable_if_t<is_map_v<Q> && is_transparent_v<H, KE>, bool> = true>
  auto operator[](K &&key) -> Q & {
    return try_emplace(std::forward<K>(key)).first->second;
  }

  auto count(Key const &key) const -> size_t { return find(key) == end() ? 0 : 1; }

  template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
  auto count(K const &key) const -> size_t {
    return find(key) == end() ? 0 : 1;
  }

  auto find(Key const &key) -> iterator { return do_find(key); }

  auto find(Key const &key) const -> const_iterator { return do_find(key); }

  template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
  auto find(K const &key) -> iterator {
    return do_find(key);
  }

  template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
  auto find(K const &key) const -> const_iterator {
    return do_find(key);
  }

  auto contains(Key const &key) const -> bool { return find(key) != end(); }

  template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
  auto contains(K const &key) const -> bool {
    return find(key) != end();
  }

  auto equal_range(Key const &key) -> std::pair<iterator, iterator> {
    auto it = do_find(key);
    return {it, it == end() ? end() : it + 1};
  }

  auto equal_range(const Key &key) const -> std::pair<const_iterator, const_iterator> {
    auto it = do_find(key);
    return {it, it == end() ? end() : it + 1};
  }

  template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
  auto equal_range(K const &key) -> std::pair<iterator, iterator> {
    auto it = do_find(key);
    return {it, it == end() ? end() : it + 1};
  }

  template <class K, class H = Hash, class KE = KeyEqual, std::enable_if_t<is_transparent_v<H, KE>, bool> = true>
  auto equal_range(K const &key) const -> std::pair<const_iterator, const_iterator> {
    auto it = do_find(key);
    return {it, it == end() ? end() : it + 1};
  }

  // bucket interface ///////////////////////////////////////////////////////

  auto bucket_count() const noexcept -> size_t {  // NOLINT(modernize-use-nodiscard)
    return m_num_buckets;
  }

  static constexpr auto max_bucket_count() noexcept -> size_t {  // NOLINT(modernize-use-nodiscard)
    return max_size();
  }

  // hash policy ////////////////////////////////////////////////////////////

  [[nodiscard]] auto load_factor() const -> float {
    return bucket_count() ? static_cast<float>(size()) / static_cast<float>(bucket_count()) : 0.0F;
  }

  [[nodiscard]] auto max_load_factor() const -> float { return m_max_load_factor; }

  void max_load_factor(float ml) {
    m_max_load_factor = ml;
    if (m_num_buckets != max_bucket_count()) {
      m_max_bucket_capacity = static_cast<value_idx_type>(static_cast<float>(bucket_count()) * max_load_factor());
    }
  }

  void rehash(size_t count) {
    count = std::min(count, max_size());
    auto shifts = calc_shifts_for_size(std::max(count, size()));
    if (shifts != m_shifts) {
      m_shifts = shifts;
      deallocate_buckets();
      m_values.shrink_to_fit();
      allocate_buckets_from_shift();
      clear_and_fill_buckets_from_values();
    }
  }

  void reserve(size_t capa) {
    capa = std::min(capa, max_size());
    if constexpr (has_reserve<value_container_type>) {
      // std::deque doesn't have reserve(). Make sure we only call when available
      m_values.reserve(capa);
    }
    auto shifts = calc_shifts_for_size(std::max(capa, size()));
    if (0 == m_num_buckets || shifts < m_shifts) {
      m_shifts = shifts;
      deallocate_buckets();
      allocate_buckets_from_shift();
      clear_and_fill_buckets_from_values();
    }
  }

  // observers //////////////////////////////////////////////////////////////

  auto hash_function() const -> hasher { return m_hash; }

  auto key_eq() const -> key_equal { return m_equal; }

  // nonstandard API: expose the underlying values container
  [[nodiscard]] auto values() const noexcept -> value_container_type const & { return m_values; }

  // non-member functions ///////////////////////////////////////////////////

  friend auto operator==(table const &a, table const &b) -> bool {
    if (&a == &b) {
      return true;
    }
    if (a.size() != b.size()) {
      return false;
    }
    for (auto const &b_entry : b) {
      auto it = a.find(get_key(b_entry));
      if constexpr (std::is_void_v<T>) {
        // set: only check that the key is here
        if (a.end() == it) {
          return false;
        }
      } else {
        // map: check that key is here, then also check that value is the same
        if (a.end() == it || !(b_entry.second == it->second)) {
          return false;
        }
      }
    }
    return true;
  }

  friend auto operator!=(table const &a, table const &b) -> bool { return !(a == b); }
};

}  // namespace detail

template <class Key,
          class T,
          class Hash = hash<Key>,
          class KeyEqual = std::equal_to<Key>,
          class AllocatorOrContainer = std::allocator<std::pair<Key, T>>,
          class Bucket = bucket_type::standard>
using map = detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket>;

template <class Key,
          class Hash = hash<Key>,
          class KeyEqual = std::equal_to<Key>,
          class AllocatorOrContainer = std::allocator<Key>,
          class Bucket = bucket_type::standard>
using set = detail::table<Key, void, Hash, KeyEqual, AllocatorOrContainer, Bucket>;

#if ANKERL_UNORDERED_DENSE_PMR

namespace pmr {

template <class Key,
          class T,
          class Hash = hash<Key>,
          class KeyEqual = std::equal_to<Key>,
          class Bucket = bucket_type::standard>
using map = detail::table<Key, T, Hash, KeyEqual, std::pmr::polymorphic_allocator<std::pair<Key, T>>, Bucket>;

template <class Key, class Hash = hash<Key>, class KeyEqual = std::equal_to<Key>, class Bucket = bucket_type::standard>
using set = detail::table<Key, void, Hash, KeyEqual, std::pmr::polymorphic_allocator<Key>, Bucket>;

}  // namespace pmr

#endif

// deduction guides ///////////////////////////////////////////////////////////

// deduction guides for alias templates are only possible since C++20
// see https://en.cppreference.com/w/cpp/language/class_template_argument_deduction

}  // namespace ANKERL_UNORDERED_DENSE_NAMESPACE
}  // namespace ankerl::unordered_dense

// std extensions /////////////////////////////////////////////////////////////

namespace std {  // NOLINT(cert-dcl58-cpp)

template <class Key, class T, class Hash, class KeyEqual, class AllocatorOrContainer, class Bucket, class Pred>
auto erase_if(ankerl::unordered_dense::detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket> &map,
              Pred pred) -> size_t {
  using map_t = ankerl::unordered_dense::detail::table<Key, T, Hash, KeyEqual, AllocatorOrContainer, Bucket>;

  // going back to front because erase() invalidates the end iterator
  auto const old_size = map.size();
  auto idx = old_size;
  while (idx) {
    --idx;
    auto it = map.begin() + static_cast<typename map_t::difference_type>(idx);
    if (pred(*it)) {
      map.erase(it);
    }
  }

  return map.size() - old_size;
}

}  // namespace std

#endif
#endif

#pragma once

#include <folly/Executor.h>
#include <folly/Synchronized.h>
#include <folly/concurrency/AtomicSharedPtr.h>
#include <folly/executors/IOThreadPoolExecutor.h>
#include <folly/net/NetworkSocket.h>
#include <memory>

#include "common/net/EventLoop.h"
#include "common/net/Network.h"
#include "common/net/Processor.h"
#include "common/net/Transport.h"
#include "common/net/TransportPool.h"
#include "common/net/WriteItem.h"
#include "common/net/ib/IBSocket.h"
#include "common/utils/ConcurrencyLimiter.h"
#include "common/utils/Duration.h"

namespace hf3fs::net {

// A worker that handles all I/O tasks.
class IOWorker {
 public:
  class Config : public ConfigBase<Config> {
    CONFIG_HOT_UPDATED_ITEM(read_write_tcp_in_event_thread, false);
    CONFIG_HOT_UPDATED_ITEM(read_write_rdma_in_event_thread, false);
    CONFIG_HOT_UPDATED_ITEM(tcp_connect_timeout, 1_s);
    CONFIG_HOT_UPDATED_ITEM(rdma_connect_timeout, 5_s);
    CONFIG_HOT_UPDATED_ITEM(wait_to_retry_send, 100_ms);
    CONFIG_ITEM(num_event_loop, 1u);

    CONFIG_OBJ(ibsocket, IBSocket::Config);
    CONFIG_OBJ(transport_pool, TransportPool::Config);
    CONFIG_OBJ(connect_concurrency_limiter, ConcurrencyLimiterConfig, [](auto &c) { c.set_max_concurrency(4); });
  };

  IOWorker(Processor &processor,
           CPUExecutorGroup &executor,
           folly::IOThreadPoolExecutor &connExecutor,
           const Config &config)
      : processor_(processor),
        executor_(executor),
        connExecutor_(connExecutor),
        config_(config),
        connExecutorShared_(std::make_shared<folly::Executor::KeepAlive<>>(&connExecutor_)),
        pool_(config_.transport_pool()),
        eventLoopPool_(config_.num_event_loop()),
        ibsocketConfigGuard_(config.ibsocket().addCallbackGuard([this] {
          auto newVal = config_.ibsocket().drop_connections();
          if (dropConnections_.exchange(newVal) != newVal) {
            XLOGF(INFO, "ioworker@{} drop all connections", fmt::ptr(this));
            dropConnections();
          }
        })),
        connectConcurrencyLimiter_(config_.connect_concurrency_limiter()) {}
  ~IOWorker() { stopAndJoin(); }

  // start and stop.
  Result<Void> start(const std::string &name);
  void stopAndJoin();

  // add TCP socket with ownership into this IO worker. [thread-safe]
  Result<TransportPtr> addTcpSocket(folly::NetworkSocket sock, bool isDomainSocket = false);

  // add IBSocket with ownership into this IO worker. [thread-safe]
  Result<TransportPtr> addIBSocket(std::unique_ptr<IBSocket> sock);

  // send a batch of items asynchronously to specified address. [thread-safe]
  void sendAsync(Address addr, WriteList list);

  // retry a batch of items asynchronously to specified address. [thread-safe]
  void retryAsync(Address addr, WriteList list);

  // drop all connections.
  void dropConnections(bool dropAll = true, bool dropIncome = false) { pool_.dropConnections(dropAll, dropIncome); }

  // drop all connections to this addr
  void dropConnections(Address addr) { pool_.dropConnections(addr); }

  // drop connections to peer.
  void checkConnections(Address addr, Duration expiredTime) { return pool_.checkConnections(addr, expiredTime); }

 protected:
  friend class Transport;
  // get a valid transport.
  TransportPtr getTransport(Address addr);
  // remove a transport.
  void remove(TransportPtr transport) { pool_.remove(std::move(transport)); }

  // connect asynchronous.
  CoTryTask<void> startConnect(TransportPtr transport, Address addr);

  // wait and retry.
  CoTryTask<void> waitAndRetry(Address addr, WriteList list);

  // start a `Transport::doRead` task in current thread or thread pool.
  void startReadTask(Transport *transport, bool error, bool logError = true);
  // start a `Transport::doWrite` task in current thread or thread pool.
  void startWriteTask(Transport *transport, bool error, bool logError = true);

  // process a message received by transport.
  void processMsg(MessageWrapper wrapper, TransportPtr tr) { processor_.processMsg(std::move(wrapper), std::move(tr)); }

  std::weak_ptr<folly::Executor::KeepAlive<>> connExecutorWeak() { return connExecutorShared_.load(); }

 private:
  Processor &processor_;
  CPUExecutorGroup &executor_;
  folly::IOThreadPoolExecutor &connExecutor_;
  const Config &config_;
  std::atomic<uint32_t> dropConnections_{0};

  // export a weak keep alive to Transport
  // note: folly's executor has a weakRef method, but the implementation seems have bug and will cause memory leak.
  folly::atomic_shared_ptr<folly::Executor::KeepAlive<>> connExecutorShared_;

  // keep transports alive.
  TransportPool pool_;
  // monitor I/O events for all transports.
  EventLoopPool eventLoopPool_;

  std::unique_ptr<ConfigCallbackGuard> ibsocketConfigGuard_;

  ConcurrencyLimiter<Address> connectConcurrencyLimiter_;

  constexpr static size_t kStopFlag = 1;
  constexpr static size_t kCountInc = 2;
  std::atomic<size_t> flags_{0};
};

}  // namespace hf3fs::net

#pragma once

#include <folly/Random.h>
#include <folly/Synchronized.h>
#include <folly/executors/task_queue/UnboundedBlockingQueue.h>

#include "common/utils/SimpleSemaphore.h"

namespace hf3fs {
namespace details {
template <typename EndPredicate, typename T>
class EndValueHolder {
 public:
  explicit EndValueHolder(EndPredicate predicate)
      : isEndPredicate_(std::move(predicate)) {}

  bool add(T &&item) {
    if (UNLIKELY(isEndPredicate_(item))) {
      {
        auto guard = endValue_.lock();
        *guard = std::move(item);
      }
      isEnd_.store(true, std::memory_order_release);
      return true;
    }
    return false;
  }

  bool hasValue() const { return isEnd_.load(std::memory_order_acquire); }

  T forceTake() {
    auto guard = endValue_.lock();
    return std::move(**guard);
  }

  folly::Optional<T> take() {
    if (UNLIKELY(isEnd_.load(std::memory_order_acquire))) {
      auto guard = endValue_.lock();
      return std::move(**guard);
    }
    return {};
  }

 private:
  EndPredicate isEndPredicate_;
  std::atomic<bool> isEnd_{false};
  folly::Synchronized<std::optional<T>, std::mutex> endValue_;
};

template <typename T>
class SharedNothingQueueStrategy {
 public:
  using ItemType = T;
  using Queue = folly::UnboundedBlockingQueue<T, SimpleSemaphore>;
  using QueuePtr = std::shared_ptr<Queue>;

  struct SharedState {
    explicit SharedState(size_t count) {
      queues.reserve(count);
      for (uint32_t i = 0; i < count; ++i) queues.push_back(std::make_shared<Queue>());
    }

    std::vector<QueuePtr> queues;
  };

  SharedNothingQueueStrategy(const std::shared_ptr<SharedState> &state, uint32_t selfPos)
      : self_(state->queues[selfPos]) {}

  void add(T &&item) { self_->add(std::move(item)); }

  size_t queueSize() const { return self_->size(); }

  template <typename EndValueHolder>
  folly::Optional<T> tryTake(std::chrono::milliseconds time, EndValueHolder &endValue) {
    if (auto item = self_->try_take_for(time); item) {
      return item;
    }
    return endValue.take();
  }

 private:
  QueuePtr self_;
};

template <typename T>
class WorkStealingQueueStrategy {
 public:
  using ItemType = T;
  using Queue = folly::UnboundedBlockingQueue<T, SimpleSemaphore>;
  using QueuePtr = std::shared_ptr<Queue>;

  struct SharedState {
    explicit SharedState(size_t count) {
      queues.reserve(count);
      for (uint32_t i = 0; i < count; ++i) queues.push_back(std::make_shared<Queue>());
    }

    std::vector<QueuePtr> queues;
  };

  WorkStealingQueueStrategy(const std::shared_ptr<SharedState> &state, uint32_t selfPos)
      : self_(state->queues[selfPos]),
        siblings_(state->queues) {
    siblings_.erase(siblings_.begin() + selfPos);
    std::shuffle(siblings_.begin(), siblings_.end(), folly::ThreadLocalPRNG{});
  }

  void add(T &&item) { self_->add(std::move(item)); }

  size_t queueSize() const { return self_->size(); }

  template <typename EndValueHolder>
  folly::Optional<T> tryTake(std::chrono::milliseconds time, EndValueHolder &endValue) {
    static thread_local size_t called = 0;
    if (LIKELY(called++ % 5 != 4)) {
      if (auto item = self_->try_take_for(time); item) {
        return std::move(item);
      }

      if (auto item = endValue.take(); item) {
        return std::move(item);
      }
    }

    if (!siblings_.empty()) {
      static thread_local size_t next = folly::Random::rand32();
      return siblings_[next++ % siblings_.size()]->try_take();
    }

    return {};
  }

 private:
  QueuePtr self_;
  std::vector<QueuePtr> siblings_;
};

template <typename T>
class RoundRobinQueueStrategy {
 public:
  using ItemType = T;
  using Queue = folly::UnboundedBlockingQueue<T, SimpleSemaphore>;
  using QueuePtr = std::shared_ptr<Queue>;

  struct SharedState {
    explicit SharedState(size_t count) {
      queues.reserve(count);
      for (uint32_t i = 0; i < count; ++i) queues.push_back(std::make_shared<Queue>());
    }

    std::atomic<size_t> next{0};
    std::vector<QueuePtr> queues;
  };

  RoundRobinQueueStrategy(const std::shared_ptr<SharedState> &state, uint32_t selfPos)
      : state_(state),
        selfPos_(selfPos) {
    for (size_t i = 0; i < 4 && i < state->queues.size(); ++i) {
      queues_.push_back(state->queues[(selfPos + i) % state->queues.size()]);
    }
    std::shuffle(queues_.begin() + 1, queues_.end(), folly::ThreadLocalPRNG{});
  }

  void add(T &&item) { queues_[0]->add(std::move(item)); }

  size_t queueSize() const { return queues_[0]->size(); }

  template <typename EndValueHolder>
  folly::Optional<T> tryTake(std::chrono::milliseconds time, EndValueHolder &endValue) {
    static thread_local uint64_t called = 0;
    static thread_local uint64_t next = 0;
    if (LIKELY(called++ % 5 != 4)) {
      for (size_t i = 0; i < queues_.size(); ++i, ++next) {
        auto pos = next % queues_.size();
        if (auto item = queues_[pos]->try_take(); item) {
          return std::move(item);
        }

        if (UNLIKELY(pos == 0 && endValue.hasValue())) {
          return endValue.forceTake();
        }
      }
    }

    ++next;
    return queues_[0]->try_take_for(time);
  }

 private:
  std::shared_ptr<SharedState> state_;
  std::vector<QueuePtr> queues_;
  const size_t selfPos_;
};

// NOTE: BlockingQueueBase should be used as a MPSC queue
template <typename Strategy, typename EndPredicate>
class BlockingQueueBase : public folly::BlockingQueue<typename Strategy::ItemType> {
 public:
  using StrategyType = Strategy;
  using SharedState = typename Strategy::SharedState;
  using T = typename Strategy::ItemType;

  BlockingQueueBase(const std::shared_ptr<SharedState> &state, uint32_t selfPos, EndPredicate isEnd)
      : strategy_(state, selfPos),
        endValue_(std::move(isEnd)) {}

  folly::BlockingQueueAddResult add(T item) override {
    if (!endValue_.add(std::move(item))) {
      strategy_.add(std::move(item));
    }
    return true;
  }

  T take() override {
    for (;;) {
      if (auto item = strategy_.tryTake(timeout_, endValue_); item) {
        return std::move(*item);
      }
    }
    __builtin_unreachable();
  }

  folly::Optional<T> try_take_for(std::chrono::milliseconds time) override {
    auto start = std::chrono::steady_clock::now();
    auto deadline = start + time;
    time = std::min(time, timeout_);
    for (;;) {
      if (auto item = strategy_.tryTake(time, endValue_); item) {
        return std::move(item);
      }
      if (std::chrono::steady_clock::now() >= deadline) {
        return {};
      }
    }
    __builtin_unreachable();
  }

  size_t size() override { return strategy_.queueSize(); }

 private:
  static constexpr auto timeout_ = std::chrono::milliseconds(1);

  Strategy strategy_;
  EndValueHolder<EndPredicate, T> endValue_;
};
}  // namespace details

template <typename T, typename EndPredicate>
using SharedNothingBlockingQueue = details::BlockingQueueBase<details::SharedNothingQueueStrategy<T>, EndPredicate>;

template <typename T, typename EndPredicate>
using WorkStealingBlockingQueue = details::BlockingQueueBase<details::WorkStealingQueueStrategy<T>, EndPredicate>;

template <typename T, typename EndPredicate>
using RoundRobinBlockingQueue = details::BlockingQueueBase<details::RoundRobinQueueStrategy<T>, EndPredicate>;
}  // namespace hf3fs

/*
  __ _ _ __ __ _ _ __   __ _ _ __ ___  ___
 / _` | '__/ _` | '_ \ / _` | '__/ __|/ _ \ Argument Parser for Modern C++
| (_| | | | (_| | |_) | (_| | |  \__ \  __/ http://github.com/p-ranav/argparse
 \__,_|_|  \__, | .__/ \__,_|_|  |___/\___|
           |___/|_|

Licensed under the MIT License <http://opensource.org/licenses/MIT>.
SPDX-License-Identifier: MIT
Copyright (c) 2019-2022 Pranav Srinivas Kumar <pranav.srinivas.kumar@gmail.com>
and other contributors.

Permission is hereby  granted, free of charge, to any  person obtaining a copy
of this software and associated  documentation files (the "Software"), to deal
in the Software  without restriction, including without  limitation the rights
to  use, copy,  modify, merge,  publish, distribute,  sublicense, and/or  sell
copies  of  the Software,  and  to  permit persons  to  whom  the Software  is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE  IS PROVIDED "AS  IS", WITHOUT WARRANTY  OF ANY KIND,  EXPRESS OR
IMPLIED,  INCLUDING BUT  NOT  LIMITED TO  THE  WARRANTIES OF  MERCHANTABILITY,
FITNESS FOR  A PARTICULAR PURPOSE AND  NONINFRINGEMENT. IN NO EVENT  SHALL THE
AUTHORS  OR COPYRIGHT  HOLDERS  BE  LIABLE FOR  ANY  CLAIM,  DAMAGES OR  OTHER
LIABILITY, WHETHER IN AN ACTION OF  CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE  OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#pragma once
#include <algorithm>
#include <any>
#include <array>
#include <cerrno>
#include <charconv>
#include <cstdlib>
#include <functional>
#include <iomanip>
#include <iostream>
#include <iterator>
#include <limits>
#include <list>
#include <map>
#include <numeric>
#include <optional>
#include <sstream>
#include <stdexcept>
#include <string>
#include <string_view>
#include <tuple>
#include <type_traits>
#include <utility>
#include <variant>
#include <vector>

namespace argparse {

namespace details {  // namespace for helper methods

template <typename T, typename = void>
struct HasContainerTraits : std::false_type {};

template <>
struct HasContainerTraits<std::string> : std::false_type {};

template <>
struct HasContainerTraits<std::string_view> : std::false_type {};

template <typename T>
struct HasContainerTraits<T,
                          std::void_t<typename T::value_type,
                                      decltype(std::declval<T>().begin()),
                                      decltype(std::declval<T>().end()),
                                      decltype(std::declval<T>().size())>> : std::true_type {};

template <typename T>
static constexpr bool IsContainer = HasContainerTraits<T>::value;

template <typename T, typename = void>
struct HasStreamableTraits : std::false_type {};

template <typename T>
struct HasStreamableTraits<T, std::void_t<decltype(std::declval<std::ostream &>() << std::declval<T>())>>
    : std::true_type {};

template <typename T>
static constexpr bool IsStreamable = HasStreamableTraits<T>::value;

constexpr std::size_t repr_max_container_size = 5;

template <typename T>
std::string repr(T const &val) {
  if constexpr (std::is_same_v<T, bool>) {
    return val ? "true" : "false";
  } else if constexpr (std::is_convertible_v<T, std::string_view>) {
    return '"' + std::string{std::string_view{val}} + '"';
  } else if constexpr (IsContainer<T>) {
    std::stringstream out;
    out << "{";
    const auto size = val.size();
    if (size > 1) {
      out << repr(*val.begin());
      std::for_each(std::next(val.begin()),
                    std::next(val.begin(),
                              static_cast<typename T::iterator::difference_type>(
                                  std::min<std::size_t>(size, repr_max_container_size) - 1)),
                    [&out](const auto &v) { out << " " << repr(v); });
      if (size <= repr_max_container_size) {
        out << " ";
      } else {
        out << "...";
      }
    }
    if (size > 0) {
      out << repr(*std::prev(val.end()));
    }
    out << "}";
    return out.str();
  } else if constexpr (IsStreamable<T>) {
    std::stringstream out;
    out << val;
    return out.str();
  } else {
    return "<not representable>";
  }
}

namespace {

template <typename T>
constexpr bool standard_signed_integer = false;
template <>
constexpr bool standard_signed_integer<signed char> = true;
template <>
constexpr bool standard_signed_integer<short int> = true;
template <>
constexpr bool standard_signed_integer<int> = true;
template <>
constexpr bool standard_signed_integer<long int> = true;
template <>
constexpr bool standard_signed_integer<long long int> = true;

template <typename T>
constexpr bool standard_unsigned_integer = false;
template <>
constexpr bool standard_unsigned_integer<unsigned char> = true;
template <>
constexpr bool standard_unsigned_integer<unsigned short int> = true;
template <>
constexpr bool standard_unsigned_integer<unsigned int> = true;
template <>
constexpr bool standard_unsigned_integer<unsigned long int> = true;
template <>
constexpr bool standard_unsigned_integer<unsigned long long int> = true;

}  // namespace

constexpr int radix_8 = 8;
constexpr int radix_10 = 10;
constexpr int radix_16 = 16;

template <typename T>
constexpr bool standard_integer = standard_signed_integer<T> || standard_unsigned_integer<T>;

template <class F, class Tuple, class Extra, std::size_t... I>
constexpr decltype(auto) apply_plus_one_impl(F &&f, Tuple &&t, Extra &&x, std::index_sequence<I...> /*unused*/) {
  return std::invoke(std::forward<F>(f), std::get<I>(std::forward<Tuple>(t))..., std::forward<Extra>(x));
}

template <class F, class Tuple, class Extra>
constexpr decltype(auto) apply_plus_one(F &&f, Tuple &&t, Extra &&x) {
  return details::apply_plus_one_impl(std::forward<F>(f),
                                      std::forward<Tuple>(t),
                                      std::forward<Extra>(x),
                                      std::make_index_sequence<std::tuple_size_v<std::remove_reference_t<Tuple>>>{});
}

constexpr auto pointer_range(std::string_view s) noexcept { return std::tuple(s.data(), s.data() + s.size()); }

template <class CharT, class Traits>
constexpr bool starts_with(std::basic_string_view<CharT, Traits> prefix,
                           std::basic_string_view<CharT, Traits> s) noexcept {
  return s.substr(0, prefix.size()) == prefix;
}

enum class chars_format { scientific = 0x1, fixed = 0x2, hex = 0x4, general = fixed | scientific };

struct ConsumeHexPrefixResult {
  bool is_hexadecimal;
  std::string_view rest;
};

using namespace std::literals;

constexpr auto consume_hex_prefix(std::string_view s) -> ConsumeHexPrefixResult {
  if (starts_with("0x"sv, s) || starts_with("0X"sv, s)) {
    s.remove_prefix(2);
    return {true, s};
  }
  return {false, s};
}

template <class T, auto Param>
inline auto do_from_chars(std::string_view s) -> T {
  T x;
  auto [first, last] = pointer_range(s);
  auto [ptr, ec] = std::from_chars(first, last, x, Param);
  if (ec == std::errc()) {
    if (ptr == last) {
      return x;
    }
    throw std::invalid_argument{"pattern does not match to the end"};
  }
  if (ec == std::errc::invalid_argument) {
    throw std::invalid_argument{"pattern not found"};
  }
  if (ec == std::errc::result_out_of_range) {
    throw std::range_error{"not representable"};
  }
  return x;  // unreachable
}

template <class T, auto Param = 0>
struct parse_number {
  auto operator()(std::string_view s) -> T { return do_from_chars<T, Param>(s); }
};

template <class T>
struct parse_number<T, radix_16> {
  auto operator()(std::string_view s) -> T {
    if (auto [ok, rest] = consume_hex_prefix(s); ok) {
      return do_from_chars<T, radix_16>(rest);
    }
    throw std::invalid_argument{"pattern not found"};
  }
};

template <class T>
struct parse_number<T> {
  auto operator()(std::string_view s) -> T {
    auto [ok, rest] = consume_hex_prefix(s);
    if (ok) {
      return do_from_chars<T, radix_16>(rest);
    }
    if (starts_with("0"sv, s)) {
      return do_from_chars<T, radix_8>(rest);
    }
    return do_from_chars<T, radix_10>(rest);
  }
};

namespace {

template <class T>
inline const auto generic_strtod = nullptr;
template <>
inline const auto generic_strtod<float> = strtof;
template <>
inline const auto generic_strtod<double> = strtod;
template <>
inline const auto generic_strtod<long double> = strtold;

}  // namespace

template <class T>
inline auto do_strtod(std::string const &s) -> T {
  if (isspace(static_cast<unsigned char>(s[0])) || s[0] == '+') {
    throw std::invalid_argument{"pattern not found"};
  }

  auto [first, last] = pointer_range(s);
  char *ptr;

  errno = 0;
  auto x = generic_strtod<T>(first, &ptr);
  if (errno == 0) {
    if (ptr == last) {
      return x;
    }
    throw std::invalid_argument{"pattern does not match to the end"};
  }
  if (errno == ERANGE) {
    throw std::range_error{"not representable"};
  }
  return x;  // unreachable
}

template <class T>
struct parse_number<T, chars_format::general> {
  auto operator()(std::string const &s) -> T {
    if (auto r = consume_hex_prefix(s); r.is_hexadecimal) {
      throw std::invalid_argument{"chars_format::general does not parse hexfloat"};
    }

    return do_strtod<T>(s);
  }
};

template <class T>
struct parse_number<T, chars_format::hex> {
  auto operator()(std::string const &s) -> T {
    if (auto r = consume_hex_prefix(s); !r.is_hexadecimal) {
      throw std::invalid_argument{"chars_format::hex parses hexfloat"};
    }

    return do_strtod<T>(s);
  }
};

template <class T>
struct parse_number<T, chars_format::scientific> {
  auto operator()(std::string const &s) -> T {
    if (auto r = consume_hex_prefix(s); r.is_hexadecimal) {
      throw std::invalid_argument{"chars_format::scientific does not parse hexfloat"};
    }
    if (s.find_first_of("eE") == std::string::npos) {
      throw std::invalid_argument{"chars_format::scientific requires exponent part"};
    }

    return do_strtod<T>(s);
  }
};

template <class T>
struct parse_number<T, chars_format::fixed> {
  auto operator()(std::string const &s) -> T {
    if (auto r = consume_hex_prefix(s); r.is_hexadecimal) {
      throw std::invalid_argument{"chars_format::fixed does not parse hexfloat"};
    }
    if (s.find_first_of("eE") != std::string::npos) {
      throw std::invalid_argument{"chars_format::fixed does not parse exponent part"};
    }

    return do_strtod<T>(s);
  }
};

template <typename StrIt>
std::string join(StrIt first, StrIt last, const std::string &separator) {
  if (first == last) {
    return "";
  }
  std::stringstream value;
  value << *first;
  ++first;
  while (first != last) {
    value << separator << *first;
    ++first;
  }
  return value.str();
}

}  // namespace details

enum class nargs_pattern { optional, any, at_least_one };

enum class default_arguments : unsigned int {
  none = 0,
  help = 1,
  version = 2,
  all = help | version,
};

inline default_arguments operator&(const default_arguments &a, const default_arguments &b) {
  return static_cast<default_arguments>(static_cast<std::underlying_type<default_arguments>::type>(a) &
                                        static_cast<std::underlying_type<default_arguments>::type>(b));
}

class ArgumentParser;

class Argument {
  friend class ArgumentParser;
  friend auto operator<<(std::ostream &stream, const ArgumentParser &parser) -> std::ostream &;

  template <std::size_t N, std::size_t... I>
  explicit Argument(std::string_view prefix_chars,
                    std::array<std::string_view, N> &&a,
                    std::index_sequence<I...> /*unused*/)
      : m_is_optional((is_optional(a[I], prefix_chars) || ...)),
        m_is_required(false),
        m_is_repeatable(false),
        m_is_used(false),
        m_prefix_chars(prefix_chars) {
    ((void)m_names.emplace_back(a[I]), ...);
    std::sort(m_names.begin(), m_names.end(), [](const auto &lhs, const auto &rhs) {
      return lhs.size() == rhs.size() ? lhs < rhs : lhs.size() < rhs.size();
    });
  }

 public:
  template <std::size_t N>
  explicit Argument(std::string_view prefix_chars, std::array<std::string_view, N> &&a)
      : Argument(prefix_chars, std::move(a), std::make_index_sequence<N>{}) {}

  Argument &help(std::string help_text) {
    m_help = std::move(help_text);
    return *this;
  }

  Argument &metavar(std::string metavar) {
    m_metavar = std::move(metavar);
    return *this;
  }

  template <typename T>
  Argument &default_value(T &&value) {
    m_default_value_repr = details::repr(value);
    m_default_value = std::forward<T>(value);
    return *this;
  }

  Argument &required() {
    m_is_required = true;
    return *this;
  }

  Argument &implicit_value(std::any value) {
    m_implicit_value = std::move(value);
    m_num_args_range = NArgsRange{0, 0};
    return *this;
  }

  template <class F, class... Args>
  auto action(F &&callable, Args &&...bound_args)
      -> std::enable_if_t<std::is_invocable_v<F, Args..., std::string const>, Argument &> {
    using action_type = std::
        conditional_t<std::is_void_v<std::invoke_result_t<F, Args..., std::string const>>, void_action, valued_action>;
    if constexpr (sizeof...(Args) == 0) {
      m_action.emplace<action_type>(std::forward<F>(callable));
    } else {
      m_action.emplace<action_type>(
          [f = std::forward<F>(callable), tup = std::make_tuple(std::forward<Args>(bound_args)...)](
              std::string const &opt) mutable { return details::apply_plus_one(f, tup, opt); });
    }
    return *this;
  }

  auto &append() {
    m_is_repeatable = true;
    return *this;
  }

  template <char Shape, typename T>
  auto scan() -> std::enable_if_t<std::is_arithmetic_v<T>, Argument &> {
    static_assert(!(std::is_const_v<T> || std::is_volatile_v<T>), "T should not be cv-qualified");
    auto is_one_of = [](char c, auto... x) constexpr { return ((c == x) || ...); };

    if constexpr (is_one_of(Shape, 'd') && details::standard_integer<T>) {
      action(details::parse_number<T, details::radix_10>());
    } else if constexpr (is_one_of(Shape, 'i') && details::standard_integer<T>) {
      action(details::parse_number<T>());
    } else if constexpr (is_one_of(Shape, 'u') && details::standard_unsigned_integer<T>) {
      action(details::parse_number<T, details::radix_10>());
    } else if constexpr (is_one_of(Shape, 'o') && details::standard_unsigned_integer<T>) {
      action(details::parse_number<T, details::radix_8>());
    } else if constexpr (is_one_of(Shape, 'x', 'X') && details::standard_unsigned_integer<T>) {
      action(details::parse_number<T, details::radix_16>());
    } else if constexpr (is_one_of(Shape, 'a', 'A') && std::is_floating_point_v<T>) {
      action(details::parse_number<T, details::chars_format::hex>());
    } else if constexpr (is_one_of(Shape, 'e', 'E') && std::is_floating_point_v<T>) {
      action(details::parse_number<T, details::chars_format::scientific>());
    } else if constexpr (is_one_of(Shape, 'f', 'F') && std::is_floating_point_v<T>) {
      action(details::parse_number<T, details::chars_format::fixed>());
    } else if constexpr (is_one_of(Shape, 'g', 'G') && std::is_floating_point_v<T>) {
      action(details::parse_number<T, details::chars_format::general>());
    } else {
      static_assert(alignof(T) == 0, "No scan specification for T");
    }

    return *this;
  }

  Argument &nargs(std::size_t num_args) {
    m_num_args_range = NArgsRange{num_args, num_args};
    return *this;
  }

  Argument &nargs(std::size_t num_args_min, std::size_t num_args_max) {
    m_num_args_range = NArgsRange{num_args_min, num_args_max};
    return *this;
  }

  Argument &nargs(nargs_pattern pattern) {
    switch (pattern) {
      case nargs_pattern::optional:
        m_num_args_range = NArgsRange{0, 1};
        break;
      case nargs_pattern::any:
        m_num_args_range = NArgsRange{0, std::numeric_limits<std::size_t>::max()};
        break;
      case nargs_pattern::at_least_one:
        m_num_args_range = NArgsRange{1, std::numeric_limits<std::size_t>::max()};
        break;
    }
    return *this;
  }

  Argument &remaining() {
    m_accepts_optional_like_value = true;
    return nargs(nargs_pattern::any);
  }

  template <typename Iterator>
  Iterator consume(Iterator start, Iterator end, std::string_view used_name = {}) {
    if (!m_is_repeatable && m_is_used) {
      throw std::runtime_error("Duplicate argument");
    }
    m_is_used = true;
    m_used_name = used_name;

    const auto num_args_max = m_num_args_range.get_max();
    const auto num_args_min = m_num_args_range.get_min();
    std::size_t dist = 0;
    if (num_args_max == 0) {
      m_values.emplace_back(m_implicit_value);
      std::visit([](const auto &f) { f({}); }, m_action);
      return start;
    }
    if ((dist = static_cast<std::size_t>(std::distance(start, end))) >= num_args_min) {
      if (num_args_max < dist) {
        end = std::next(start, static_cast<typename Iterator::difference_type>(num_args_max));
      }
      if (!m_accepts_optional_like_value) {
        end = std::find_if(start, end, std::bind(is_optional, std::placeholders::_1, m_prefix_chars));
        dist = static_cast<std::size_t>(std::distance(start, end));
        if (dist < num_args_min) {
          throw std::runtime_error("Too few arguments");
        }
      }

      struct ActionApply {
        void operator()(valued_action &f) { std::transform(first, last, std::back_inserter(self.m_values), f); }

        void operator()(void_action &f) {
          std::for_each(first, last, f);
          if (!self.m_default_value.has_value()) {
            if (!self.m_accepts_optional_like_value) {
              self.m_values.resize(static_cast<std::size_t>(std::distance(first, last)));
            }
          }
        }

        Iterator first, last;
        Argument &self;
      };
      std::visit(ActionApply{start, end, *this}, m_action);
      return end;
    }
    if (m_default_value.has_value()) {
      return start;
    }
    throw std::runtime_error("Too few arguments for '" + std::string(m_used_name) + "'.");
  }

  /*
   * @throws std::runtime_error if argument values are not valid
   */
  void validate() const {
    if (m_is_optional) {
      // TODO: check if an implicit value was programmed for this argument
      if (!m_is_used && !m_default_value.has_value() && m_is_required) {
        throw_required_arg_not_used_error();
      }
      if (m_is_used && m_is_required && m_values.empty()) {
        throw_required_arg_no_value_provided_error();
      }
    } else {
      if (!m_num_args_range.contains(m_values.size()) && !m_default_value.has_value()) {
        throw_nargs_range_validation_error();
      }
    }
  }

  std::string get_inline_usage() const {
    std::stringstream usage;
    // Find the longest variant to show in the usage string
    std::string longest_name = m_names.front();
    for (const auto &s : m_names) {
      if (s.size() > longest_name.size()) {
        longest_name = s;
      }
    }
    if (!m_is_required) {
      usage << "[";
    }
    usage << longest_name;
    const std::string metavar = !m_metavar.empty() ? m_metavar : "VAR";
    if (m_num_args_range.get_max() > 0) {
      usage << " " << metavar;
      if (m_num_args_range.get_max() > 1) {
        usage << "...";
      }
    }
    if (!m_is_required) {
      usage << "]";
    }
    return usage.str();
  }

  std::size_t get_arguments_length() const {
    std::size_t names_size =
        std::accumulate(std::begin(m_names), std::end(m_names), std::size_t(0), [](const auto &sum, const auto &s) {
          return sum + s.size();
        });

    if (is_positional(m_names.front(), m_prefix_chars)) {
      // A set metavar means this replaces the names
      if (!m_metavar.empty()) {
        // Indent and metavar
        return 2 + m_metavar.size();
      }

      // Indent and space-separated
      return 2 + names_size + (m_names.size() - 1);
    }
    // Is an option - include both names _and_ metavar
    // size = text + (", " between names)
    std::size_t size = names_size + 2 * (m_names.size() - 1);
    if (!m_metavar.empty() && m_num_args_range == NArgsRange{1, 1}) {
      size += m_metavar.size() + 1;
    }
    return size + 2;  // indent
  }

  friend std::ostream &operator<<(std::ostream &stream, const Argument &argument) {
    std::stringstream name_stream;
    name_stream << "  ";  // indent
    if (argument.is_positional(argument.m_names.front(), argument.m_prefix_chars)) {
      if (!argument.m_metavar.empty()) {
        name_stream << argument.m_metavar;
      } else {
        name_stream << details::join(argument.m_names.begin(), argument.m_names.end(), " ");
      }
    } else {
      name_stream << details::join(argument.m_names.begin(), argument.m_names.end(), ", ");
      // If we have a metavar, and one narg - print the metavar
      if (!argument.m_metavar.empty() && argument.m_num_args_range == NArgsRange{1, 1}) {
        name_stream << " " << argument.m_metavar;
      }
    }
    stream << name_stream.str() << "\t" << argument.m_help;

    // print nargs spec
    if (!argument.m_help.empty()) {
      stream << " ";
    }
    stream << argument.m_num_args_range;

    if (argument.m_default_value.has_value() && argument.m_num_args_range != NArgsRange{0, 0}) {
      stream << "[default: " << argument.m_default_value_repr << "]";
    } else if (argument.m_is_required) {
      stream << "[required]";
    }
    stream << "\n";
    return stream;
  }

  template <typename T>
  bool operator!=(const T &rhs) const {
    return !(*this == rhs);
  }

  /*
   * Compare to an argument value of known type
   * @throws std::logic_error in case of incompatible types
   */
  template <typename T>
  bool operator==(const T &rhs) const {
    if constexpr (!details::IsContainer<T>) {
      return get<T>() == rhs;
    } else {
      auto lhs = get<T>();
      return std::equal(std::begin(lhs),
                        std::end(lhs),
                        std::begin(rhs),
                        std::end(rhs),
                        [](const auto &a, const auto &b) { return a == b; });
    }
  }

 private:
  class NArgsRange {
    std::size_t m_min;
    std::size_t m_max;

   public:
    NArgsRange(std::size_t minimum, std::size_t maximum)
        : m_min(minimum),
          m_max(maximum) {
      if (minimum > maximum) {
        throw std::logic_error("Range of number of arguments is invalid");
      }
    }

    bool contains(std::size_t value) const { return value >= m_min && value <= m_max; }

    bool is_exact() const { return m_min == m_max; }

    bool is_right_bounded() const { return m_max < std::numeric_limits<std::size_t>::max(); }

    std::size_t get_min() const { return m_min; }

    std::size_t get_max() const { return m_max; }

    // Print help message
    friend auto operator<<(std::ostream &stream, const NArgsRange &range) -> std::ostream & {
      if (range.m_min == range.m_max) {
        if (range.m_min != 0 && range.m_min != 1) {
          stream << "[nargs: " << range.m_min << "] ";
        }
      } else {
        if (range.m_max == std::numeric_limits<std::size_t>::max()) {
          stream << "[nargs: " << range.m_min << " or more] ";
        } else {
          stream << "[nargs=" << range.m_min << ".." << range.m_max << "] ";
        }
      }
      return stream;
    }

    bool operator==(const NArgsRange &rhs) const { return rhs.m_min == m_min && rhs.m_max == m_max; }

    bool operator!=(const NArgsRange &rhs) const { return !(*this == rhs); }
  };

  void throw_nargs_range_validation_error() const {
    std::stringstream stream;
    if (!m_used_name.empty()) {
      stream << m_used_name << ": ";
    } else {
      stream << m_names.front() << ": ";
    }
    if (m_num_args_range.is_exact()) {
      stream << m_num_args_range.get_min();
    } else if (m_num_args_range.is_right_bounded()) {
      stream << m_num_args_range.get_min() << " to " << m_num_args_range.get_max();
    } else {
      stream << m_num_args_range.get_min() << " or more";
    }
    stream << " argument(s) expected. " << m_values.size() << " provided.";
    throw std::runtime_error(stream.str());
  }

  void throw_required_arg_not_used_error() const {
    std::stringstream stream;
    stream << m_names.front() << ": required.";
    throw std::runtime_error(stream.str());
  }

  void throw_required_arg_no_value_provided_error() const {
    std::stringstream stream;
    stream << m_used_name << ": no value provided.";
    throw std::runtime_error(stream.str());
  }

  static constexpr int eof = std::char_traits<char>::eof();

  static auto lookahead(std::string_view s) -> int {
    if (s.empty()) {
      return eof;
    }
    return static_cast<int>(static_cast<unsigned char>(s[0]));
  }

  /*
   * decimal-literal:
   *    '0'
   *    nonzero-digit digit-sequence_opt
   *    integer-part fractional-part
   *    fractional-part
   *    integer-part '.' exponent-part_opt
   *    integer-part exponent-part
   *
   * integer-part:
   *    digit-sequence
   *
   * fractional-part:
   *    '.' post-decimal-point
   *
   * post-decimal-point:
   *    digit-sequence exponent-part_opt
   *
   * exponent-part:
   *    'e' post-e
   *    'E' post-e
   *
   * post-e:
   *    sign_opt digit-sequence
   *
   * sign: one of
   *    '+' '-'
   */
  static bool is_decimal_literal(std::string_view s) {
    auto is_digit = [](auto c) constexpr {
      switch (c) {
        case '0':
        case '1':
        case '2':
        case '3':
        case '4':
        case '5':
        case '6':
        case '7':
        case '8':
        case '9':
          return true;
        default:
          return false;
      }
    };

    // precondition: we have consumed or will consume at least one digit
    auto consume_digits = [=](std::string_view sd) {
      // NOLINTNEXTLINE(readability-qualified-auto)
      auto it = std::find_if_not(std::begin(sd), std::end(sd), is_digit);
      return sd.substr(static_cast<std::size_t>(it - std::begin(sd)));
    };

    switch (lookahead(s)) {
      case '0': {
        s.remove_prefix(1);
        if (s.empty()) {
          return true;
        }
        goto integer_part;
      }
      case '1':
      case '2':
      case '3':
      case '4':
      case '5':
      case '6':
      case '7':
      case '8':
      case '9': {
        s = consume_digits(s);
        if (s.empty()) {
          return true;
        }
        goto integer_part_consumed;
      }
      case '.': {
        s.remove_prefix(1);
        goto post_decimal_point;
      }
      default:
        return false;
    }

  integer_part:
    s = consume_digits(s);
  integer_part_consumed:
    switch (lookahead(s)) {
      case '.': {
        s.remove_prefix(1);
        if (is_digit(lookahead(s))) {
          goto post_decimal_point;
        } else {
          goto exponent_part_opt;
        }
      }
      case 'e':
      case 'E': {
        s.remove_prefix(1);
        goto post_e;
      }
      default:
        return false;
    }

  post_decimal_point:
    if (is_digit(lookahead(s))) {
      s = consume_digits(s);
      goto exponent_part_opt;
    }
    return false;

  exponent_part_opt:
    switch (lookahead(s)) {
      case eof:
        return true;
      case 'e':
      case 'E': {
        s.remove_prefix(1);
        goto post_e;
      }
      default:
        return false;
    }

  post_e:
    switch (lookahead(s)) {
      case '-':
      case '+':
        s.remove_prefix(1);
    }
    if (is_digit(lookahead(s))) {
      s = consume_digits(s);
      return s.empty();
    }
    return false;
  }

  static bool is_optional(std::string_view name, std::string_view prefix_chars) {
    return !is_positional(name, prefix_chars);
  }

  /*
   * positional:
   *    _empty_
   *    '-'
   *    '-' decimal-literal
   *    !'-' anything
   */
  static bool is_positional(std::string_view name, std::string_view prefix_chars) {
    auto first = lookahead(name);

    if (first == eof) {
      return true;
    } else if (prefix_chars.find(static_cast<char>(first)) != std::string_view::npos) {
      name.remove_prefix(1);
      if (name.empty()) {
        return true;
      }
      return is_decimal_literal(name);
    }
    return true;
  }

  /*
   * Get argument value given a type
   * @throws std::logic_error in case of incompatible types
   */
  template <typename T>
  auto get() const -> std::conditional_t<details::IsContainer<T>, T, const T &> {
    if (!m_values.empty()) {
      if constexpr (details::IsContainer<T>) {
        return any_cast_container<T>(m_values);
      } else {
        return *std::any_cast<T>(&m_values.front());
      }
    }
    if (m_default_value.has_value()) {
      return *std::any_cast<T>(&m_default_value);
    }
    if constexpr (details::IsContainer<T>) {
      if (!m_accepts_optional_like_value) {
        return any_cast_container<T>(m_values);
      }
    }

    throw std::logic_error("No value provided for '" + m_names.back() + "'.");
  }

  /*
   * Get argument value given a type.
   * @pre The object has no default value.
   * @returns The stored value if any, std::nullopt otherwise.
   */
  template <typename T>
  auto present() const -> std::optional<T> {
    if (m_default_value.has_value()) {
      throw std::logic_error("Argument with default value always presents");
    }
    if (m_values.empty()) {
      return std::nullopt;
    }
    if constexpr (details::IsContainer<T>) {
      return any_cast_container<T>(m_values);
    }
    return std::any_cast<T>(m_values.front());
  }

  template <typename T>
  static auto any_cast_container(const std::vector<std::any> &operand) -> T {
    using ValueType = typename T::value_type;

    T result;
    std::transform(std::begin(operand), std::end(operand), std::back_inserter(result), [](const auto &value) {
      return *std::any_cast<ValueType>(&value);
    });
    return result;
  }

  std::vector<std::string> m_names;
  std::string_view m_used_name;
  std::string m_help;
  std::string m_metavar;
  std::any m_default_value;
  std::string m_default_value_repr;
  std::any m_implicit_value;
  using valued_action = std::function<std::any(const std::string &)>;
  using void_action = std::function<void(const std::string &)>;
  std::variant<valued_action, void_action> m_action{std::in_place_type<valued_action>,
                                                    [](const std::string &value) { return value; }};
  std::vector<std::any> m_values;
  NArgsRange m_num_args_range{1, 1};
  bool m_accepts_optional_like_value = false;
  bool m_is_optional : true;
  bool m_is_required : true;
  bool m_is_repeatable : true;
  bool m_is_used : true;            // True if the optional argument is used by user
  std::string_view m_prefix_chars;  // ArgumentParser has the prefix_chars
};

class ArgumentParser {
 public:
  explicit ArgumentParser(std::string program_name = {},
                          std::string version = "1.0",
                          default_arguments add_args = default_arguments::none)
      : m_program_name(std::move(program_name)),
        m_version(std::move(version)),
        m_parser_path(m_program_name) {
    if ((add_args & default_arguments::help) == default_arguments::help) {
      add_argument("-h", "--help")
          .action([&](const auto & /*unused*/) {
            std::cout << help().str();
            std::exit(0);
          })
          .default_value(false)
          .help("shows help message and exits")
          .implicit_value(true)
          .nargs(0);
    }
    if ((add_args & default_arguments::version) == default_arguments::version) {
      add_argument("-v", "--version")
          .action([&](const auto & /*unused*/) {
            std::cout << m_version << std::endl;
            std::exit(0);
          })
          .default_value(false)
          .help("prints version information and exits")
          .implicit_value(true)
          .nargs(0);
    }
  }

  ArgumentParser(ArgumentParser &&) noexcept = default;
  ArgumentParser &operator=(ArgumentParser &&) = default;

  ArgumentParser(const ArgumentParser &other)
      : m_program_name(other.m_program_name),
        m_version(other.m_version),
        m_description(other.m_description),
        m_epilog(other.m_epilog),
        m_prefix_chars(other.m_prefix_chars),
        m_assign_chars(other.m_assign_chars),
        m_is_parsed(other.m_is_parsed),
        m_positional_arguments(other.m_positional_arguments),
        m_optional_arguments(other.m_optional_arguments),
        m_parser_path(other.m_parser_path),
        m_subparsers(other.m_subparsers) {
    for (auto it = std::begin(m_positional_arguments); it != std::end(m_positional_arguments); ++it) {
      index_argument(it);
    }
    for (auto it = std::begin(m_optional_arguments); it != std::end(m_optional_arguments); ++it) {
      index_argument(it);
    }
    for (auto it = std::begin(m_subparsers); it != std::end(m_subparsers); ++it) {
      m_subparser_map.insert_or_assign(it->get().m_program_name, it);
      m_subparser_used.insert_or_assign(it->get().m_program_name, false);
    }
  }

  ~ArgumentParser() = default;

  ArgumentParser &operator=(const ArgumentParser &other) {
    auto tmp = other;
    std::swap(*this, tmp);
    return *this;
  }

  explicit operator bool() const {
    auto arg_used =
        std::any_of(m_argument_map.cbegin(), m_argument_map.cend(), [](auto &it) { return it.second->m_is_used; });
    auto subparser_used =
        std::any_of(m_subparser_used.cbegin(), m_subparser_used.cend(), [](auto &it) { return it.second; });

    return m_is_parsed && (arg_used || subparser_used);
  }

  // Parameter packing
  // Call add_argument with variadic number of string arguments
  template <typename... Targs>
  Argument &add_argument(Targs... f_args) {
    using array_of_sv = std::array<std::string_view, sizeof...(Targs)>;
    auto argument =
        m_optional_arguments.emplace(std::cend(m_optional_arguments), m_prefix_chars, array_of_sv{f_args...});

    if (!argument->m_is_optional) {
      m_positional_arguments.splice(std::cend(m_positional_arguments), m_optional_arguments, argument);
    }

    index_argument(argument);
    return *argument;
  }

  // Parameter packed add_parents method
  // Accepts a variadic number of ArgumentParser objects
  template <typename... Targs>
  ArgumentParser &add_parents(const Targs &...f_args) {
    for (const ArgumentParser &parent_parser : {std::ref(f_args)...}) {
      for (const auto &argument : parent_parser.m_positional_arguments) {
        auto it = m_positional_arguments.insert(std::cend(m_positional_arguments), argument);
        index_argument(it);
      }
      for (const auto &argument : parent_parser.m_optional_arguments) {
        auto it = m_optional_arguments.insert(std::cend(m_optional_arguments), argument);
        index_argument(it);
      }
    }
    return *this;
  }

  ArgumentParser &add_description(std::string description) {
    m_description = std::move(description);
    return *this;
  }

  ArgumentParser &add_epilog(std::string epilog) {
    m_epilog = std::move(epilog);
    return *this;
  }

  /* Getter for arguments and subparsers.
   * @throws std::logic_error in case of an invalid argument or subparser name
   */
  template <typename T = Argument>
  T &at(std::string_view name) {
    if constexpr (std::is_same_v<T, Argument>) {
      return (*this)[name];
    } else {
      auto subparser_it = m_subparser_map.find(name);
      if (subparser_it != m_subparser_map.end()) {
        return subparser_it->second->get();
      }
      throw std::logic_error("No such subparser: " + std::string(name));
    }
  }

  ArgumentParser &set_prefix_chars(std::string prefix_chars) {
    m_prefix_chars = std::move(prefix_chars);
    return *this;
  }

  ArgumentParser &set_assign_chars(std::string assign_chars) {
    m_assign_chars = std::move(assign_chars);
    return *this;
  }

  /* Call parse_args_internal - which does all the work
   * Then, validate the parsed arguments
   * This variant is used mainly for testing
   * @throws std::runtime_error in case of any invalid argument
   */
  void parse_args(const std::vector<std::string> &arguments) {
    parse_args_internal(arguments);
    // Check if all arguments are parsed
    for ([[maybe_unused]] const auto &[unused, argument] : m_argument_map) {
      argument->validate();
    }
  }

  /* Call parse_known_args_internal - which does all the work
   * Then, validate the parsed arguments
   * This variant is used mainly for testing
   * @throws std::runtime_error in case of any invalid argument
   */
  std::vector<std::string> parse_known_args(const std::vector<std::string> &arguments) {
    auto unknown_arguments = parse_known_args_internal(arguments);
    // Check if all arguments are parsed
    for ([[maybe_unused]] const auto &[unused, argument] : m_argument_map) {
      argument->validate();
    }
    return unknown_arguments;
  }

  /* Main entry point for parsing command-line arguments using this
   * ArgumentParser
   * @throws std::runtime_error in case of any invalid argument
   */
  // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays)
  void parse_args(int argc, const char *const argv[]) { parse_args({argv, argv + argc}); }

  /* Main entry point for parsing command-line arguments using this
   * ArgumentParser
   * @throws std::runtime_error in case of any invalid argument
   */
  // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays)
  auto parse_known_args(int argc, const char *const argv[]) { return parse_known_args({argv, argv + argc}); }

  /* Getter for options with default values.
   * @throws std::logic_error if parse_args() has not been previously called
   * @throws std::logic_error if there is no such option
   * @throws std::logic_error if the option has no value
   * @throws std::bad_any_cast if the option is not of type T
   */
  template <typename T = std::string>
  auto get(std::string_view arg_name) const -> std::conditional_t<details::IsContainer<T>, T, const T &> {
    if (!m_is_parsed) {
      throw std::logic_error("Nothing parsed, no arguments are available.");
    }
    return (*this)[arg_name].get<T>();
  }

  /* Getter for options without default values.
   * @pre The option has no default value.
   * @throws std::logic_error if there is no such option
   * @throws std::bad_any_cast if the option is not of type T
   */
  template <typename T = std::string>
  auto present(std::string_view arg_name) const -> std::optional<T> {
    return (*this)[arg_name].present<T>();
  }

  /* Getter that returns true for user-supplied options. Returns false if not
   * user-supplied, even with a default value.
   */
  auto is_used(std::string_view arg_name) const { return (*this)[arg_name].m_is_used; }

  /* Getter that returns true if a subcommand is used.
   */
  auto is_subcommand_used(std::string_view subcommand_name) const { return m_subparser_used.at(subcommand_name); }

  /* Getter that returns true if a subcommand is used.
   */
  auto is_subcommand_used(const ArgumentParser &subparser) const {
    return is_subcommand_used(subparser.m_program_name);
  }

  /* Indexing operator. Return a reference to an Argument object
   * Used in conjunction with Argument.operator== e.g., parser["foo"] == true
   * @throws std::logic_error in case of an invalid argument name
   */
  Argument &operator[](std::string_view arg_name) const {
    auto it = m_argument_map.find(arg_name);
    if (it != m_argument_map.end()) {
      return *(it->second);
    }
    if (!is_valid_prefix_char(arg_name.front())) {
      std::string name(arg_name);
      const auto legal_prefix_char = get_any_valid_prefix_char();
      const auto prefix = std::string(1, legal_prefix_char);

      // "-" + arg_name
      name = prefix + name;
      it = m_argument_map.find(name);
      if (it != m_argument_map.end()) {
        return *(it->second);
      }
      // "--" + arg_name
      name = prefix + name;
      it = m_argument_map.find(name);
      if (it != m_argument_map.end()) {
        return *(it->second);
      }
    }
    throw std::logic_error("No such argument: " + std::string(arg_name));
  }

  // Print help message
  friend auto operator<<(std::ostream &stream, const ArgumentParser &parser) -> std::ostream & {
    stream.setf(std::ios_base::left);

    auto longest_arg_length = parser.get_length_of_longest_argument();

    stream << parser.usage() << "\n\n";

    if (!parser.m_description.empty()) {
      stream << parser.m_description << "\n\n";
    }

    if (!parser.m_positional_arguments.empty()) {
      stream << "Positional arguments:\n";
    }

    for (const auto &argument : parser.m_positional_arguments) {
      stream.width(static_cast<std::streamsize>(longest_arg_length));
      stream << argument;
    }

    if (!parser.m_optional_arguments.empty()) {
      stream << (parser.m_positional_arguments.empty() ? "" : "\n") << "Optional arguments:\n";
    }

    for (const auto &argument : parser.m_optional_arguments) {
      stream.width(static_cast<std::streamsize>(longest_arg_length));
      stream << argument;
    }

    if (!parser.m_subparser_map.empty()) {
      stream << (parser.m_positional_arguments.empty() ? (parser.m_optional_arguments.empty() ? "" : "\n") : "\n")
             << "Subcommands:\n";
      for (const auto &[command, subparser] : parser.m_subparser_map) {
        stream << std::setw(2) << " ";
        stream << std::setw(static_cast<int>(longest_arg_length - 2)) << command;
        stream << " " << subparser->get().m_description << "\n";
      }
    }

    if (!parser.m_epilog.empty()) {
      stream << '\n';
      stream << parser.m_epilog << "\n\n";
    }

    return stream;
  }

  const std::string &program_name() const { return m_program_name; }

  // Format help message
  auto help() const -> std::stringstream {
    std::stringstream out;
    out << *this;
    return out;
  }

  // Format usage part of help only
  auto usage() const -> std::string {
    std::stringstream stream;

    stream << "Usage: " << this->m_program_name;

    // Add any options inline here
    for (const auto &argument : this->m_optional_arguments) {
      stream << " " << argument.get_inline_usage();
    }
    // Put positional arguments after the optionals
    for (const auto &argument : this->m_positional_arguments) {
      if (!argument.m_metavar.empty()) {
        stream << " " << argument.m_metavar;
      } else {
        stream << " " << argument.m_names.front();
      }
    }
    // Put subcommands after positional arguments
    if (!m_subparser_map.empty()) {
      stream << " {";
      std::size_t i{0};
      for (const auto &[command, unused] : m_subparser_map) {
        if (i == 0) {
          stream << command;
        } else {
          stream << "," << command;
        }
        ++i;
      }
      stream << "}";
    }

    return stream.str();
  }

  // Printing the one and only help message
  // I've stuck with a simple message format, nothing fancy.
  [[deprecated("Use cout << program; instead.  See also help().")]] std::string print_help() const {
    auto out = help();
    std::cout << out.rdbuf();
    return out.str();
  }

  void add_subparser(ArgumentParser &parser) {
    parser.m_parser_path = m_program_name + " " + parser.m_program_name;
    auto it = m_subparsers.emplace(std::cend(m_subparsers), parser);
    m_subparser_map.insert_or_assign(parser.m_program_name, it);
    m_subparser_used.insert_or_assign(parser.m_program_name, false);
  }

 private:
  bool is_valid_prefix_char(char c) const { return m_prefix_chars.find(c) != std::string::npos; }

  char get_any_valid_prefix_char() const { return m_prefix_chars[0]; }

  /*
   * Pre-process this argument list. Anything starting with "--", that
   * contains an =, where the prefix before the = has an entry in the
   * options table, should be split.
   */
  std::vector<std::string> preprocess_arguments(const std::vector<std::string> &raw_arguments) const {
    std::vector<std::string> arguments{};
    for (const auto &arg : raw_arguments) {
      const auto argument_starts_with_prefix_chars = [this](const std::string &a) -> bool {
        if (!a.empty()) {
          const auto legal_prefix = [this](char c) -> bool { return m_prefix_chars.find(c) != std::string::npos; };

          // Windows-style
          // if '/' is a legal prefix char
          // then allow single '/' followed by argument name, followed by an
          // assign char, e.g., ':' e.g., 'test.exe /A:Foo'
          const auto windows_style = legal_prefix('/');

          if (windows_style) {
            if (legal_prefix(a[0])) {
              return true;
            }
          } else {
            // Slash '/' is not a legal prefix char
            // For all other characters, only support long arguments
            // i.e., the argument must start with 2 prefix chars, e.g,
            // '--foo' e,g, './test --foo=Bar -DARG=yes'
            if (a.size() > 1) {
              return (legal_prefix(a[0]) && legal_prefix(a[1]));
            }
          }
        }
        return false;
      };

      // Check that:
      // - We don't have an argument named exactly this
      // - The argument starts with a prefix char, e.g., "--"
      // - The argument contains an assign char, e.g., "="
      auto assign_char_pos = arg.find_first_of(m_assign_chars);

      if (m_argument_map.find(arg) == m_argument_map.end() && argument_starts_with_prefix_chars(arg) &&
          assign_char_pos != std::string::npos) {
        // Get the name of the potential option, and check it exists
        std::string opt_name = arg.substr(0, assign_char_pos);
        if (m_argument_map.find(opt_name) != m_argument_map.end()) {
          // This is the name of an option! Split it into two parts
          arguments.push_back(std::move(opt_name));
          arguments.push_back(arg.substr(assign_char_pos + 1));
          continue;
        }
      }
      // If we've fallen through to here, then it's a standard argument
      arguments.push_back(arg);
    }
    return arguments;
  }

  /*
   * @throws std::runtime_error in case of any invalid argument
   */
  void parse_args_internal(const std::vector<std::string> &raw_arguments) {
    auto arguments = preprocess_arguments(raw_arguments);
    if (m_program_name.empty() && !arguments.empty()) {
      m_program_name = arguments.front();
    }
    auto end = std::end(arguments);
    auto positional_argument_it = std::begin(m_positional_arguments);
    for (auto it = std::next(std::begin(arguments)); it != end;) {
      const auto &current_argument = *it;
      if (Argument::is_positional(current_argument, m_prefix_chars)) {
        if (positional_argument_it == std::end(m_positional_arguments)) {
          std::string_view maybe_command = current_argument;

          // Check sub-parsers
          auto subparser_it = m_subparser_map.find(maybe_command);
          if (subparser_it != m_subparser_map.end()) {
            // build list of remaining args
            const auto unprocessed_arguments = std::vector<std::string>(it, end);

            // invoke subparser
            m_is_parsed = true;
            m_subparser_used[maybe_command] = true;
            return subparser_it->second->get().parse_args(unprocessed_arguments);
          }

          throw std::runtime_error("Maximum number of positional arguments exceeded");
        }
        auto argument = positional_argument_it++;
        it = argument->consume(it, end);
        continue;
      }

      auto arg_map_it = m_argument_map.find(current_argument);
      if (arg_map_it != m_argument_map.end()) {
        auto argument = arg_map_it->second;
        it = argument->consume(std::next(it), end, arg_map_it->first);
      } else if (const auto &compound_arg = current_argument; compound_arg.size() > 1 &&
                                                              is_valid_prefix_char(compound_arg[0]) &&
                                                              !is_valid_prefix_char(compound_arg[1])) {
        ++it;
        for (std::size_t j = 1; j < compound_arg.size(); j++) {
          auto hypothetical_arg = std::string{'-', compound_arg[j]};
          auto arg_map_it2 = m_argument_map.find(hypothetical_arg);
          if (arg_map_it2 != m_argument_map.end()) {
            auto argument = arg_map_it2->second;
            it = argument->consume(it, end, arg_map_it2->first);
          } else {
            throw std::runtime_error("Unknown argument: " + current_argument);
          }
        }
      } else {
        throw std::runtime_error("Unknown argument: " + current_argument);
      }
    }
    m_is_parsed = true;
  }

  /*
   * Like parse_args_internal but collects unused args into a vector<string>
   */
  std::vector<std::string> parse_known_args_internal(const std::vector<std::string> &raw_arguments) {
    auto arguments = preprocess_arguments(raw_arguments);

    std::vector<std::string> unknown_arguments{};

    if (m_program_name.empty() && !arguments.empty()) {
      m_program_name = arguments.front();
    }
    auto end = std::end(arguments);
    auto positional_argument_it = std::begin(m_positional_arguments);
    for (auto it = std::next(std::begin(arguments)); it != end;) {
      const auto &current_argument = *it;
      if (Argument::is_positional(current_argument, m_prefix_chars)) {
        if (positional_argument_it == std::end(m_positional_arguments)) {
          std::string_view maybe_command = current_argument;

          // Check sub-parsers
          auto subparser_it = m_subparser_map.find(maybe_command);
          if (subparser_it != m_subparser_map.end()) {
            // build list of remaining args
            const auto unprocessed_arguments = std::vector<std::string>(it, end);

            // invoke subparser
            m_is_parsed = true;
            m_subparser_used[maybe_command] = true;
            return subparser_it->second->get().parse_known_args_internal(unprocessed_arguments);
          }

          // save current argument as unknown and go to next argument
          unknown_arguments.push_back(current_argument);
          ++it;
        } else {
          // current argument is the value of a positional argument
          // consume it
          auto argument = positional_argument_it++;
          it = argument->consume(it, end);
        }
        continue;
      }

      auto arg_map_it = m_argument_map.find(current_argument);
      if (arg_map_it != m_argument_map.end()) {
        auto argument = arg_map_it->second;
        it = argument->consume(std::next(it), end, arg_map_it->first);
      } else if (const auto &compound_arg = current_argument; compound_arg.size() > 1 &&
                                                              is_valid_prefix_char(compound_arg[0]) &&
                                                              !is_valid_prefix_char(compound_arg[1])) {
        ++it;
        for (std::size_t j = 1; j < compound_arg.size(); j++) {
          auto hypothetical_arg = std::string{'-', compound_arg[j]};
          auto arg_map_it2 = m_argument_map.find(hypothetical_arg);
          if (arg_map_it2 != m_argument_map.end()) {
            auto argument = arg_map_it2->second;
            it = argument->consume(it, end, arg_map_it2->first);
          } else {
            unknown_arguments.push_back(current_argument);
            break;
          }
        }
      } else {
        // current argument is an optional-like argument that is unknown
        // save it and move to next argument
        unknown_arguments.push_back(current_argument);
        ++it;
      }
    }
    m_is_parsed = true;
    return unknown_arguments;
  }

  // Used by print_help.
  std::size_t get_length_of_longest_argument() const {
    if (m_argument_map.empty()) {
      return 0;
    }
    std::size_t max_size = 0;
    for ([[maybe_unused]] const auto &[unused, argument] : m_argument_map) {
      max_size = std::max(max_size, argument->get_arguments_length());
    }
    for ([[maybe_unused]] const auto &[command, unused] : m_subparser_map) {
      max_size = std::max(max_size, command.size());
    }
    return max_size;
  }

  using argument_it = std::list<Argument>::iterator;
  using argument_parser_it = std::list<std::reference_wrapper<ArgumentParser>>::iterator;

  void index_argument(argument_it it) {
    for (const auto &name : std::as_const(it->m_names)) {
      m_argument_map.insert_or_assign(name, it);
    }
  }

  std::string m_program_name;
  std::string m_version;
  std::string m_description;
  std::string m_epilog;
  std::string m_prefix_chars{"-"};
  std::string m_assign_chars{"="};
  bool m_is_parsed = false;
  std::list<Argument> m_positional_arguments;
  std::list<Argument> m_optional_arguments;
  std::map<std::string_view, argument_it> m_argument_map;
  std::string m_parser_path;
  std::list<std::reference_wrapper<ArgumentParser>> m_subparsers;
  std::map<std::string_view, argument_parser_it> m_subparser_map;
  std::map<std::string_view, bool> m_subparser_used;
};

}  // namespace argparse

#pragma once

#include <any>
#include <common/utils/StatusCode.h>
#include <common/utils/String.h>
#include <cstdint>
#include <fmt/format.h>
#include <memory>
#include <string_view>

#include "folly/Portability.h"

namespace hf3fs {

#if !FOLLY_X64 && !FOLLY_AARCH64
#error "The platform must be 64bit!"
#endif
static_assert(std::endian::native == std::endian::little);

// `Status` imitates `abseil::Status` which contains:
// - code
// - (optional) message
// - (optional) payload of any type
class [[nodiscard]] Status {
  struct status_ok_t {};

 public:
  Status() = delete;
  explicit Status(status_code_t code)
      : data_(construct(code, nullptr)) {}
  Status(const Status &other) { *this = other; }
  Status(Status &&other) = default;

  constexpr static status_ok_t OK{};
  /* implicit */ Status(status_ok_t)
      : Status(StatusCode::kOK) {}

  Status(status_code_t code, std::string_view msg) {
    auto rep = std::make_unique<StatusRep>();
    rep->message = msg;
    data_ = construct(code, std::move(rep));
  }

  Status(status_code_t code, std::string &&msg) {
    auto rep = std::make_unique<StatusRep>();
    rep->message = std::move(msg);
    data_ = construct(code, std::move(rep));
  }

  Status(status_code_t code, const char *msg)
      : Status(code, std::string_view(msg)) {}

  template <typename T>
  Status(status_code_t code, std::string_view msg, T &&payload)
      : Status(code, msg) {
    setPayload(std::forward<T>(payload));
  }

  Status &operator=(const Status &other) {
    if (std::addressof(other) != this) {
      data_ = construct(other.code(), other.rep() ? std::make_unique<StatusRep>(*other.rep()) : nullptr);
    }
    return *this;
  }
  Status &operator=(Status &&other) = default;

  status_code_t code() const { return reinterpret_cast<uintptr_t>(data_.get()) >> kPtrBits; }
  std::string_view message() const { return rep() ? std::string_view(rep()->message) : std::string_view(); }

  Status convert(status_code_t code) const {
    Status status = Status::OK;
    status.data_ = construct(code, rep() ? std::make_unique<StatusRep>(*rep()) : nullptr);
    return status;
  }

  String describe() const {
    return rep() ? fmt::format("{}({}) {}", StatusCode::toString(code()), code(), rep()->message)
                 : fmt::format("{}({})", StatusCode::toString(code()), code());
  }
  std::ostream &operator<<(std::ostream &os) const { return os << describe(); }

  bool isOK() const { return code() == StatusCode::kOK; }
  explicit operator bool() const { return isOK(); }

  bool hasPayload() const { return rep() && rep()->payload.has_value(); }

  template <typename T>
  T *payload() {
    return std::any_cast<T>(&rep()->payload);
  }

  template <typename T>
  const T *payload() const {
    return std::any_cast<const T>(&rep()->payload);
  }

  template <typename T>
  void setPayload(T &&payload) {
    ensuredRep()->payload = std::forward<T>(payload);
  }

  template <typename T, typename... Args>
  void emplacePayload(Args &&...args) {
    ensuredRep()->payload.emplace<T>(std::forward<Args>(args)...);
  }

  void resetPayload() { rep() ? rep()->payload.reset() : void(); }

 private:
  static_assert(StatusCode::kOK == 0, "StatusCode::kOK must be 0!");
  static_assert(sizeof(status_code_t) == 2, "The width of status_code_t must be 16b");

  static constexpr auto kPtrBits = 48u;
  static constexpr auto kPtrMask = ((1ul << kPtrBits) - 1);

  struct StatusRep {
    String message;
    std::any payload;
  };
  struct StatusRepDeleter {
    void operator()(StatusRep *rep) { delete extractPtr(rep); }
  };
  using StatusPtr = std::unique_ptr<StatusRep, StatusRepDeleter>;

  static StatusPtr construct(status_code_t code, std::unique_ptr<StatusRep> rep) {
    return StatusPtr(
        reinterpret_cast<StatusRep *>(reinterpret_cast<uintptr_t>(rep.release()) | (uintptr_t(code) << kPtrBits)));
  }
  static StatusRep *extractPtr(StatusRep *rep) {
    return reinterpret_cast<StatusRep *>(reinterpret_cast<uintptr_t>(rep) & kPtrMask);
  }

  StatusRep *rep() { return extractPtr(data_.get()); }
  const StatusRep *rep() const { return extractPtr(data_.get()); }

  StatusRep *ensuredRep() {
    if (rep() == nullptr) {
      data_ = construct(code(), std::make_unique<StatusRep>());
    }
    return rep();
  }

 private:
  StatusPtr data_;  // |<-- low 48 bits: rep ptr -->|<-- high 16 bits: status code -->|
};

class StatusException : public std::runtime_error {
 public:
  using std::runtime_error::runtime_error;
  explicit StatusException(Status status)
      : std::runtime_error(status.describe()),
        status_(std::move(status)) {}

  const Status &get() const { return status_; }
  Status &get() { return status_; }

 private:
  Status status_;
};
}  // namespace hf3fs

FMT_BEGIN_NAMESPACE

template <>
struct formatter<hf3fs::Status> : formatter<hf3fs::status_code_t> {
  template <typename FormatContext>
  auto format(const hf3fs::Status &status, FormatContext &ctx) const {
    auto msg = status.message();
    if (msg.empty()) {
      return fmt::format_to(ctx.out(), "{}({})", hf3fs::StatusCode::toString(status.code()), status.code());
    }
    return fmt::format_to(ctx.out(), "{}({}) {}", hf3fs::StatusCode::toString(status.code()), status.code(), msg);
  }
};

FMT_END_NAMESPACE

#pragma once

#include <array>
#include <bit>
#include <cstdint>
#include <folly/hash/Checksum.h>
#include <iterator>
#include <string>
#include <vector>

#include "client/mgmtd/RoutingInfo.h"
#include "common/app/ClientId.h"
#include "common/net/ib/RDMABuf.h"
#include "common/utils/Reflection.h"
#include "common/utils/Result.h"
#include "common/utils/StrongType.h"
#include "common/utils/Uuid.h"
#include "fbs/core/user/User.h"
#include "fbs/mgmtd/MgmtdTypes.h"
#include "fbs/mgmtd/NodeInfo.h"

namespace hf3fs::storage {
using ChainId = ::hf3fs::flat::ChainId;
using ChainVer = ::hf3fs::flat::ChainVersion;
using TargetId = ::hf3fs::flat::TargetId;
using NodeId = ::hf3fs::flat::NodeId;

STRONG_TYPEDEF(uint32_t, ChunkVer);
STRONG_TYPEDEF(uint64_t, RequestId);
STRONG_TYPEDEF(uint16_t, ChannelId);
STRONG_TYPEDEF(uint64_t, ChannelSeqNum);

#define BITFLAGS_SET(x, bits) ((x) |= static_cast<uint32_t>(bits))
#define BITFLAGS_CLEAR(x, bits) ((x) &= (~static_cast<uint32_t>(bits)))
#define BITFLAGS_CONTAIN(x, bits) (((x) & static_cast<uint32_t>(bits)) == static_cast<uint32_t>(bits))

#define ALIGN_LOWER(mem, align) ((mem) / (align) * (align))
#define ALIGN_UPPER(mem, align) (((mem) + (align)-1) / (align) * (align))

/* start of fault injection helpers */

#ifndef NDEBUG
#define FAULT_INJECTION_POINT(injectError, injectedRes, funcRes) \
  (UNLIKELY(injectError) ? (XLOGF(WARN, "Injected error: {}", (injectedRes)), (injectedRes)) : (funcRes))
#else
#define FAULT_INJECTION_POINT(injectError, injectedRes, funcRes) (boost::ignore_unused(injectError), (funcRes))
#endif

/* end of fault injection helpers */

enum class UpdateType : uint8_t {
  INVALID = 0,
  WRITE = 1,
  REMOVE = 2,
  TRUNCATE = 4,
  EXTEND = 8,
  COMMIT = 16,
};

enum class ChunkState : uint8_t {
  COMMIT = 0,
  DIRTY = 1,
  CLEAN = 2,
};

enum class ChecksumType : uint8_t {
  NONE = 0,
  CRC32C = 1,
  CRC32 = 2,
};

enum class FeatureFlags : uint32_t {
  DEFAULT = 0,
  BYPASS_DISKIO = 1,
  BYPASS_RDMAXMIT = 2,
  SEND_DATA_INLINE = 4,
  ALLOW_READ_UNCOMMITTED = 8,
};

constexpr auto kAIOAlignSize = 4096ul;

class ChunkId {
 public:
  ChunkId() = default;
  explicit ChunkId(const std::string &data)
      : data_(data) {}
  explicit ChunkId(std::string_view data)
      : data_(data) {}
  explicit ChunkId(std::string &&data)
      : data_(std::move(data)) {}

  // special constructors to create 128-bit chunk id from 64-bit integers
  ChunkId(uint64_t high, uint64_t low);
  ChunkId(const ChunkId &baseChunkId, uint64_t chunkIndex);

  bool operator==(const ChunkId &other) const { return other.data_ == data_; }
  auto operator<=>(const ChunkId &other) const { return data_ <=> other.data_; }

  ChunkId nextChunkId() const;

  ChunkId rangeEndForCurrentChunk() const;

  auto &data() const { return data_; }
  std::string describe() const;
  std::string toString() const { return describe(); }
  static Result<ChunkId> fromString(std::string_view str);

 private:
  std::string data_;
};
static_assert(serde::Serializable<ChunkId>);

struct ChecksumInfo {
  SERDE_STRUCT_FIELD(type, ChecksumType::NONE);
  SERDE_STRUCT_FIELD(value, uint32_t{});

 public:
  static constexpr size_t kChunkSize = 1_MB;

  class DataIterator {
   public:
    virtual ~DataIterator() = default;
    virtual std::pair<const uint8_t *, size_t> next() = 0;
  };

  class MemoryDataIterator : public DataIterator {
   public:
    MemoryDataIterator(const uint8_t *buffer, size_t length)
        : buffer_(buffer),
          length_(length) {}

    std::pair<const uint8_t *, size_t> next() override {
      if (length_ == 0) return {nullptr, 0};
      const uint8_t *data = buffer_;
      size_t size = std::min(length_, ChecksumInfo::kChunkSize);
      buffer_ += size;
      length_ -= size;
      return {data, size};
    }

   private:
    const uint8_t *buffer_;
    size_t length_;
  };

  static ChecksumInfo create(ChecksumType type, DataIterator *iter, size_t length, uint32_t startingChecksum = ~0U) {
    ChecksumInfo checksum = {type, startingChecksum};
    size_t iterBytes = 0;

    if (type == ChecksumType::NONE) return ChecksumInfo{ChecksumType::NONE, 0U};

    for (auto data = iter->next(); data.first != nullptr && iterBytes < length; data = iter->next()) {
      iterBytes += data.second;
      switch (checksum.type) {
        case ChecksumType::NONE:
          break;
        case ChecksumType::CRC32C:
          checksum.value = folly::crc32c(data.first, data.second, checksum.value);
          break;
        case ChecksumType::CRC32:
          checksum.value = folly::crc32(data.first, data.second, checksum.value);
          break;
      }
    }

    if (iterBytes != length) {
      XLOGF(WARN, "Iterated bytes {} not equal to length {}, checksum {}", iterBytes, length, checksum);
      return ChecksumInfo{ChecksumType::NONE, 0U};
    }

    return checksum;
  }

  static ChecksumInfo create(ChecksumType type, const uint8_t *buffer, size_t length, uint32_t startingChecksum = ~0U) {
    MemoryDataIterator iter(buffer, length);
    return create(type, &iter, length, startingChecksum);
  }

  Result<Void> combine(const ChecksumInfo &o, size_t length) {
    if (type != ChecksumType::NONE && type != o.type) {
      return makeError(StorageCode::kChecksumMismatch,
                       fmt::format("diffrent type {} != {}", serde::toJsonString(*this), serde::toJsonString(o)));
    }
    if (length == 0) return Void{};
    switch (type) {
      case ChecksumType::NONE:
        *this = o;
        return Void{};

      case ChecksumType::CRC32C:
        value = folly::crc32c_combine(~value, o.value, length);
        return Void{};

      case ChecksumType::CRC32:
        value = folly::crc32_combine(~value, o.value, length);
        return Void{};
    }
  }

  bool operator==(const ChecksumInfo &) const = default;
};
static_assert(serde::Serializable<ChecksumInfo>);

}  // namespace hf3fs::storage

template <>
struct ::hf3fs::serde::SerdeMethod<::hf3fs::storage::ChunkId> {
  static std::string_view serdeTo(const storage::ChunkId &chunkId) { return chunkId.data(); }
  static Result<storage::ChunkId> serdeFrom(std::string_view str) { return storage::ChunkId(str); }
  static std::string serdeToReadable(const storage::ChunkId &chunkId) { return chunkId.describe(); };
  static Result<storage::ChunkId> serdeFromReadable(std::string_view s) { return storage::ChunkId::fromString(s); }
};

template <>
struct std::hash<hf3fs::storage::ChunkId> {
  size_t operator()(const hf3fs::storage::ChunkId &chunkId) const { return std::hash<std::string>{}(chunkId.data()); }
};

namespace hf3fs::storage {

struct IOResult {
  SERDE_STRUCT_FIELD(lengthInfo, Result<uint32_t>{makeError(StorageClientCode::kNotInitialized)});
  SERDE_STRUCT_FIELD(commitVer, ChunkVer{});
  SERDE_STRUCT_FIELD(updateVer, ChunkVer{});
  SERDE_STRUCT_FIELD(checksum, ChecksumInfo{});
  SERDE_STRUCT_FIELD(commitChainVer, ChainVer{});

 public:
  IOResult() = default;
  // for IOResult(makeError())
  IOResult(folly::Unexpected<Status> &&status)
      : lengthInfo(std::move(status)) {}
  IOResult(uint32_t statusCode)
      : IOResult(makeError(statusCode)) {}
  IOResult(Result<uint32_t> lengthInfo,
           ChunkVer commitVer,
           ChunkVer updateVer,
           ChecksumInfo checksum = {},
           ChainVer commitChainVer = {})
      : lengthInfo(lengthInfo),
        commitVer(commitVer),
        updateVer(updateVer),
        checksum(checksum),
        commitChainVer(commitChainVer) {}

  bool operator==(const IOResult &other) const = default;

  friend void PrintTo(const IOResult &res, std::ostream *os) { *os << fmt::to_string(res); }
};
static_assert(serde::Serializable<IOResult>);

struct VersionedChainId {
  bool operator==(const VersionedChainId &) const = default;
  SERDE_STRUCT_FIELD(chainId, ChainId{});
  SERDE_STRUCT_FIELD(chainVer, ChainVer{});
};
static_assert(serde::Serializable<VersionedChainId>);

struct GlobalKey {
  SERDE_STRUCT_FIELD(vChainId, VersionedChainId{});
  SERDE_STRUCT_FIELD(chunkId, ChunkId{});

  static GlobalKey fromFileOffset(const std::vector<VersionedChainId> &chainIds,
                                  const ChunkId &baseChunkId,
                                  const size_t chunkSize,
                                  const size_t fileOffset);
};
static_assert(serde::Serializable<GlobalKey>);

struct UpdateChannel {
  SERDE_STRUCT_FIELD(id, ChannelId{});
  SERDE_STRUCT_FIELD(seqnum, ChannelSeqNum{});
};
static_assert(serde::Serializable<UpdateChannel>);

struct MessageTag {
  SERDE_STRUCT_FIELD(clientId, ClientId{});
  SERDE_STRUCT_FIELD(requestId, RequestId{});
  SERDE_STRUCT_FIELD(channel, UpdateChannel{});

 public:
  MessageTag() = default;
  MessageTag(ClientId clientId, RequestId requestId, UpdateChannel channel = UpdateChannel{})
      : clientId(clientId),
        requestId(requestId),
        channel(channel) {}
};
static_assert(serde::Serializable<MessageTag>);

struct DebugFlags {
  SERDE_STRUCT_FIELD(injectRandomServerError, false);
  SERDE_STRUCT_FIELD(injectRandomClientError, false);
  SERDE_STRUCT_FIELD(numOfInjectPtsBeforeFail, uint16_t{});

 private:
  bool isFailPoint() {
    bool fail = numOfInjectPtsBeforeFail == 1;
    if (numOfInjectPtsBeforeFail > 0) numOfInjectPtsBeforeFail--;
    return fail;
  }

 public:
  bool faultInjectionEnabled() const { return injectRandomServerError || injectRandomClientError; }
  bool injectServerError() { return injectRandomServerError && isFailPoint(); }
  bool injectClientError() { return injectRandomClientError && isFailPoint(); }
};
static_assert(serde::Serializable<DebugFlags>);

struct ReadIO {
  SERDE_STRUCT_FIELD(offset, uint32_t{});
  SERDE_STRUCT_FIELD(length, uint32_t{});
  SERDE_STRUCT_FIELD(key, GlobalKey{});
  SERDE_STRUCT_FIELD(rdmabuf, net::RDMARemoteBuf{});
};
static_assert(serde::Serializable<ReadIO>);

struct UInt8Vector {
  SERDE_STRUCT_FIELD(data, std::vector<uint8_t>{});

 public:
  std::string serdeToReadable() const { return fmt::format("std::vector<uint8_t>({})", data.size()); }
  static Result<UInt8Vector> serdeFromReadable(const std::string &) { return UInt8Vector{}; }
};
static_assert(serde::Serializable<UInt8Vector>);

struct UpdateIO {
  SERDE_STRUCT_FIELD(offset, uint32_t{});
  SERDE_STRUCT_FIELD(length, uint32_t{});
  SERDE_STRUCT_FIELD(chunkSize, uint32_t{});
  SERDE_STRUCT_FIELD(key, GlobalKey{});
  SERDE_STRUCT_FIELD(rdmabuf, net::RDMARemoteBuf{});
  SERDE_STRUCT_FIELD(updateVer, ChunkVer{});
  SERDE_STRUCT_FIELD(updateType, UpdateType{});
  SERDE_STRUCT_FIELD(checksum, ChecksumInfo{});
  SERDE_STRUCT_FIELD(inlinebuf, UInt8Vector{});

 public:
  bool isWrite() const { return updateType == UpdateType::WRITE; }
  bool isTruncate() const { return updateType == UpdateType::TRUNCATE; }
  bool isRemove() const { return updateType == UpdateType::REMOVE; }
  bool isExtend() const { return updateType == UpdateType::EXTEND; }
  bool isCommit() const { return updateType == UpdateType::COMMIT; }
  bool isWriteTruncateExtend() const { return isWrite() || isTruncate() || isExtend(); }
};
static_assert(serde::Serializable<UpdateIO>);

struct CommitIO {
  SERDE_STRUCT_FIELD(key, GlobalKey{});
  SERDE_STRUCT_FIELD(commitVer, ChunkVer{});
  SERDE_STRUCT_FIELD(isSyncing, false);
  SERDE_STRUCT_FIELD(commitChainVer, ChainVer{});
  SERDE_STRUCT_FIELD(isRemove, false);
  SERDE_STRUCT_FIELD(isForce, false);
};

struct BatchReadReq {
  SERDE_STRUCT_FIELD(payloads, std::vector<ReadIO>{});
  SERDE_STRUCT_FIELD(tag, MessageTag{});
  SERDE_STRUCT_FIELD(retryCount, uint32_t{});
  SERDE_STRUCT_FIELD(userInfo, flat::UserInfo{});
  SERDE_STRUCT_FIELD(featureFlags, uint32_t{});
  SERDE_STRUCT_FIELD(checksumType, ChecksumType{});
  SERDE_STRUCT_FIELD(debugFlags, DebugFlags{});
};
static_assert(serde::Serializable<BatchReadReq>);

struct BatchReadRsp {
  SERDE_STRUCT_FIELD(tag, MessageTag{});
  SERDE_STRUCT_FIELD(results, std::vector<IOResult>{});
  SERDE_STRUCT_FIELD(inlinebuf, UInt8Vector{});
};
static_assert(serde::Serializable<BatchReadRsp>);

struct WriteReq {
  SERDE_STRUCT_FIELD(payload, UpdateIO{});
  SERDE_STRUCT_FIELD(tag, MessageTag{});
  SERDE_STRUCT_FIELD(retryCount, uint32_t{});
  SERDE_STRUCT_FIELD(userInfo, flat::UserInfo{});
  SERDE_STRUCT_FIELD(featureFlags, uint32_t{});
  SERDE_STRUCT_FIELD(debugFlags, DebugFlags{});
};
static_assert(serde::Serializable<WriteReq>);

struct WriteRsp {
  SERDE_STRUCT_FIELD(tag, MessageTag{});
  SERDE_STRUCT_FIELD(result, IOResult{});
};
static_assert(serde::Serializable<WriteRsp>);

struct UpdateOptions {
  SERDE_STRUCT_FIELD(isSyncing, bool{});
  SERDE_STRUCT_FIELD(fromClient, bool{});
  SERDE_STRUCT_FIELD(commitChainVer, ChainVer{});
};

struct UpdateReq {
  SERDE_STRUCT_FIELD(payload, UpdateIO{});
  SERDE_STRUCT_FIELD(options, UpdateOptions{});
  SERDE_STRUCT_FIELD(tag, MessageTag{});
  SERDE_STRUCT_FIELD(retryCount, uint32_t{});
  SERDE_STRUCT_FIELD(userInfo, flat::UserInfo{});
  SERDE_STRUCT_FIELD(featureFlags, uint32_t{});
  SERDE_STRUCT_FIELD(debugFlags, DebugFlags{});
};
static_assert(serde::Serializable<UpdateReq>);

struct UpdateRsp {
  SERDE_STRUCT_FIELD(tag, MessageTag{});
  SERDE_STRUCT_FIELD(result, IOResult{});
};
static_assert(serde::Serializable<UpdateRsp>);

/* queryLastChunk */

struct ChunkIdRange {
  SERDE_STRUCT_FIELD(begin, ChunkId{});
  SERDE_STRUCT_FIELD(end, ChunkId{});
  SERDE_STRUCT_FIELD(maxNumChunkIdsToProcess, uint32_t{});
};
static_assert(serde::Serializable<ChunkIdRange>);

struct QueryLastChunkOp {
  SERDE_STRUCT_FIELD(vChainId, VersionedChainId{});
  SERDE_STRUCT_FIELD(chunkIdRange, ChunkIdRange{});
};
static_assert(serde::Serializable<QueryLastChunkOp>);

struct QueryLastChunkReq {
  SERDE_STRUCT_FIELD(payloads, std::vector<QueryLastChunkOp>{});
  SERDE_STRUCT_FIELD(tag, MessageTag{});
  SERDE_STRUCT_FIELD(retryCount, uint32_t{});
  SERDE_STRUCT_FIELD(userInfo, flat::UserInfo{});
  SERDE_STRUCT_FIELD(featureFlags, uint32_t{});
  SERDE_STRUCT_FIELD(debugFlags, DebugFlags{});
};
static_assert(serde::Serializable<QueryLastChunkReq>);

struct QueryLastChunkResult {
  SERDE_STRUCT_FIELD(statusCode, Result<Void>{makeError(StorageClientCode::kNotInitialized)});
  SERDE_STRUCT_FIELD(lastChunkId, ChunkId{});
  SERDE_STRUCT_FIELD(lastChunkLen, uint32_t{});
  SERDE_STRUCT_FIELD(totalChunkLen, uint64_t{});
  SERDE_STRUCT_FIELD(totalNumChunks, uint64_t{});
  SERDE_STRUCT_FIELD(moreChunksInRange, bool{});

 public:
  QueryLastChunkResult() = default;
  QueryLastChunkResult(uint32_t statusCode)
      : QueryLastChunkResult(makeError(statusCode), ChunkId{}, 0, 0, 0, false) {}
  QueryLastChunkResult(const Result<Void> &statusCode,
                       const ChunkId &lastChunkId,
                       uint32_t lastChunkLen,
                       uint64_t totalChunkLen,
                       uint64_t totalNumChunks,
                       bool moreChunksInRange)
      : statusCode(statusCode),
        lastChunkId(lastChunkId),
        lastChunkLen(lastChunkLen),
        totalChunkLen(totalChunkLen),
        totalNumChunks(totalNumChunks),
        moreChunksInRange(moreChunksInRange) {}
};
static_assert(serde::Serializable<QueryLastChunkResult>);

struct QueryLastChunkRsp {
  SERDE_STRUCT_FIELD(results, std::vector<QueryLastChunkResult>{});
};
static_assert(serde::Serializable<QueryLastChunkRsp>);

/* removeChunks */

struct RemoveChunksOp {
  SERDE_STRUCT_FIELD(vChainId, VersionedChainId{});
  SERDE_STRUCT_FIELD(chunkIdRange, ChunkIdRange{});
  SERDE_STRUCT_FIELD(tag, MessageTag{});
  SERDE_STRUCT_FIELD(retryCount, uint32_t{});
};
static_assert(serde::Serializable<RemoveChunksOp>);

struct RemoveChunksReq {
  SERDE_STRUCT_FIELD(payloads, std::vector<RemoveChunksOp>{});
  SERDE_STRUCT_FIELD(userInfo, flat::UserInfo{});
  SERDE_STRUCT_FIELD(featureFlags, uint32_t{});
  SERDE_STRUCT_FIELD(debugFlags, DebugFlags{});
};
static_assert(serde::Serializable<RemoveChunksReq>);

struct RemoveChunksResult {
  SERDE_STRUCT_FIELD(statusCode, Result<Void>{makeError(StorageClientCode::kNotInitialized)});
  SERDE_STRUCT_FIELD(numChunksRemoved, uint32_t{});
  SERDE_STRUCT_FIELD(moreChunksInRange, bool{});

 public:
  RemoveChunksResult() = default;
  RemoveChunksResult(uint32_t statusCode)
      : RemoveChunksResult(makeError(statusCode), 0, false) {}
  RemoveChunksResult(const Result<Void> &statusCode, uint32_t numChunksRemoved, bool moreChunksInRange)
      : statusCode(statusCode),
        numChunksRemoved(numChunksRemoved),
        moreChunksInRange(moreChunksInRange) {}
};
static_assert(serde::Serializable<RemoveChunksResult>);

struct RemoveChunksRsp {
  SERDE_STRUCT_FIELD(results, std::vector<RemoveChunksResult>{});
};
static_assert(serde::Serializable<RemoveChunksRsp>);

/* truncateChunks */

struct TruncateChunkOp {
  SERDE_STRUCT_FIELD(vChainId, VersionedChainId{});
  SERDE_STRUCT_FIELD(chunkId, ChunkId{});
  SERDE_STRUCT_FIELD(chunkLen, uint32_t{});
  SERDE_STRUCT_FIELD(chunkSize, uint32_t{});
  SERDE_STRUCT_FIELD(onlyExtendChunk, bool{});
  SERDE_STRUCT_FIELD(tag, MessageTag{});
  SERDE_STRUCT_FIELD(retryCount, uint32_t{});
};
static_assert(serde::Serializable<TruncateChunkOp>);

struct TruncateChunksReq {
  SERDE_STRUCT_FIELD(payloads, std::vector<TruncateChunkOp>{});
  SERDE_STRUCT_FIELD(userInfo, flat::UserInfo{});
  SERDE_STRUCT_FIELD(featureFlags, uint32_t{});
  SERDE_STRUCT_FIELD(debugFlags, DebugFlags{});
};
static_assert(serde::Serializable<TruncateChunksReq>);

struct TruncateChunksRsp {
  SERDE_STRUCT_FIELD(results, std::vector<IOResult>{});
};
static_assert(serde::Serializable<TruncateChunksRsp>);

/* GetAllChunkMetadata */

struct ChunkMeta {
  SERDE_STRUCT_FIELD(chunkId, ChunkId{});
  SERDE_STRUCT_FIELD(updateVer, ChunkVer{});
  SERDE_STRUCT_FIELD(commitVer, ChunkVer{});
  SERDE_STRUCT_FIELD(chainVer, ChainVer{});
  SERDE_STRUCT_FIELD(chunkState, ChunkState{});
  SERDE_STRUCT_FIELD(checksum, ChecksumInfo{});
  SERDE_STRUCT_FIELD(length, uint32_t{});
};
static_assert(serde::Serializable<ChunkMeta>);

using ChunkMetaVector = std::vector<ChunkMeta>;

struct GetAllChunkMetadataReq {
  SERDE_STRUCT_FIELD(targetId, TargetId{});
};
static_assert(serde::Serializable<GetAllChunkMetadataReq>);

struct GetAllChunkMetadataRsp {
  SERDE_STRUCT_FIELD(chunkMetaVec, ChunkMetaVector{});
};
static_assert(serde::Serializable<GetAllChunkMetadataRsp>);

struct SyncStartReq {
  SERDE_STRUCT_FIELD(vChainId, VersionedChainId{});
};
static_assert(serde::Serializable<SyncStartReq>);

struct TargetSyncInfo {
  SERDE_STRUCT_FIELD(metas, ChunkMetaVector{});
};
static_assert(serde::Serializable<TargetSyncInfo>);

struct SyncDoneReq {
  SERDE_STRUCT_FIELD(vChainId, VersionedChainId{});
};
static_assert(serde::Serializable<SyncDoneReq>);

struct SyncDoneRsp {
  SERDE_STRUCT_FIELD(result, IOResult{});
};
static_assert(serde::Serializable<SyncDoneRsp>);

struct SpaceInfoReq {
  SERDE_STRUCT_FIELD(tag, MessageTag{});
  SERDE_STRUCT_FIELD(force, bool{});
};
static_assert(serde::Serializable<SpaceInfoReq>);

struct SpaceInfo {
  SERDE_STRUCT_FIELD(path, std::string{});
  SERDE_STRUCT_FIELD(capacity, uint64_t{});
  SERDE_STRUCT_FIELD(free, uint64_t{});
  SERDE_STRUCT_FIELD(available, uint64_t{});
  SERDE_STRUCT_FIELD(targetIds, std::vector<hf3fs::flat::TargetId>{});
  SERDE_STRUCT_FIELD(manufacturer, std::string{});
};
static_assert(serde::Serializable<SpaceInfo>);

struct SpaceInfoRsp {
  SERDE_STRUCT_FIELD(spaceInfos, std::vector<SpaceInfo>{});
};
static_assert(serde::Serializable<SpaceInfoRsp>);

struct CreateTargetReq {
  SERDE_STRUCT_FIELD(targetId, TargetId{});
  SERDE_STRUCT_FIELD(diskIndex, uint32_t{});
  SERDE_STRUCT_FIELD(physicalFileCount, 256u);
  SERDE_STRUCT_FIELD(chunkSizeList, (std::vector<Size>{512_KB, 1_MB, 2_MB, 4_MB, 16_MB, 64_MB}));
  SERDE_STRUCT_FIELD(allowExistingTarget, true);
  SERDE_STRUCT_FIELD(chainId, ChainId{});
  SERDE_STRUCT_FIELD(addChunkSize, false);
  SERDE_STRUCT_FIELD(onlyChunkEngine, false);
};

struct CreateTargetRsp {
  SERDE_STRUCT_FIELD(dummy, Void{});
};

struct OfflineTargetReq {
  SERDE_STRUCT_FIELD(targetId, TargetId{});
  SERDE_STRUCT_FIELD(force, false);
};

struct OfflineTargetRsp {
  SERDE_STRUCT_FIELD(dummy, Void{});
};

struct RemoveTargetReq {
  SERDE_STRUCT_FIELD(targetId, TargetId{});
  SERDE_STRUCT_FIELD(force, false);
};

struct RemoveTargetRsp {
  SERDE_STRUCT_FIELD(dummy, Void{});
};

struct QueryChunkReq {
  SERDE_STRUCT_FIELD(chainId, ChainId{});
  SERDE_STRUCT_FIELD(chunkId, ChunkId{});
};

struct ChunkFileId {
  bool operator==(const ChunkFileId &o) const = default;
  SERDE_STRUCT_FIELD(chunkSize, uint32_t{}, nullptr);
  SERDE_STRUCT_FIELD(chunkIdx, uint32_t{}, nullptr);
};

enum class RecycleState : uint8_t {
  NORMAL,
  REMOVAL_IN_PROGRESS,
  REMOVAL_IN_RETRYING,
};

// Metadata of chunk. The order of members has been adjusted for smaller size.
struct ChunkMetadata {
  bool operator==(const ChunkMetadata &o) const = default;

  SERDE_STRUCT_FIELD(commitVer, ChunkVer{}, nullptr);
  SERDE_STRUCT_FIELD(updateVer, ChunkVer{}, nullptr);
  SERDE_STRUCT_FIELD(chainVer, ChainVer{}, nullptr);

  SERDE_STRUCT_FIELD(size, uint32_t{}, nullptr);
  SERDE_STRUCT_FIELD(chunkState, ChunkState{}, nullptr);
  SERDE_STRUCT_FIELD(recycleState, RecycleState::NORMAL, nullptr);
  SERDE_STRUCT_FIELD(checksumType, ChecksumType::NONE, nullptr);
  SERDE_STRUCT_FIELD(checksumValue, uint32_t{}, 0);

  SERDE_STRUCT_FIELD(lastNodeId, uint16_t{}, nullptr);
  SERDE_STRUCT_FIELD(lastRequestId, RequestId{}, nullptr);

  SERDE_STRUCT_FIELD(innerOffset, 0_B, nullptr);
  SERDE_STRUCT_FIELD(innerFileId, ChunkFileId{}, nullptr);

  SERDE_STRUCT_FIELD(lastClientUuid, Uuid{});
  SERDE_STRUCT_FIELD(timestamp, UtcTime{});

 public:
  bool readyToRemove() const { return recycleState != RecycleState::NORMAL; }
  ChecksumInfo checksum() const { return ChecksumInfo{checksumType, checksumValue}; }
};

struct Successor {
  SERDE_STRUCT_FIELD(nodeInfo, flat::NodeInfo{});
  SERDE_STRUCT_FIELD(targetInfo, flat::TargetInfo{});
};

class StorageTarget;
struct Target {
  std::shared_ptr<StorageTarget> storageTarget;
  std::weak_ptr<bool> weakStorageTarget;
  SERDE_STRUCT_FIELD(targetId, TargetId{});
  SERDE_STRUCT_FIELD(path, Path{});
  SERDE_STRUCT_FIELD(diskError, false);
  SERDE_STRUCT_FIELD(lowSpace, false);
  SERDE_STRUCT_FIELD(rejectCreateChunk, false);
  SERDE_STRUCT_FIELD(isHead, false);
  SERDE_STRUCT_FIELD(isTail, false);
  SERDE_STRUCT_FIELD(vChainId, VersionedChainId{});
  SERDE_STRUCT_FIELD(localState, flat::LocalTargetState::INVALID);
  SERDE_STRUCT_FIELD(publicState, flat::PublicTargetState::INVALID);
  SERDE_STRUCT_FIELD(successor, std::optional<Successor>{});
  SERDE_STRUCT_FIELD(diskIndex, uint32_t{});
  SERDE_STRUCT_FIELD(chainId, ChainId{});
  SERDE_STRUCT_FIELD(offlineUponUserRequest, false);
  SERDE_STRUCT_FIELD(useChunkEngine, false);

 public:
  Result<net::Address> getSuccessorAddr() const;

  bool upToDate() const {
    return localState == flat::LocalTargetState::UPTODATE && publicState == flat::PublicTargetState::SERVING;
  }

  bool unrecoverableOffline() const { return diskError || offlineUponUserRequest; }
};
using TargetPtr = std::shared_ptr<const Target>;

struct QueryChunkRsp {
  SERDE_STRUCT_FIELD(target, Target{});
  SERDE_STRUCT_FIELD(meta, Result<ChunkMetadata>{makeError(StatusCode::kInvalidArg)});
};

struct ServiceRequestContext {
  const std::string_view requestType = "dummyReq";
  const MessageTag tag{};
  const uint32_t retryCount{};
  const flat::UserInfo userInfo{};
  DebugFlags debugFlags{};
};

struct StorageEventTrace {
  SERDE_STRUCT_FIELD(clusterId, String{});
  SERDE_STRUCT_FIELD(nodeId, NodeId{});
  SERDE_STRUCT_FIELD(targetId, TargetId{});
  SERDE_STRUCT_FIELD(updateReq, UpdateReq{});
  SERDE_STRUCT_FIELD(updateRes, IOResult{});
  SERDE_STRUCT_FIELD(forwardRes, IOResult{});
  SERDE_STRUCT_FIELD(commitIO, CommitIO{});
  SERDE_STRUCT_FIELD(commitRes, IOResult{});
};

}  // namespace hf3fs::storage

FMT_BEGIN_NAMESPACE

template <>
struct formatter<hf3fs::storage::ChunkId> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(const hf3fs::storage::ChunkId &chunkId, FormatContext &ctx) const {
    return fmt::format_to(ctx.out(), "ChunkId({})", chunkId.describe());
  }
};

template <>
struct formatter<hf3fs::storage::ChunkIdRange> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(const hf3fs::storage::ChunkIdRange &range, FormatContext &ctx) const {
    if (range.maxNumChunkIdsToProcess == 1) {
      return fmt::format_to(ctx.out(), "{}", range.begin);
    } else {
      return fmt::format_to(ctx.out(),
                            "ChunkIdRange[{}, {}){{{}}}",
                            range.begin.describe(),
                            range.end.describe(),
                            range.maxNumChunkIdsToProcess);
    }
  }
};

template <>
struct formatter<hf3fs::storage::ChecksumInfo> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(const hf3fs::storage::ChecksumInfo &checksum, FormatContext &ctx) const {
    return fmt::format_to(ctx.out(), "{}#{:08X}", magic_enum::enum_name(checksum.type), ~checksum.value);
  }
};

template <>
struct formatter<hf3fs::storage::IOResult> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(const hf3fs::storage::IOResult &result, FormatContext &ctx) const {
    return fmt::format_to(ctx.out(),
                          "length{{{}}} version{{{}/{}}} checksum{{{}}} {{{}}}",
                          result.lengthInfo,
                          result.updateVer,
                          result.commitVer,
                          result.checksum,
                          result.commitChainVer);
  }
};

template <>
struct formatter<hf3fs::storage::UpdateChannel> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(const hf3fs::storage::UpdateChannel &channel, FormatContext &ctx) const {
    return fmt::format_to(ctx.out(), "@{}#{}", channel.id, channel.seqnum);
  }
};

template <>
struct formatter<hf3fs::storage::MessageTag> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(const hf3fs::storage::MessageTag &tag, FormatContext &ctx) const {
    return fmt::format_to(ctx.out(), "@{}#{}:{}", tag.clientId, tag.requestId, tag.channel);
  }
};

FMT_END_NAMESPACE

#pragma once

#include <limits>

#include "chunk_engine/src/cxx.rs.h"
#include "common/utils/UtcTime.h"
#include "fbs/storage/Common.h"
#include "storage/aio/BatchReadJob.h"
#include "storage/update/UpdateJob.h"

namespace hf3fs::storage {

struct ChunkEngine {
  static void copyMeta(const chunk_engine::RawMeta &in, ChunkMetadata &out) {
    out.commitVer = ChunkVer{in.chunk_ver};
    out.updateVer = ChunkVer{in.chunk_ver};
    out.chainVer = ChainVer{in.chain_ver};
    out.size = in.len;
    out.chunkState = ChunkState::COMMIT;
    out.recycleState = RecycleState::NORMAL;
    out.checksumType = ChecksumType::CRC32C;
    out.checksumValue = ~in.checksum;
    out.innerFileId = ChunkFileId{std::max(uint32_t(in.pos >> 48 << 16), 512u * 1024), 256};
    out.innerOffset = in.pos;
    out.timestamp = UtcTime::fromMicroseconds(in.timestamp);
    out.lastRequestId = RequestId{in.last_request_id};
    out.lastClientUuid = Uuid::from(in.last_client_low, in.last_client_high);
  }

  static rust::Slice<const uint8_t> toSlice(const std::string &key) {
    return rust::Slice<const uint8_t>{(const uint8_t *)key.data(), key.size()};
  }

  static Result<Void> aioPrepareRead(chunk_engine::Engine &engine, AioReadJob &job) {
    auto &state = job.state();

    if (!state.chunkEngineJob.has_chunk()) {
      const auto &chunkId = job.readIO().key.chunkId;
      auto chainId = job.readIO().key.vChainId.chainId;

      std::string key;
      key.reserve(sizeof(chainId) + chunkId.data().size());
      key.append((const char *)&chainId, sizeof(chainId));
      key.append(chunkId.data());

      std::string error;
      auto chunk = engine.get_raw_chunk(toSlice(key), error);
      if (UNLIKELY(!error.empty())) {
        return makeError(StorageCode::kChunkMetadataGetError, std::move(error));
      }

      if (chunk == nullptr) {
        return makeError(StorageCode::kChunkMetadataNotFound);
      }

      state.chunkEngineJob.set(&engine, chunk);
    }

    auto &result = job.result();
    auto &meta = state.chunkEngineJob.chunk()->raw_meta();
    result.commitVer = ChunkVer{meta.chunk_ver};
    result.updateVer = ChunkVer{meta.chunk_ver};
    result.commitChainVer = ChainVer{meta.chain_ver};
    state.chunkLen = meta.len;
    state.chunkChecksum = ChecksumInfo{ChecksumType::CRC32C, ~meta.checksum};

    auto chunkInfo = state.chunkEngineJob.chunk()->fd_and_offset();
    state.readLength = job.alignedLength();
    state.readFd = chunkInfo.fd;
    state.readOffset = chunkInfo.offset + job.alignedOffset();

    return Void{};
  }

  static Result<uint32_t> update(chunk_engine::Engine &engine, UpdateJob &job);

  static Result<uint32_t> commit(chunk_engine::Engine &engine, UpdateJob &job, bool sync);

  static Result<ChunkMetadata> queryChunk(chunk_engine::Engine &engine, const ChunkId &chunkId, ChainId chainId) {
    std::string key;
    key.reserve(sizeof(chainId) + chunkId.data().size());
    key.append((const char *)&chainId, sizeof(chainId));
    key.append(chunkId.data());

    std::string error;
    auto chunk = engine.get_raw_chunk(toSlice(key), error);
    if (UNLIKELY(!error.empty())) {
      return makeError(StorageCode::kChunkMetadataGetError, std::move(error));
    }

    if (chunk == nullptr) {
      return makeError(StorageCode::kChunkMetadataNotFound);
    }

    ChunkMetadata out;
    copyMeta(chunk->raw_meta(), out);

    engine.release_raw_chunk(chunk);
    return out;
  }

  static Result<std::vector<std::pair<ChunkId, ChunkMetadata>>> queryChunks(chunk_engine::Engine &engine,
                                                                            const ChunkIdRange &chunkIdRange,
                                                                            ChainId chainId) {
    const auto &beginChunkId = chunkIdRange.begin;
    std::string beginKey;
    beginKey.reserve(sizeof(chainId) + beginChunkId.data().size());
    beginKey.append((const char *)&chainId, sizeof(chainId));
    beginKey.append(beginChunkId.data());

    const auto &endChunkId = chunkIdRange.end;
    std::string endKey;
    endKey.reserve(sizeof(chainId) + endChunkId.data().size());
    endKey.append((const char *)&chainId, sizeof(chainId));
    endKey.append(endChunkId.data());

    std::string error;
    auto chunks =
        engine.query_raw_chunks(toSlice(beginKey), toSlice(endKey), chunkIdRange.maxNumChunkIdsToProcess, error);
    if (UNLIKELY(!error.empty())) {
      return makeError(StorageCode::kChunkMetadataGetError, std::move(error));
    }

    auto len = chunks->len();
    std::vector<std::pair<ChunkId, ChunkMetadata>> out;
    out.reserve(len);
    for (size_t i = 0; i < len; ++i) {
      auto chunkId = chunks->chunk_id(i);
      auto &in = chunks->chunk_meta(i);
      out.emplace_back();
      out.back().first =
          ChunkId(std::string_view{(const char *)chunkId.data() + sizeof(chainId), chunkId.length() - sizeof(chainId)});
      copyMeta(in, out.back().second);
    }
    return out;
  }

  static Result<std::vector<ChunkId>> queryUncommittedChunks(chunk_engine::Engine &engine, ChainId chainId) {
    rust::Slice<const uint8_t> prefix{(const uint8_t *)&chainId, sizeof(chainId)};

    std::string error;
    auto chunks = engine.query_uncommitted_raw_chunks(prefix, error);
    if (UNLIKELY(!error.empty())) {
      return makeError(StorageCode::kChunkMetadataGetError, std::move(error));
    }

    auto len = chunks->len();
    std::vector<ChunkId> out;
    out.reserve(len);
    for (size_t i = 0; i < len; ++i) {
      auto chunkId = chunks->chunk_id(i);
      out.push_back(ChunkId(
          std::string_view{(const char *)chunkId.data() + sizeof(chainId), chunkId.length() - sizeof(chainId)}));
    }
    return out;
  }

  static Result<Void> resetUncommittedChunks(chunk_engine::Engine &engine, ChainId chainId, ChainVer chainVer) {
    rust::Slice<const uint8_t> prefix{(const uint8_t *)&chainId, sizeof(chainId)};

    std::string error;
    auto chunks = engine.handle_uncommitted_raw_chunks(prefix, chainVer, error);
    if (UNLIKELY(!error.empty())) {
      XLOGF(CRITICAL, "reset uncommitted chunks failed: {}, chain {}", error, chainId);
      return makeError(StorageCode::kChunkMetadataGetError, std::move(error));
    }

    auto len = chunks->len();
    XLOGF_IF(CRITICAL, len > 0, "reset uncommitted chunks succ, chain: {}, size: {}", chainId, len);
    for (size_t i = 0; i < len; ++i) {
      auto chunkId = chunks->chunk_id(i);
      auto id =
          ChunkId(std::string_view{(const char *)chunkId.data() + sizeof(chainId), chunkId.length() - sizeof(chainId)});
      auto &in = chunks->chunk_meta(i);
      ChunkMetadata meta{};
      copyMeta(in, meta);
      XLOGF(CRITICAL, "reset uncommitted chain {} chunk {} meta {}", chainId, id, meta);
    }

    return Void{};
  }

  static Result<Void> removeAllChunks(chunk_engine::Engine &engine, ChainId chainId) {
    std::string key;
    key.reserve(sizeof(chainId));
    key.append((const char *)&chainId, sizeof(chainId));

    std::string error;
    engine.raw_batch_remove(toSlice(key), toSlice(key), std::numeric_limits<uint64_t>::max(), error);
    if (UNLIKELY(!error.empty())) {
      return makeError(StorageCode::kChunkMetadataSetError, std::move(error));
    }
    return Void{};
  }

  static Result<Void> getAllMetadata(chunk_engine::Engine &engine, ChainId chainId, ChunkMetaVector &metadataVec) {
    rust::Slice<const uint8_t> prefix{(const uint8_t *)&chainId, sizeof(chainId)};
    std::string error;
    auto chunks = engine.query_all_raw_chunks(prefix, error);
    if (UNLIKELY(!error.empty())) {
      return makeError(StorageCode::kChunkMetadataGetError, std::move(error));
    }

    auto len = chunks->len();
    metadataVec.reserve(len);
    for (size_t i = 0; i < len; ++i) {
      auto chunkId = chunks->chunk_id(i);
      auto &in = chunks->chunk_meta(i);

      metadataVec.emplace_back();
      auto &out = metadataVec.back();

      out.chunkId =
          ChunkId(std::string_view{(const char *)chunkId.data() + sizeof(chainId), chunkId.length() - sizeof(chainId)});
      out.updateVer = ChunkVer{in.chunk_ver};
      out.commitVer = ChunkVer{in.chunk_ver};
      out.chainVer = ChainVer{in.chain_ver};
      out.chunkState = ChunkState::COMMIT;
      out.checksum = ChecksumInfo{ChecksumType::CRC32C, ~in.checksum};
      out.length = in.len;
      if (chunks->chunk_uncommitted(i)) {
        out.commitVer = ChunkVer{out.commitVer - 1};
        out.chunkState = ChunkState::CLEAN;
      }
    }
    std::sort(metadataVec.begin(), metadataVec.end(), [](auto &a, auto &b) { return a.chunkId > b.chunkId; });
    return Void{};
  }

  static Result<Void> getAllMetadataMap(chunk_engine::Engine &engine,
                                        std::unordered_map<ChunkId, ChunkMetadata> &metas,
                                        ChainId chainId) {
    rust::Slice<const uint8_t> prefix{(const uint8_t *)&chainId, sizeof(chainId)};
    std::string error;
    auto chunks = engine.query_all_raw_chunks(prefix, error);
    if (UNLIKELY(!error.empty())) {
      return makeError(StorageCode::kChunkMetadataGetError, std::move(error));
    }

    auto len = chunks->len();
    metas.reserve(metas.size() + len);
    for (size_t i = 0; i < len; ++i) {
      auto chunkId = chunks->chunk_id(i);
      auto &meta = metas[ChunkId(
          std::string_view{(const char *)chunkId.data() + sizeof(chainId), chunkId.length() - sizeof(chainId)})];
      copyMeta(chunks->chunk_meta(i), meta);
      if (chunks->chunk_uncommitted(i)) {
        meta.commitVer = ChunkVer{meta.commitVer - 1};
        meta.chunkState = ChunkState::CLEAN;
      }
    }

    return Void{};
  }

  static uint64_t chainUsedSize(chunk_engine::Engine &engine, ChainId chainId) {
    rust::Slice<const uint8_t> slice{(const uint8_t *)&chainId, sizeof(chainId)};
    std::string error;
    auto size = engine.query_raw_used_size(slice, error);
    if (UNLIKELY(!error.empty())) {
      XLOGF(ERR, "query chunk engine chain used size error: chain {} error {}", chainId, error);
    }
    return size;
  }
};

}  // namespace hf3fs::storage

#pragma once

#include <array>
#include <boost/detail/bitmask.hpp>
#include <cstdint>
#include <fcntl.h>
#include <optional>
#include <scn/scn.h>
#include <scn/tuple_return.h>
#include <string_view>
#include <utility>
#include <variant>

#include "common/app/ClientId.h"
#include "common/serde/Serde.h"
#include "common/serde/SerdeComparisons.h"
#include "common/utils/Path.h"
#include "common/utils/Result.h"
#include "common/utils/StrongType.h"
#include "common/utils/UtcTime.h"
#include "common/utils/UtcTimeSerde.h"
#include "fbs/core/user/User.h"

#define VALID Void()
#define INVALID(str) makeError(StatusCode::kInvalidArg, str)

#define FS_CHAIN_ALLOCATION_FL FS_INDEX_FL /* reuse attr 'I', this directory has it't own chain allocation counter */
#define FS_NEW_CHUNK_ENGINE FS_SECRM_FL    /* reuse attr 's', files under this directory will use new chunk engine */
#define FS_FL_SUPPORTED (FS_IMMUTABLE_FL | FS_CHAIN_ALLOCATION_FL | FS_NEW_CHUNK_ENGINE | FS_HUGE_FILE_FL)
#define FS_FL_INHERITABLE (FS_CHAIN_ALLOCATION_FL | FS_NEW_CHUNK_ENGINE)

namespace hf3fs::meta {

using flat::Gid;
using flat::Permission;
using flat::Uid;
using flat::UserAttr;
using flat::UserInfo;

// see: ioctl_iflags
STRONG_TYPEDEF(uint32_t, IFlags);

struct SessionInfo {
  SERDE_STRUCT_FIELD(client, ClientId::zero());
  SERDE_STRUCT_FIELD(session, Uuid::zero());

 public:
  SessionInfo() = default;
  SessionInfo(ClientId client, Uuid session)
      : client(client),
        session(session) {}
  constexpr bool valid() const { return client != ClientId::zero() && session != Uuid::zero(); }
  constexpr operator bool() const { return valid(); }
};

enum AccessType {
  EXEC = 1,
  WRITE = 2,
  READ = 4,
};

BOOST_BITMASK(AccessType)

template <typename C, typename T>
class BitFlags {
 public:
  using Self = C;
  constexpr BitFlags(T val = 0)
      : val_(val) {}
  using is_serde_copyable = void;

  T mask(T m) const { return val_ & m; }
  bool contains(T bits) const { return mask(bits) == bits; }
  void set(T bits) { val_ |= bits; }
  void clear(T bits) { val_ &= (~bits); }

  T toUnderType() const { return val_; }
  operator const T &() const { return val_; }
  operator T &() { return val_; }
  Self operator|(const T other) const { return Self(val_ | other); }

  std::string serdeToReadable() const { return fmt::format("{:x}", val_); }

  static Result<Self> serdeFromReadable(std::string_view str) {
    auto [r, v] = scn::scan_tuple<T>(str, "{:x}");
    if (!r)
      return makeError(StatusCode::kDataCorruption,
                       fmt::format("SCN({}): {}", magic_enum::enum_name(r.error().code()), r.error().msg()));
    return Self(v);
  }

 private:
  T val_;
};

class OpenFlags : public BitFlags<OpenFlags, int32_t> {
 public:
  using Base = BitFlags<OpenFlags, int32_t>;
  using Base::Base;

  AccessType accessType() const {
    switch (mask(O_ACCMODE)) {
      case O_RDONLY:
        return AccessType::READ;
      case O_WRONLY:
        return AccessType::WRITE;
      case O_RDWR:
      default:
        return AccessType::READ | AccessType::WRITE;
    }
  }
  operator AccessType() const { return accessType(); }

  Result<Void> valid() const {
    if (contains(O_DIRECTORY)) {
      if (accessType() != AccessType::READ) return makeError(StatusCode::kInvalidArg, "O_DIRECTORY & WRITE");
      if (contains(O_TRUNC)) return makeError(StatusCode::kInvalidArg, "O_DIRECTORY & O_TRUNC");
    }
    return Void{};
  }
};

class AtFlags : public BitFlags<AtFlags, int32_t> {
 public:
  explicit AtFlags(int32_t val = 0)
      : BitFlags<AtFlags, int32_t>(val) {}

  Result<Void> valid() const { return Void{}; }
  bool followLastSymlink() const { return contains(AT_SYMLINK_FOLLOW) && !contains(AT_SYMLINK_NOFOLLOW); }
};

class InodeId {
 public:
  // Use little endian form as key in FoundationDB, it helps to avoid hot spot
  using Key = std::array<uint8_t, 8>;
  using is_serde_copyable = void;

  static constexpr size_t trees() { return 2; }
  static constexpr InodeId root() { return InodeId(0); }
  static constexpr InodeId gcRoot() { return InodeId(1); }

  // InodeId: 0 ~ 0x01ffffffffffffff
  // If InodeId starts with 0x01: use new 3FS Chunk Engine
  static constexpr uint64_t kNewChunkEngineMask = (1ull << 56);
  static_assert(kNewChunkEngineMask == 0x0100000000000000ull);
  static constexpr InodeId withNewChunkEngine(InodeId orig) { return InodeId(orig.u64() | kNewChunkEngineMask); }
  static constexpr InodeId normalMax() { return InodeId::withNewChunkEngine(InodeId((1ull << 56) - 1)); }

  // special InodeId used by Client
  // 3fs-virt InodeId: 0xfffffffffffffffe
  static constexpr InodeId virt() { return InodeId(uint64_t(-2)); }
  // iov InodeId range [0xffffffff7ffe0002, 0xffffffff7fff0001]
  static constexpr int iovIidStart = 65535;
  static constexpr InodeId iovDir() { return InodeId((uint64_t) - (1ull << 31)); }
  static constexpr InodeId iov(int iovd) { return InodeId(iovDir().u64() - iovIidStart - iovd); }

  // 3fs-virt/rm-rf: 0xfffffffffffffffd
  static constexpr InodeId rmRf() { return InodeId(-3); }
  // temporary InodeId used by rm-rf or mv symlink: [0xfdffffe700000001, 0xffffffe700000000]
  static constexpr InodeId virtTemporary(InodeId toRemove) { return InodeId(-(100ull << 30) - toRemove.u64()); }

  // 0xffffffff00000000
  static constexpr InodeId getConf() { return InodeId(-((uint64_t)2 << 31)); }
  // 0xfffffffe80000000
  static constexpr InodeId setConf() { return InodeId(-((uint64_t)3 << 31)); }

  explicit constexpr InodeId(uint64_t val = 0)
      : val_(val) {}

  // InodeId &operator=(uint64_t val) {
  //   val_ = val;
  //   return *this;
  // }

  Key packKey() const {
    auto le = folly::Endian::little(val_);
    return folly::bit_cast<Key>(le);
  }
  static InodeId unpackKey(const Key &key) {
    auto le = folly::bit_cast<uint64_t>(key);
    return InodeId(folly::Endian::little(le));
  }

  std::string toHexString() const { return fmt::format("0x{:016x}", val_); }
  std::string serdeToReadable() const { return toHexString(); }
  operator folly::dynamic() const { return toHexString(); }
  constexpr uint64_t u64() const { return val_; }
  // constexpr operator uint64_t() const { return u64(); }
  constexpr auto operator<=>(const InodeId &o) const { return this->val_ <=> o.val_; }
  constexpr auto operator==(const InodeId &o) const { return this->val_ == o.val_; }

  bool isTreeRoot() const { return *this == root() || *this == gcRoot(); }

  bool useNewChunkEngine() const { return u64() & kNewChunkEngineMask; }

 private:
  uint64_t val_;
};

// check 3fs-virt InodeIds
static_assert(InodeId::normalMax().u64() == 0x01ffffffffffffffull);
static_assert(InodeId::virt().u64() == 0xfffffffffffffffeull);
static_assert(InodeId::iovDir().u64() == 0xffffffff80000000ull);
static_assert(InodeId::iov(0).u64() == 0xffffffff7fff0001ull);
static_assert(InodeId::iov(65535).u64() == 0xffffffff7ffe0002ull);
static_assert(InodeId::rmRf().u64() == 0xfffffffffffffffdull);
static_assert(InodeId::virtTemporary(InodeId(0)).u64() == 0xffffffe700000000ull);
static_assert(InodeId::virtTemporary(InodeId(0x1ffffffffffffff)).u64() == 0xfdffffe700000001ull);
static_assert(InodeId::getConf().u64() == 0xffffffff00000000ull);
static_assert(InodeId::setConf().u64() == 0xfffffffe80000000ull);

// check 3fs-virt InodeIds won't conflict with each other and normal InodeIds
static constexpr auto spacialInodeRanges =
    std::to_array({InodeId::normalMax(),
                   InodeId(0xfd00000000000000ull),  // all virt must start with 0xf
                   InodeId::virtTemporary(InodeId::normalMax()),
                   InodeId::virtTemporary(InodeId(0)),
                   InodeId::setConf(),
                   InodeId::getConf(),
                   InodeId::iov(65535),
                   InodeId::iov(0),
                   InodeId::iovDir(),
                   InodeId::rmRf(),
                   InodeId::virt()});
constexpr inline bool checkSpecialInode() {
  for (uint64_t i = 1; i < spacialInodeRanges.size(); i++) {
    if (spacialInodeRanges[i] <= spacialInodeRanges[i - 1]) {
      return false;
    }
  }
  return true;
}
static_assert(checkSpecialInode());

struct PathAt {
  SERDE_STRUCT_FIELD(parent, InodeId(0));
  SERDE_STRUCT_FIELD(path, std::optional<Path>());

 public:
  PathAt(InodeId parent = InodeId::root())
      : PathAt(parent, std::nullopt) {}
  PathAt(Path path)
      : PathAt(InodeId::root(), std::move(path)) {}
  PathAt(std::string path)
      : PathAt(Path(path)) {}
  PathAt(const char *p)
      : PathAt(Path(p)) {}
  PathAt(InodeId parent, std::optional<Path> path)
      : parent(parent),
        path(std::move(path)) {}
  Result<Void> validForCreate() const {
    if (!path.has_value() || path->empty()) return INVALID("path not set");
    if (!path->has_filename()) return INVALID("doesn't have filename");
    if (path->filename_is_dot()) return INVALID("filename is .");
    if (path->filename_is_dot_dot()) return INVALID("filename is ..");
    if (path->filename() == "/") return INVALID("filename is /");
    return VALID;
  }
  bool operator==(const PathAt &o) const { return serde::equals(*this, o); }
};

}  // namespace hf3fs::meta

template <>
struct std::hash<hf3fs::meta::InodeId> {
  size_t operator()(const hf3fs::meta::InodeId &id) const { return robin_hood::hash_int(id.u64()); }
};

template <>
struct hf3fs::serde::SerdeMethod<hf3fs::meta::InodeId> {
  static constexpr auto serdeTo(const hf3fs::meta::InodeId &id) { return id.u64(); }
  static Result<hf3fs::meta::InodeId> serdeFrom(uint64_t id) { return hf3fs::meta::InodeId(id); }
};

FMT_BEGIN_NAMESPACE

template <>
struct formatter<hf3fs::meta::InodeId> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(const hf3fs::meta::InodeId &id, FormatContext &ctx) const {
    return fmt::format_to(ctx.out(), "0x{:016x}", id.u64());
  }
};

FMT_END_NAMESPACE

#pragma once
#include <arrow/io/file.h>
#include <parquet/stream_writer.h>
#include <type_traits>
#include <utility>
#include <variant>

#include "SerdeObjectVisitor.h"
#include "SerdeSchemaBuilder.h"
#include "common/serde/Serde.h"
#include "common/utils/Nameof.hpp"
#include "common/utils/StrongType.h"
#include "common/utils/UtcTime.h"

namespace hf3fs::analytics {

template <serde::SerdeType SerdeType>
class SerdeObjectWriter : public BaseObjectVisitor<SerdeObjectWriter<SerdeType>> {
 public:
  SerdeObjectWriter(parquet::StreamWriter &&writer)
      : writer_(std::move(writer)),
        createTime_(UtcClock::now()) {}

  static std::shared_ptr<SerdeObjectWriter> open(const Path path,
                                                 const bool append = false,
                                                 const size_t maxRowGroupLength = 1'000'000,
                                                 const std::vector<parquet::SortingColumn> &sortedColumns = {}) {
    // open file
    auto openStream = arrow::io::FileOutputStream::Open(path.string(), append);

    if (!openStream.ok()) {
      XLOGF(ERR, "Failed to open file output stream: {}, error: {}", path.string(), openStream.status().message());
      return nullptr;
    }

    std::shared_ptr<arrow::io::FileOutputStream> outfile;
    PARQUET_ASSIGN_OR_THROW(outfile, openStream);

    // generate schema
    SerdeSchemaBuilder<SerdeType> schemaBuilder;
    auto schemaNode = schemaBuilder.getSchema();

    if (schemaNode == nullptr) {
      return nullptr;
    }

    parquet::WriterProperties::Builder writerBuilder;
    writerBuilder.set_sorting_columns(sortedColumns);
    writerBuilder.max_row_group_length(maxRowGroupLength);
    writerBuilder.data_page_version(parquet::ParquetDataPageVersion::V2);

    // set global encoding and compression method
    // writerBuilder.encoding(parquet::Encoding::DELTA_BINARY_PACKED);
    writerBuilder.compression(parquet::Compression::ZSTD);

    // set encoding for string columns
    // for (int fieldIndex = 0; fieldIndex < schemaNode->field_count(); fieldIndex++) {
    //   auto fieldNode = schemaNode->field(fieldIndex);
    //   auto fieldType = fieldNode->logical_type();
    //   if (fieldType->is_string()) writerBuilder.encoding(fieldNode->name(),
    //   parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY);
    // }

    try {
      auto fileWriter = parquet::ParquetFileWriter::Open(outfile, schemaNode, writerBuilder.build());
      if (fileWriter == nullptr) {
        XLOGF(ERR, "Failed to open file writer: {}", path.string());
        return nullptr;
      }

      parquet::StreamWriter streamWriter(std::move(fileWriter));
      return std::make_shared<SerdeObjectWriter>(std::move(streamWriter));

    } catch (const std::exception &ex) {
      XLOGF(ERR, "Failed to create stream writer: {}, error: {}", path.string(), ex.what());
      return nullptr;
    }
  }

  SerdeObjectWriter &operator<<(const SerdeType &v) {
    if (!bool(*this)) return *this;

    try {
      visit("", v);
      writer_ << parquet::EndRow;
    } catch (const parquet::ParquetException &ex) {
      XLOGF(CRITICAL, "Failed to write to parquet file, error: {}", ex.what());
      isOk_ = false;
    }

    return *this;
  }

  void endRowGroup() { writer_.EndRowGroup(); }

  UtcTime createTime() { return createTime_; }

  operator bool() const { return ok(); }

  bool ok() const { return isOk_; }

 public:
  // default
  template <typename T>
  void visit(std::string_view k, const T &v) = delete;

  template <typename T>
  requires std::is_arithmetic_v<T>
  void visit(std::string_view k, const T &v) {
    XLOGF(DBG3, "arithmetic visit({})", k);
    writer_ << v;
  }

  template <typename T>
  requires std::is_enum_v<T>
  void visit(std::string_view k, const T &v) {
    XLOGF(DBG3, "enum visit({})", k);
    writer_ << (int32_t)v;
  }

  template <serde::ConvertibleToString T>
  void visit(std::string_view k, const T &v) {
    XLOGF(DBG3, "string visit({})", k);
    writer_ << v;
  }

  template <StrongTyped T>
  void visit(std::string_view k, const T &v) {
    XLOGF(DBG3, "strongtyped visit({})", k);
    visit<typename T::UnderlyingType>(k, v.toUnderType());
  }

  template <serde::WithReadableSerdeMethod T>
  void visit(std::string_view k, const T &val) {
    auto serialized = serde::SerdeMethod<T>::serdeToReadable(val);
    XLOGF(DBG3,
          "WithReadableSerdeMethod visit({}), serialized: {} {}",
          k,
          nameof::nameof_type<decltype(serialized)>(),
          serialized);
    visit<serde::SerdeToReadableReturnType<T>>(k, serialized);
  }

  template <serde::WithSerdeMethod T>
  void visit(std::string_view k, const T &val) {
    auto serialized = serde::SerdeMethod<T>::serdeTo(val);
    XLOGF(DBG3,
          "WithSerdeMethod visit({}), serialized: {} {}",
          k,
          nameof::nameof_type<decltype(serialized)>(),
          serialized);
    visit(k, serialized);
  }

  template <serde::WithReadableSerdeMemberMethod T>
  void visit(std::string_view k, const T &v) {
    auto serialized = v.serdeToReadable();
    XLOGF(DBG3,
          "WithReadableSerdeMemberMethod visit({}), serialized: {} {}",
          k,
          nameof::nameof_type<decltype(serialized)>(),
          serialized);
    visit(k, serialized);
  }

  template <serde::WithSerdeMemberMethod T>
  void visit(std::string_view k, const T &v) {
    auto serialized = v.serdeTo();
    XLOGF(DBG3,
          "WithSerdeMemberMethod visit({}), serialized: {} {}",
          k,
          nameof::nameof_type<decltype(serialized)>(),
          serialized);
    visit(k, serialized);
  }

  template <serde::SerdeTypeWithoutSpecializedSerdeMethod T>
  void visit(std::string_view k, const T &val) {
    XLOGF(DBG3, "serdetype visit({})", k);
    BaseObjectVisitor<SerdeObjectWriter>::visit(k, const_cast<T &>(val));
  }

  template <typename T>
  requires is_specialization_of_v<T, folly::Expected>
  void visit(std::string_view k, const T &val) {
    XLOGF(DBG3, "result visit({})", k);
    std::string errorColumnName = std::string{k} + kResultErrorTypeColumnSuffix;

    if (val.hasValue()) {
      Status ok(StatusCode::kOK);
      visit<typename T::error_type>(errorColumnName, ok);
      visit<typename T::value_type>(k, val.value());
    } else {
      typename T::value_type value{};
      visit<typename T::error_type>(errorColumnName, val.error());
      visit<typename T::value_type>(k, value);
    }
  }

  template <typename T>
  requires is_variant_v<T>
  void visit(std::string_view k, const T &val) {
    XLOGF(DBG3, "variant visit({})", k);
    std::string valIdxColumnName = std::string{k} + kVariantValueIndexColumnSuffix;
    visit<uint32_t>(valIdxColumnName, val.index());
    BaseObjectVisitor<SerdeObjectWriter>::visit(k, const_cast<T &>(val));
  }

  template <typename T>
  requires is_vector_v<T> || is_set_v<T>
  void visit(std::string_view k, const T &val) {
    XLOGF(DBG3, "container visit({})", k);
    auto str = serde::toJsonString(val);
    writer_ << str;
  }

  template <typename T>
  requires is_optional_v<T>
  void visit(std::string_view k, const T &val) {
    XLOGF(DBG3, "optional visit({})", k);
    if (!val.has_value()) {
      writer_ << "";
    } else {
      auto str = serde::toJsonString(*val);
      writer_ << str;
    }
  }

 private:
  parquet::StreamWriter writer_;
  UtcTime createTime_;
  bool isOk_{true};
};

template <typename T>
SerdeObjectWriter<T> &operator<<(SerdeObjectWriter<T> &writer, parquet::EndRowGroupType) {
  writer.endRowGroup();
  return writer;
}

}  // namespace hf3fs::analytics

#pragma once

#include <folly/Likely.h>
#include <memory>
#include <utility>

#include "common/utils/ObjectPool.h"

namespace hf3fs::net {

// A memory allocator with TLS cache support.
template <size_t DefaultSize = 1_KB, size_t NumTLSCachedItem = 1024, size_t NumGlobalCachedItem = 64 * 1024>
class Allocator {
 public:
  static constexpr auto kDefaultSize = DefaultSize;

  static std::uint8_t *allocate(size_t size) {
    if (LIKELY(size <= DefaultSize)) {
      return Pool::get().release()->data();
    }
    return new uint8_t[size];
  }

  static void deallocate(uint8_t *buf, size_t size) {
    if (LIKELY(size <= DefaultSize)) {
      auto autoRelease = typename Pool::Ptr{reinterpret_cast<Memory *>(buf)};
      return;
    }
    delete[] buf;
  }

 private:
  using Memory = std::array<uint8_t, DefaultSize>;
  using Pool = ObjectPool<Memory, NumTLSCachedItem, NumGlobalCachedItem>;
};

template <size_t DefaultSize = 1_KB>
class SystemAllocator {
 public:
  static constexpr auto kDefaultSize = DefaultSize;
  static std::uint8_t *allocate(size_t size) { return new uint8_t[size]; }
  static void deallocate(uint8_t *buf, size_t) { delete[] buf; }
};

}  // namespace hf3fs::net

#pragma once

#include <algorithm>
#include <array>
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <fmt/core.h>
#include <folly/Likely.h>
#include <folly/Random.h>
#include <folly/Utility.h>
#include <folly/lang/Bits.h>
#include <folly/logging/xlog.h>
#include <memory>
#include <optional>
#include <random>
#include <string_view>
#include <utility>
#include <vector>

#include "common/kv/IKVEngine.h"
#include "common/kv/ITransaction.h"
#include "common/kv/WithTransaction.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Result.h"
#include "common/utils/String.h"

namespace hf3fs {

/**
 * Generate 64bits unique id range from 1 to uint64_t::max.
 *
 * The uniqueness of generated ids is guaranteed by transaction. IdAllocator can use multiple keys in FoundationDB to
 * avoid transaction conflict, strictly monotonically increasing isn't guaranteed when numShard > 1.
 */
template <typename RetryStrategy>
class IdAllocator : folly::MoveOnly {
 public:
  IdAllocator(kv::IKVEngine &kvEngine, RetryStrategy strategy, String keyPrefix, uint8_t numShard)
      : kvEngine_(kvEngine),
        strategy_(strategy),
        keyPrefix_(std::move(keyPrefix)),
        numShard_(numShard),
        shardIdx_(0),
        shardList_() {
    XLOGF_IF(FATAL, numShard_ == 0, "Shared shouldn't be zero!!!");
    shardList_.reserve(numShard_);
    for (uint8_t i = 0; i < numShard_; i++) {
      shardList_.push_back(i);
    }
    std::shuffle(shardList_.begin(), shardList_.end(), std::mt19937_64(folly::Random::rand64()));
  }

  CoTryTask<uint64_t> allocate() {
    auto txn = kvEngine_.createReadWriteTransaction();
    auto result = co_await kv::WithTransaction<RetryStrategy>(strategy_).run(
        *txn,
        [&](kv::IReadWriteTransaction &txn) -> CoTryTask<uint64_t> { co_return co_await allocateTxn(txn); });

    XLOGF_IF(ERR, result.hasError(), "Failed to allocate ID, error {}", result.error());
    co_return result;
  }

  CoTryTask<std::vector<uint64_t>> status() {
    auto txn = kvEngine_.createReadonlyTransaction();
    auto result = co_await kv::WithTransaction<RetryStrategy>(strategy_).run(
        *txn,
        [&](kv::IReadOnlyTransaction &txn) -> CoTryTask<std::vector<uint64_t>> { co_return co_await statusTxn(txn); });

    XLOGF_IF(ERR, result.hasError(), "Failed to query status, error {}", result.error());
    co_return result;
  }

 private:
  std::string getShardKey(uint8_t shard) {
    XLOGF_IF(FATAL, shard > numShard_, "{} > {}", shard, numShard_);
    return fmt::format("{}-{}", keyPrefix_, shard);
  }

  Result<uint64_t> unpackShardValue(uint8_t shard, std::optional<String> value) {
    uint64_t minVal = shard == 0 ? 1 : 0;
    uint64_t val = minVal;
    if (value.has_value()) {
      if (UNLIKELY(value->size() != sizeof(uint64_t))) {
        XLOGF(CRITICAL,
              "Value of shard {} key {} has a unexpected length {} != 8.",
              shard,
              getShardKey(shard),
              value->size());
        return makeError(StatusCode::kDataCorruption);
      }
      val = folly::Endian::little64(folly::loadUnaligned<uint64_t>(value->data()));
      if (UNLIKELY(val < minVal)) {
        XLOGF(CRITICAL, "Value of shard {} key {} has a val {} < minVal {}.", shard, getShardKey(shard), val, minVal);
        return makeError(StatusCode::kDataCorruption);
      }
    }

    return val;
  }

  CoTryTask<uint64_t> allocateTxn(kv::IReadWriteTransaction &txn) {
    auto shardIdx = shardIdx_++;
    auto shard = shardList_[shardIdx % shardList_.size()];
    auto shardKey = getShardKey(shard);

    // load old value
    auto getResult = co_await txn.get(shardKey);
    CO_RETURN_ON_ERROR(getResult);
    auto unpackResult = unpackShardValue(shard, getResult.value());
    CO_RETURN_ON_ERROR(unpackResult);
    uint64_t val = unpackResult.value();

    // set new value
    auto bytes = folly::bit_cast<std::array<char, 8>>(folly::Endian::little64(val + 1));
    auto setResult = co_await txn.set(shardKey, std::string_view(bytes.begin(), bytes.size()));
    CO_RETURN_ON_ERROR(setResult);

    XLOGF(DBG, "IdAllocator allocate from shard {}, val {}, id {}\n", shard, val, val * numShard_ + shard);

    co_return (val * numShard_) + shard;
  }

  CoTryTask<std::vector<uint64_t>> statusTxn(kv::IReadOnlyTransaction &txn) {
    std::vector<uint64_t> vec;
    for (uint8_t shard = 0; shard < numShard_; shard++) {
      auto shardKey = getShardKey(shard);
      auto getResult = co_await txn.snapshotGet(shardKey);
      CO_RETURN_ON_ERROR(getResult);
      auto unpackResult = unpackShardValue(shard, getResult.value());
      CO_RETURN_ON_ERROR(unpackResult);
      vec.push_back(unpackResult.value());
    }

    co_return vec;
  }

  kv::IKVEngine &kvEngine_;
  RetryStrategy strategy_;
  String keyPrefix_;
  uint8_t numShard_;
  std::atomic<size_t> shardIdx_;
  std::vector<uint8_t> shardList_;
};

}  // namespace hf3fs

#pragma once

#include <folly/concurrency/AtomicSharedPtr.h>
#include <memory>
#include <ranges>
#include <shared_mutex>
#include <vector>

#include "common/utils/Result.h"
#include "common/utils/RobinHood.h"

namespace hf3fs::lib {
template <typename T, uint32_t KeyError, uint32_t OverflowError>
class PerProcTable {
 public:
  using Item = T;
  using ItemPtr = std::shared_ptr<Item>;

  PerProcTable() = delete;
  PerProcTable(int p, int pp, size_t cap)
      : pid_(p),
        ppid_(pp),
        table_(cap) {
    free_.reserve(cap);
  }
  PerProcTable(const PerProcTable &rhs, int p, int pp, size_t cap)
      : pid_(p),
        ppid_(pp) {
    if (ppid_ == rhs.pid_) {
      {
        std::shared_lock lock(rhs.mtx_);
        std::vector<AtomicItemPtr> t(std::max(rhs.table_.size(), cap));
        nextAvail_ = rhs.nextAvail_;
        for (int i = 0; i < nextAvail_; ++i) {
          t[i] = rhs.table_[i].load();
        }
        swap(t, table_);
        free_ = rhs.free_;
      }
      if (table_.size() < cap) {  // we don't shrink the table or items may drop out
        free_.reserve(cap);
      }
    } else {  // rhs is not our parent proc, so do not copy its fd table
      std::vector<AtomicItemPtr> t(cap);
      swap(t, table_);
    }
  }
  int pid() const { return pid_; }
  int ppid() const { return ppid_; }
  hf3fs::Result<ItemPtr> at(int idx, bool check = true) {
    if (idx < 0) {
      XLOGF(DBG, "bad index {}", idx);
      return makeError(KeyError,
                       fmt::format("invalid index {} to lookup in table for proc {} parent {}", idx, pid_, ppid_));
    } else {
      {
        //        std::shared_lock lock(mtx_);
        if (table_.size() < (size_t)idx) {
          XLOGF(DBG, "invalid index {}", idx);
          return makeError(KeyError, fmt::format("invalid index {} in table for proc {} parent {}", idx, pid_, ppid_));
        } else {
          auto p = table_[idx].load();
          if (check && !p) {
            XLOGF(DBG, "no item at index {}", idx);
            return makeError(KeyError,
                             fmt::format("index {} not found in table for proc {} parent {}", idx, pid_, ppid_));
          }

          return p;
        }
      }
    }
  }
  void setAt(int idx, const ItemPtr &v) {
    std::unique_lock lock(mtx_);
    auto p = table_[idx].load();
    XLOGF(DBG,
          "before set at idx {} table idx {} next avail {} free size {}",
          idx,
          (void *)p.get(),
          nextAvail_,
          free_.size());
    if (!p) {  // add to specific place
      if (idx >= nextAvail_) {
        for (auto i = nextAvail_; i < idx; ++i) {
          free_.push_back(i);
        }
        std::ranges::make_heap(free_, std::greater<int>());
        nextAvail_ = idx + 1;
      } else {
        if (free_.size() == 1) {
          assert(free_[0] == idx);
          free_.clear();
        } else {
          *std::ranges::find(free_, idx) = free_.back();
          free_.pop_back();
          std::ranges::make_heap(free_, std::greater<int>());
        }
      }
    }
    table_[idx] = v;
  }
  hf3fs::Result<int> add(const ItemPtr &v) {
    std::unique_lock lock(mtx_);
    auto useFree = !free_.empty();
    if (!useFree && (size_t)nextAvail_ >= table_.size()) {
      return makeError(OverflowError,
                       fmt::format("no free slot in table for proc {} parent {} table size {} next avail {}",
                                   pid_,
                                   ppid_,
                                   table_.size(),
                                   nextAvail_));
    }
    auto idx = useFree ? free_.front() : nextAvail_++;
    if (useFree) {
      std::ranges::pop_heap(free_, std::greater<int>());
      free_.pop_back();
    }
    auto p = table_[idx].load();

    XLOGF(DBG,
          "idx {} use free {} next avail {} free size {} table idx {}",
          idx,
          useFree,
          nextAvail_,
          free_.size(),
          (void *)p.get());
    assert(!p);
    table_[idx] = v;
    return idx;
  }
  void resetAt(int idx) {
    std::unique_lock lock(mtx_);
    table_[idx] = ItemPtr{};
    free_.push_back(idx);
    std::ranges::push_heap(free_, std::greater<int>());
    XLOGF(DBG, "reset idx {} next avail {} free size {} first free {}", idx, nextAvail_, free_.size(), free_.front());
  }
  size_t size() const {
    std::shared_lock lock(mtx_);
    return (size_t)nextAvail_ - free_.size();
  }
  std::vector<int> allUsed() const {
    std::vector<int> used;
    used.reserve(table_.size());

    std::shared_lock lock(mtx_);
    for (int i = 1; i < (int)table_.size(); ++i) {
      if (table_[i].load()) {
        used.push_back(i);
      }
    }

    return used;
  }
  std::unique_lock<std::shared_mutex> lock() const { return std::unique_lock<std::shared_mutex>(mtx_); }

 private:
  int pid_;
  int ppid_;
  mutable std::shared_mutex mtx_;
  using AtomicItemPtr = folly::atomic_shared_ptr<Item>;
  std::vector<AtomicItemPtr> table_;
  int nextAvail_ = 0;
  std::vector<int> free_;
};

template <typename T, uint32_t KeyError, uint32_t OverflowError>
class AllProcMap {
 public:
  using Table = PerProcTable<T, KeyError, OverflowError>;
  using TablePtr = std::shared_ptr<Table>;
  using ItemPtr = typename Table::ItemPtr;

  TablePtr procTable(int pid, int ppid, size_t cap) {
    TablePtr parentTable;
    {
      std::shared_lock lock(mtx_);
      auto it = map_.find(pid);
      if (it != map_.end()) {
        if (ppid == it->second->ppid()) {
          return it->second;
        }  // else, same pid but diff proc, old proc must have died
      } else {
        auto it = map_.find(ppid);
        if (it != map_.end()) {
          parentTable = it->second;
        }
      }
    }

    TablePtr newTable;
    if (parentTable) {
      newTable = std::make_shared<Table>(*parentTable, pid, ppid, cap);
    } else {
      newTable = std::make_shared<Table>(pid, ppid, cap);
    }

    std::unique_lock lock(mtx_);
    return map_[pid] = std::move(newTable);
  }

  void removeProc(int pid) {
    std::unique_lock lock(mtx_);
    map_.erase(pid);
  }

  std::vector<std::pair<int, int>> allProcs() const {
    std::vector<std::pair<int, int>> procs;
    std::shared_lock lock(mtx_);
    procs.reserve(map_.size());
    for (auto &&p : map_) {
      if (p.first != 0) {
        procs.emplace_back(std::make_pair(p.first, p.second->ppid()));
      }
    }

    return procs;
  }

 private:
  mutable std::shared_mutex mtx_;
  robin_hood::unordered_map<int, TablePtr> map_;
};
}  // namespace hf3fs::lib

#pragma once

#include <algorithm>
#include <atomic>
#include <cstddef>
#include <cstdint>
#include <folly/MPMCQueue.h>
#include <folly/Math.h>
#include <folly/Synchronized.h>
#include <folly/Utility.h>
#include <folly/executors/IOThreadPoolExecutor.h>
#include <folly/experimental/coro/Mutex.h>
#include <folly/fibers/Semaphore.h>
#include <folly/logging/xlog.h>
#include <memory>
#include <mutex>
#include <optional>
#include <string>
#include <sys/types.h>
#include <thread>
#include <utility>

#include "common/utils/BackgroundRunner.h"
#include "common/utils/CoroutinesPool.h"
#include "common/utils/Result.h"
#include "common/utils/Semaphore.h"
#include "common/utils/UtcTime.h"
#include "fbs/core/user/User.h"
#include "fbs/meta/Common.h"
#define FUSE_USE_VERSION 312
#define OP_LOG_LEVEL DBG

#include <folly/concurrency/AtomicSharedPtr.h>
#include <fuse3/fuse_lowlevel.h>

#include "FuseConfig.h"
#include "IoRing.h"
#include "IovTable.h"
#include "PioV.h"
#include "UserConfig.h"
#include "client/meta/MetaClient.h"
#include "client/mgmtd/MgmtdClientForClient.h"
#include "client/storage/StorageClient.h"
#include "fbs/meta/Schema.h"

namespace hf3fs::fuse {
using flat::Gid;
using flat::Uid;
using flat::UserInfo;
using lib::agent::PioV;
using meta::Acl;
using meta::Directory;
using meta::DirEntry;
using meta::Inode;
using meta::InodeData;
using meta::InodeId;
using meta::Permission;
using storage::client::IOBuffer;

struct InodeWriteBuf {
  std::vector<uint8_t> buf;
  std::unique_ptr<storage::client::IOBuffer> memh;
  off_t off{0};
  size_t len{0};
};

struct RcInode {
  struct DynamicAttr {
    uint64_t written = 0;
    uint64_t synced = 0;   // period sync
    uint64_t fsynced = 0;  // fsync, close, truncate, etc...
    flat::Uid writer = flat::Uid(0);

    uint32_t dynStripe = 1;  // dynamic stripe

    uint64_t truncateVer = 0;                         // largest known truncate version.
    std::optional<meta::VersionedLength> hintLength;  // local hint length
    std::optional<UtcTime> atime;                     // local read time, but only update for write open
    std::optional<UtcTime> mtime;                     // local write time

    void update(const Inode &inode, uint64_t syncver = 0, bool fsync = false) {
      if (!inode.isFile()) {
        return;
      }

      synced = std::max(synced, syncver);
      if (written == synced) {
        // clear local hint, since not write happens after sync
        hintLength = meta::VersionedLength{0, 0};
      }
      if (fsync) {
        fsynced = std::max(fsynced, syncver);
      }
      truncateVer = std::max(truncateVer, inode.asFile().truncateVer);
      dynStripe = inode.asFile().dynStripe;
    }
  };

  Inode inode;
  int refcount;
  std::atomic<int> opened;

  std::mutex wbMtx;
  std::shared_ptr<InodeWriteBuf> writeBuf;

  folly::Synchronized<DynamicAttr> dynamicAttr;
  folly::coro::Mutex extendStripeLock;

  RcInode(Inode inode, int refcount = 1)
      : inode(inode),
        refcount(refcount),
        extendStripeLock() {
    if (inode.isFile()) {
      auto guard = dynamicAttr.wlock();
      guard->truncateVer = inode.asFile().truncateVer;
      guard->hintLength = meta::VersionedLength{0, guard->truncateVer};
      guard->dynStripe = inode.asFile().dynStripe;
    }
  }

  uint64_t getTruncateVer() const { return dynamicAttr.rlock()->truncateVer; }

  void update(const Inode &inode, uint64_t syncver = 0, bool fsync = false) {
    if (!inode.isFile()) {
      return;
    } else {
      auto guard = dynamicAttr.wlock();
      return guard->update(inode, syncver, fsync);
    }
  }

  // clear hint length, force calculate length on next sync
  void clearHintLength() {
    auto guard = dynamicAttr.wlock();
    guard->hintLength = std::nullopt;
  }

  CoTryTask<uint64_t> beginWrite(flat::UserInfo userInfo,
                                 meta::client::MetaClient &meta,
                                 uint64_t offset,
                                 uint64_t length);

  void finishWrite(flat::UserInfo userInfo, uint64_t truncateVer, uint64_t offset, ssize_t ret);
};

struct FileHandle {
  std::shared_ptr<RcInode> rcinode;
  bool oDirect;
  Uuid sessionId;

  /* FileHandle(std::shared_ptr<RcInode> rcinode, bool oDirect, Uuid sessionId) */
  /*       : rcinode(rcinode), */
  /*         sessionId(sessionId) {} */
};

struct DirHandle {
  size_t dirId;
  pid_t pid;
  bool iovDir;
};

struct DirEntryVector {
  std::shared_ptr<std::vector<DirEntry>> dirEntries;

  DirEntryVector(std::shared_ptr<std::vector<DirEntry>> &&dirEntries)
      : dirEntries(std::move(dirEntries)) {}
};

struct DirEntryInodeVector {
  std::shared_ptr<std::vector<DirEntry>> dirEntries;
  std::shared_ptr<std::vector<std::optional<Inode>>> inodes;

  DirEntryInodeVector(std::shared_ptr<std::vector<DirEntry>> dirEntries,
                      std::shared_ptr<std::vector<std::optional<Inode>>> inodes)
      : dirEntries(std::move(dirEntries)),
        inodes(std::move(inodes)) {}
};

struct FuseClients {
  FuseClients() = default;
  ~FuseClients();

  Result<Void> init(const flat::AppInfo &appInfo,
                    const String &mountPoint,
                    const String &tokenFile,
                    FuseConfig &fuseConfig);
  void stop();

  CoTask<void> ioRingWorker(int i, int ths);
  void watch(int prio, std::stop_token stop);

  CoTask<void> periodicSyncScan();
  CoTask<void> periodicSync(InodeId inodeId);

  std::unique_ptr<net::Client> client;
  std::shared_ptr<client::MgmtdClientForClient> mgmtdClient;
  std::shared_ptr<storage::client::StorageClient> storageClient;
  std::shared_ptr<meta::client::MetaClient> metaClient;

  std::string fuseToken;
  std::string fuseMount;
  Path fuseMountpoint;
  std::optional<Path> fuseRemountPref;
  std::atomic<bool> memsetBeforeRead = false;
  int maxIdleThreads = 0;
  int maxThreads = 0;
  bool enableWritebackCache = false;

  std::unique_ptr<ConfigCallbackGuard> onFuseConfigUpdated;

  std::unordered_map<InodeId, std::shared_ptr<RcInode>> inodes = {
      {InodeId::root(), std::make_shared<RcInode>(Inode{}, 2)}};
  std::mutex inodesMutex;

  std::unordered_map<uint64_t, DirEntryInodeVector> readdirplusResults;
  std::mutex readdirplusResultsMutex;

  std::atomic_uint64_t dirHandle{0};

  std::shared_ptr<net::RDMABufPool> bufPool;
  int maxBufsize = 0;

  fuse_session *se = nullptr;

  std::atomic<std::chrono::nanoseconds> jitter;

  IovTable iovs;
  IoRingTable iors;
  std::vector<std::unique_ptr<BoundedQueue<IoRingJob>>> iojqs;  // job queues
  std::vector<std::jthread> ioWatches;
  folly::CancellationSource cancelIos;

  UserConfig userConfig;

  folly::Synchronized<std::set<InodeId>, std::mutex> dirtyInodes;
  std::atomic<InodeId> lastSynced;
  std::unique_ptr<BackgroundRunner> periodicSyncRunner;
  std::unique_ptr<CoroutinesPool<InodeId>> periodicSyncWorker;

  std::unique_ptr<folly::IOThreadPoolExecutor> notifyInvalExec;
  const FuseConfig *config;
};
}  // namespace hf3fs::fuse

#pragma once

#include <atomic>
#include <cassert>
#include <folly/Likely.h>
#include <folly/Preprocessor.h>
#include <folly/Random.h>
#include <folly/io/async/Request.h>
#include <memory>
#include <optional>

// Set fault injection parameter in current code scope and automatically restore after leaving the scope.
// Probability is expressed as a percentage between 0 and 100.
// Times determines the maximum number of faults that can be injected (-1 means unlimited).
// eg: `FAULT_INJECTION_SET(1, 5)` will inject faults with a probability of 1% up to five times.
#define FAULT_INJECTION_SET(prob, times) hf3fs::FaultInjection::ScopeGuard FB_ANONYMOUS_VARIABLE(guard)(prob, times)

// Fault injection code may be wrapped in loops, which may cause same fault injected many times. In this case, the
// effect of the loop can be reduced by setting a probability factor. The probability factor only works in the current
// scope and is automatically restored after leaving the scope.
#define FAULT_INJECTION_SET_FACTOR(factor) hf3fs::FaultInjectionFactor::ScopeGuard FB_ANONYMOUS_VARIABLE(guard)(factor)

// Use probability factor of current scope.
#define FAULT_INJECTION()                               \
  (UNLIKELY(hf3fs::FaultInjection::get() != nullptr) && \
   hf3fs::FaultInjection::get()->injectWithFactor(hf3fs::FaultInjectionFactor::get()))
// Specify a probability factor.
#define FAULT_INJECTION_WITH_FACTOR(factor) \
  (UNLIKELY(hf3fs::FaultInjection::get() != nullptr) && hf3fs::FaultInjection::get()->injectWithFactor(factor))

namespace hf3fs {

class FaultInjection : public folly::RequestData {
 public:
  class ScopeGuard {
   public:
    ScopeGuard(double prob, int times)
        : guard_(std::nullopt) {
      if (LIKELY(prob == 0)) {
        if (LIKELY(FaultInjection::get() == nullptr)) {
          // fast path, no need to record anything.
          return;
        }

        guard_.emplace(token(), nullptr);
        return;
      }
      guard_.emplace(token(),
                     std::unique_ptr<FaultInjection>(new FaultInjection(prob / 100 * kProbabilityBase, times)));
    }

   private:
    std::optional<folly::ShallowCopyRequestContextScopeGuard> guard_;
  };

  static ScopeGuard clone() {
    auto fi = FaultInjection::get();
    return fi ? ScopeGuard((double)fi->probability_ / kProbabilityBase * 100, fi->times_) : ScopeGuard(0, 0);
  }

  static inline FaultInjection *get() {
    auto requestContext = folly::RequestContext::try_get();
    if (LIKELY(requestContext == nullptr)) {
      return nullptr;
    }
    return dynamic_cast<FaultInjection *>(requestContext->getContextData(token()));
  }

  bool hasCallback() override { return false; }

  bool injectWithFactor(uint32_t factor) {
    bool inject = folly::Random::rand32(kProbabilityBase) < probability_ && folly::Random::oneIn(factor);
    if (LIKELY(!inject)) {
      return false;
    }
    if (times_ == -1) {
      return true;
    }
    if (injected_ >= times_) return false;
    return injected_.fetch_add(1) < times_;
  }

  /** Get fault injection percentage probability. */
  double getProbability() const { return (double)probability_ * 100.0 / kProbabilityBase; }

 private:
  static constexpr const char *kTokenName = "hf3fs::FaultInjection";
  static constexpr uint32_t kProbabilityBase = 1000000;

  static folly::RequestToken const &token() {
    static folly::RequestToken const token(kTokenName);
    return token;
  }

  explicit FaultInjection(uint32_t prob, int times)
      : probability_(prob),
        times_(times),
        injected_(0) {
    probability_ = std::min(probability_, (uint32_t)kProbabilityBase);
  }

  uint32_t probability_;          // probability to trigger fault injection, between 0 and kProbabilityBase
  int32_t times_;                 // max fault injection times, -1 means unlimited
  std::atomic_int32_t injected_;  // injected times, use std::atomic for thread safety.
};

// This is read only, so it's thread safe as required by folly::RequestData.
class FaultInjectionFactor : public folly::RequestData {
 public:
  class ScopeGuard {
   public:
    explicit ScopeGuard(uint32_t factor)
        : guard_(std::nullopt) {
      if (LIKELY(FaultInjection::get() == nullptr)) {
        // fast path, fault injection not enabled.
        return;
      }
      guard_.emplace(token(), std::unique_ptr<FaultInjectionFactor>(new FaultInjectionFactor(factor)));
    }

   private:
    std::optional<folly::ShallowCopyRequestContextScopeGuard> guard_;
  };

  static inline uint32_t get() {
    auto requestContext = folly::RequestContext::try_get();
    if (LIKELY(requestContext == nullptr)) {
      return 1;
    }
    auto ptr = dynamic_cast<FaultInjectionFactor *>(requestContext->getContextData(token()));
    return ptr ? ptr->factor_ : 1;
  }

  bool hasCallback() override { return false; }

  uint32_t getFactor() const { return factor_; }

 private:
  static constexpr const char *kTokenName = "hf3fs::FaultInjectionFactor";
  static folly::RequestToken const &token() {
    static folly::RequestToken const token(kTokenName);
    return token;
  }

  FaultInjectionFactor(uint32_t factor)
      : factor_(factor) {}
  uint32_t factor_;
};
}  // namespace hf3fs

#pragma once

#include "common/serde/MessagePacket.h"
#include "common/serde/Serde.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Reflection.h"

namespace hf3fs::net {
struct UserRequestOptions;
}

namespace hf3fs::serde {

template <NameWrapper NAME, uint32_t ID>
struct ServiceBase {
  static constexpr auto kServiceNameWrapper = NAME;
  static constexpr std::string_view kServiceName = NAME;
  static constexpr uint16_t kServiceID = ID;
};

template <NameWrapper NAME, class T, class REQ, class RSP, uint16_t ID, auto METHOD>
struct MethodInfo {
  static constexpr auto nameWrapper = NAME;
  static constexpr std::string_view name = NAME;
  using Object = T;
  using ReqType = REQ;
  using RspType = RSP;
  static constexpr auto id = ID;
  static constexpr auto method = METHOD;
};

template <class T>
constexpr inline uint16_t MaxMethodId = 0;
template <class... T>
constexpr inline uint16_t MaxMethodId<std::tuple<T...>> = std::max({T::id...});

template <class T, template <class> class Service>
struct ServiceWrapper {
  static constexpr std::string_view kServiceName = Service<void>::kServiceName;
  static constexpr uint16_t kServiceID = Service<void>::kServiceID;

 protected:
  friend struct ::hf3fs::refl::Helper;
  template <class U = T>
  static auto CollectField(::hf3fs::refl::Rank<> rank) -> refl::Helper::FieldInfoList<Service<U>>;
};

template <class T, class C, auto DEFAULT = nullptr>
class MethodExtractor {
 public:
  static auto get(uint16_t id) {
    constexpr MethodExtractor ins;
    return id <= ins.kMaxThreadId ? ins.table[id] : DEFAULT;
  }

 protected:
  consteval MethodExtractor() {
    for (uint16_t i = 0; i <= kMaxThreadId; ++i) {
      table[i] = calc(i);
    }
  }

  template <size_t I = 0>
  consteval auto calc(uint16_t id) {
    if constexpr (I == std::tuple_size_v<FieldInfoList>) {
      return DEFAULT;
    } else {
      using FieldInfo = std::tuple_element_t<I, FieldInfoList>;
      return FieldInfo::id == id ? &C::template call<FieldInfo> : calc<I + 1>(id);
    }
  }

 private:
  using FieldInfoList = refl::Helper::FieldInfoList<T>;
  using Method = decltype(&C::template call<std::tuple_element_t<0, FieldInfoList>>);
  static constexpr auto kMaxThreadId = MaxMethodId<FieldInfoList>;
  std::array<Method, kMaxThreadId + 1> table;
};

#define SERDE_SERVICE(NAME, ID) SERDE_SERVICE_2(NAME, NAME, ID)

#define SERDE_SERVICE_2(STRUCT_NAME, SERVICE_NAME, ID) \
  template <class T = void>                            \
  struct STRUCT_NAME : public ::hf3fs::serde::ServiceBase<#SERVICE_NAME, ID>

#define SERDE_SERVICE_METHOD(NAME, ID, REQ, RSP)  \
  SERDE_SERVICE_METHOD_SENDER(NAME, ID, REQ, RSP) \
  SERDE_SERVICE_METHOD_REFL(NAME, ID, REQ, RSP)

#define SERDE_SERVICE_METHOD_SENDER(NAME, ID, REQ, RSP)                                                    \
 public:                                                                                \
  static constexpr auto NAME##MethodId = ID;                                                               \
                                                                                \
  template <class Context>                                                                                \
  static CoTryTask<RSP> NAME(Context &ctx,                                                                 \
                             const REQ &req,                                                               \
                             const ::hf3fs::net::UserRequestOptions *options = nullptr,                    \
                             ::hf3fs::serde::Timestamp * timestamp = nullptr) {                            \
    co_return co_await ctx.template call<kServiceNameWrapper, #NAME, REQ, RSP, kServiceID, ID>(req,        \
                                                                                options,    \
                                                                                timestamp); \
  }                                                                                \
                                                                                \
  template <class Context>                                                                                \
  static Result<RSP> NAME##Sync(Context &ctx,                                                              \
                                const REQ &req,                                                            \
                                const ::hf3fs::net::UserRequestOptions *options = nullptr,                 \
                                ::hf3fs::serde::Timestamp * timestamp = nullptr) {                         \
    return ctx.template callSync<kServiceNameWrapper, #NAME "Sync", REQ, RSP, kServiceID, ID>(req,         \
                                                                                options,     \
                                                                                timestamp);  \
  }

#define SERDE_SERVICE_METHOD_REFL(NAME, ID, REQ, RSP)                                                 \
 private:                                                                                \
  struct MethodId##ID : std::type_identity<REFL_NOW> {};                                              \
  static auto CollectField(::hf3fs::refl::Rank<std::tuple_size_v<typename MethodId##ID::type> + 1>) { \
    if constexpr (std::is_void_v<T>) {                                                                \
      return ::hf3fs::refl::Append_t<typename MethodId##ID::type,                                     \
                                     ::hf3fs::serde::MethodInfo<#NAME, T, REQ, RSP, ID, nullptr>>{};  \
    } else {                                                                                \
      return ::hf3fs::refl::Append_t<typename MethodId##ID::type,                                     \
                                     ::hf3fs::serde::MethodInfo<#NAME, T, REQ, RSP, ID, &T::NAME>>{}; \
    }                                                                                \
  }                                                                                \
  friend struct ::hf3fs::refl::Helper

#define SERDE_SERVICE_CLIENT(CLIENT_NAME, BASE_NAME)                                                  \
  template <class T = void>                                                                           \
  struct CLIENT_NAME {                                                                                \
    static constexpr auto kServiceName = BASE_NAME<T>::kServiceName;                                  \
    static constexpr auto kServiceID = BASE_NAME<T>::kServiceID;                                      \
                                                                                \
    template <NameWrapper name, typename Context, typename Req>                                       \
    static auto send(Context &ctx,                                                                    \
                     const Req &req,                                                                  \
                     const ::hf3fs::net::UserRequestOptions *options = nullptr,                       \
                     ::hf3fs::serde::Timestamp *timestamp = nullptr) {                                \
      auto handler = [&](auto type) requires(decltype(type)::name == name &&                          \
                                             std::is_same_v<Req, typename decltype(type)::ReqType>) { \
        using M = std::decay_t<decltype(type)>;                                                       \
        return ctx.template call<BASE_NAME<T>::kServiceNameWrapper,                                   \
                                 name,                                                                \
                                 typename M::ReqType,                                                 \
                                 typename M::RspType,                                                 \
                                 BASE_NAME<T>::kServiceID,                                            \
                                 M::id>(req, options, timestamp);                                     \
      };                                                                                \
      return ::hf3fs::refl::Helper::visit<BASE_NAME<T>>(std::move(handler));                          \
    }                                                                                \
  }

}  // namespace hf3fs::serde

#pragma once

#include <atomic>
#include <folly/Memory.h>
#include <folly/ThreadLocal.h>
#include <folly/concurrency/ConcurrentHashMap.h>
#include <folly/container/HeterogeneousAccess.h>
#include <folly/stats/DigestBuilder.h>
#include <folly/stats/TDigest.h>
#include <functional>
#include <memory>
#include <mutex>
#include <optional>
#include <string>
#include <string_view>
#include <utility>
#include <vector>

#include "DigestBuilder.h"
#include "common/monitor/Sample.h"
#include "common/utils/Address.h"
#include "common/utils/Duration.h"
#include "common/utils/RobinHood.h"
#include "common/utils/Size.h"
#include "common/utils/UtcTime.h"

namespace hf3fs::monitor {

class MonitorInstance;
class Collector;

class Recorder {
 public:
  Recorder(std::string_view name, std::optional<TagSet> tag, MonitorInstance &monitor)
      : register_(false),
        monitor_(monitor) {
    if (tag.has_value()) {
      name_ = name;
      tag_ = tag.value();
    } else {
      name_ = name;
    }
  }
  virtual ~Recorder();

  const std::string &name() { return name_; }

  virtual void collectAndClean(std::vector<Sample> &samples, bool cleanInactive);

  virtual void collect(std::vector<Sample> &samples) = 0;

  static void setHostname(std::string hostname, std::string podname);

 public:
  using ConcurrentMap = folly::ConcurrentHashMap<TagSet,
                                                 std::unique_ptr<Recorder>,
                                                 folly::HeterogeneousAccessHash<TagSet>,
                                                 folly::HeterogeneousAccessEqualTo<TagSet>>;

  template <typename T>
  class TagRef : public ConcurrentMap::ConstIterator {
   public:
    TagRef(ConcurrentMap::ConstIterator &&iter)
        : ConcurrentMap::ConstIterator(std::move(iter)) {}

    T *operator->() const {
      auto ptr = ConcurrentMap::ConstIterator::operator->()->second.get();
      return static_cast<T *>(ptr);
    }
  };

  // Expose recorder associated with a tag set
  template <typename T>
  TagRef<T> getRecorderWithTag(const TagSet &tag);

 protected:
  friend class Collector;
  friend bool checkRecorderHasTag(const Recorder &rec, const TagSet &tag);

 protected:
  void registerRecorder();
  void unregisterRecorder();

 protected:
  std::string name_;
  TagSet tag_;
  bool register_;
  bool logPer30s_ = true;
  UtcTime lastLogTime_;
  bool active_ = true;

  ConcurrentMap map_;
  MonitorInstance &monitor_;
};

template <class ThreadLocalTag>
class CountRecorderWithTLSTag final : public Recorder {
 public:
  CountRecorderWithTLSTag(std::string_view name, bool resetWhenCollect)
      : CountRecorderWithTLSTag(name, std::nullopt, resetWhenCollect) {}

  CountRecorderWithTLSTag(std::string_view name,
                          std::optional<TagSet> tag = std::nullopt,
                          bool resetWhenCollect = true);
  CountRecorderWithTLSTag(MonitorInstance &monitor,
                          std::string_view name,
                          std::optional<TagSet> tag = std::nullopt,
                          bool resetWhenCollect = true)
      : CountRecorderWithTLSTag(monitor, name, tag, resetWhenCollect, true) {}
  CountRecorderWithTLSTag(CountRecorderWithTLSTag &parent, const TagSet &tag)
      : CountRecorderWithTLSTag(parent.monitor_, parent.name(), tag, parent.resetWhenCollect_, false) {}
  ~CountRecorderWithTLSTag() final { unregisterRecorder(); }

  void collectAndClean(std::vector<Sample> &samples, bool cleanInactive) final;
  void collect(std::vector<Sample> &samples) final;
  void addSample(int64_t val) { tls_->addSample(val); }
  void addSample(int64_t val, const TagSet &tag);

  TagRef<CountRecorderWithTLSTag> getRecorderWithTag(const TagSet &tag) {
    return Recorder::getRecorderWithTag<CountRecorderWithTLSTag>(tag);
  }

 private:
  CountRecorderWithTLSTag(MonitorInstance &monitor,
                          std::string_view name,
                          std::optional<TagSet> tag,
                          bool resetWhenCollect,
                          bool needRegister)
      : Recorder(name, tag, monitor),
        resetWhenCollect_(resetWhenCollect) {
    if (needRegister) {
      registerRecorder();
    }
  }

  struct TLS {
    TLS(CountRecorderWithTLSTag *parent)
        : parent_(parent) {}
    ~TLS() { parent_->sum_ += exchange(); }
    void addSample(int64_t val) { val_ += val; }
    int64_t exchange() { return val_.exchange(0); }
    int64_t load() { return val_.load(std::memory_order_acquire); }

   private:
    CountRecorderWithTLSTag *parent_ = nullptr;
    std::atomic<int64_t> val_{0};
  };

  std::atomic<int64_t> sum_{0};
  folly::ThreadLocal<TLS, ThreadLocalTag> tls_{[this] { return new TLS(this); }};

  bool resetWhenCollect_;
  int64_t cumulativeVal_ = 0;
};

struct SharedThreadLocalTag {};
struct AllocatedMemoryCounterTag {};

using CountRecorder = CountRecorderWithTLSTag<SharedThreadLocalTag>;

class DistributionRecorder : public Recorder {
 public:
  DistributionRecorder(std::string_view name, std::optional<TagSet> tag = std::nullopt);
  DistributionRecorder(MonitorInstance &monitor, std::string_view name, std::optional<TagSet> tag = std::nullopt)
      : DistributionRecorder(monitor, name, tag, true) {}
  DistributionRecorder(DistributionRecorder &parent, const TagSet &tag)
      : DistributionRecorder(parent.monitor_, parent.name(), tag, false) {}
  ~DistributionRecorder() override { unregisterRecorder(); }

  void collect(std::vector<Sample> &samples) override;
  void addSample(double value) { tdigest_.append(value); }
  void addSample(double val, const TagSet &tag);
  virtual void logPer30s(const folly::TDigest &digest);

  TagRef<DistributionRecorder> getRecorderWithTag(const TagSet &tag) {
    return Recorder::getRecorderWithTag<DistributionRecorder>(tag);
  }

 protected:
  DistributionRecorder(MonitorInstance &monitor, std::string_view name, std::optional<TagSet> tag, bool needRegister)
      : Recorder(name, tag, monitor) {
    if (needRegister) {
      registerRecorder();
    }
  }

  static constexpr size_t kDigestMaxSize = 512;
  static constexpr size_t kDigestBufferSize = 128_KB;
  folly::DigestBuilder<folly::TDigest> tdigest_{kDigestBufferSize, kDigestMaxSize};
  std::vector<folly::TDigest> cumulativeDigests_;
};

class LatencyRecorder : public DistributionRecorder {
 public:
  LatencyRecorder(std::string_view name, std::optional<TagSet> tag = std::nullopt);
  LatencyRecorder(LatencyRecorder &parent, const TagSet &tag);
  ~LatencyRecorder() override { unregisterRecorder(); }

  void addSample(double value) = delete;
  void addSample(uint64_t val, const TagSet &tag) = delete;

  void addSample(std::chrono::nanoseconds duration) { tdigest_.append(duration.count()); }
  void addSample(std::chrono::nanoseconds duration, const TagSet &tag);
  void logPer30s(const folly::TDigest &digest) override;

  TagRef<LatencyRecorder> getRecorderWithTag(const TagSet &tag) {
    return Recorder::getRecorderWithTag<LatencyRecorder>(tag);
  }
};

class SimpleDistributionRecorder : public Recorder {
 public:
  SimpleDistributionRecorder(std::string_view name, std::optional<TagSet> tag = std::nullopt);
  SimpleDistributionRecorder(MonitorInstance &monitor, std::string_view name, std::optional<TagSet> tag = std::nullopt)
      : SimpleDistributionRecorder(monitor, name, tag, true) {}
  SimpleDistributionRecorder(SimpleDistributionRecorder &parent, const TagSet &tag)
      : SimpleDistributionRecorder(parent.monitor_, parent.name(), tag, false) {}
  ~SimpleDistributionRecorder() override { unregisterRecorder(); }

  void collect(std::vector<Sample> &samples) override;
  void addSample(int64_t val) { tls_->addSample(val); }
  void addSample(int64_t val, const TagSet &tag);

  void addSample(std::chrono::nanoseconds duration) { addSample(duration.count()); }
  void addSample(std::chrono::nanoseconds duration, const TagSet &tag) { addSample(duration.count(), tag); }

  TagRef<SimpleDistributionRecorder> getRecorderWithTag(const TagSet &tag) {
    return Recorder::getRecorderWithTag<SimpleDistributionRecorder>(tag);
  }

 protected:
  SimpleDistributionRecorder(MonitorInstance &monitor,
                             std::string_view name,
                             std::optional<TagSet> tag,
                             bool needRegister)
      : Recorder(name, tag, monitor) {
    if (needRegister) {
      registerRecorder();
    }
  }

  struct Tls {
    Tls(SimpleDistributionRecorder *parent)
        : parent_(parent) {}

    ~Tls() {
      parent_->sum_.fetch_add(exchangeSum(), std::memory_order_relaxed);
      parent_->count_.fetch_add(exchangeCount(), std::memory_order_relaxed);
      parent_->min_.store(std::min(parent_->min_.load(std::memory_order_relaxed), exchangeMin()),
                          std::memory_order_relaxed);
      parent_->max_.store(std::max(parent_->max_.load(std::memory_order_relaxed), exchangeMax()),
                          std::memory_order_relaxed);
    }

    void addSample(int64_t val) {
      sum_.fetch_add(val, std::memory_order_relaxed);
      count_.fetch_add(1, std::memory_order_relaxed);
      min_.store(std::min(min_.load(std::memory_order_relaxed), val), std::memory_order_relaxed);
      max_.store(std::max(max_.load(std::memory_order_relaxed), val), std::memory_order_relaxed);
    }

    int64_t exchangeSum() { return sum_.exchange(0); }
    int64_t exchangeCount() { return count_.exchange(0); }
    int64_t exchangeMin() { return min_.exchange(std::numeric_limits<int64_t>::max()); }
    int64_t exchangeMax() { return max_.exchange(0); }

   private:
    SimpleDistributionRecorder *parent_ = nullptr;
    std::atomic<int64_t> sum_{0};
    std::atomic<int64_t> count_{0};
    std::atomic<int64_t> min_{std::numeric_limits<int64_t>::max()};
    std::atomic<int64_t> max_{0};
  };

  struct TlsTag {};

  std::atomic<int64_t> sum_{0};
  std::atomic<int64_t> count_{0};
  std::atomic<int64_t> min_{std::numeric_limits<int64_t>::max()};
  std::atomic<int64_t> max_{0};
  folly::ThreadLocal<Tls, TlsTag> tls_{[this] { return new Tls(this); }};
};

template <typename LatencyRecorderT>
class OperationRecorderT {
 public:
  OperationRecorderT(std::string_view name, std::optional<TagSet> tag = std::nullopt, bool recordErrorCode = false);

  struct Guard {
    explicit Guard(OperationRecorderT &recorder)
        : recorder_(recorder) {}
    Guard(OperationRecorderT &recorder, const TagSet &tags)
        : recorder_(recorder),
          tags_(tags) {}
    ~Guard() { report(false); }

    void reportWithCode(status_code_t code);
    void report(bool success) {
      if (success) {
        reportWithCode(StatusCode::kOK);
      } else {
        reportWithCode(StatusCode::kUnknown);
      }
    }
    void succ() { success_ = true; }
    std::optional<Duration> latency() const { return latency_; }
    Duration ellipse() const { return RelativeTime::now() - startTime_; }
    void dismiss() {
      if (!std::exchange(reported_, true)) {
        if (tags_.has_value()) {
          recorder_.current_.addSample(-1, *tags_);
        } else {
          recorder_.current_.addSample(-1);
        }
      }
    }
    RelativeTime startTime() const { return startTime_; }

   private:
    OperationRecorderT &recorder_;
    RelativeTime startTime_ = RelativeTime::now();
    std::optional<TagSet> tags_;
    bool success_ = false;
    bool reported_ = false;
    std::optional<Duration> latency_;
  };

  [[nodiscard]] Guard record() {
    total_.addSample(1);
    current_.addSample(1);
    return Guard(*this);
  }
  [[nodiscard]] Guard record(const TagSet &tag) {
    total_.addSample(1, tag);
    current_.addSample(1, tag);
    return Guard(*this, tag);
  }

 private:
  CountRecorder total_;
  CountRecorder fails_;
  CountRecorder current_;
  LatencyRecorderT succ_latencies_;
  LatencyRecorderT fail_latencies_;
  bool recordErrorCode_;
};

using OperationRecorder = OperationRecorderT<LatencyRecorder>;
using SimpleOperationRecorder = OperationRecorderT<SimpleDistributionRecorder>;

class ValueRecorder : public Recorder {
 public:
  ValueRecorder(std::string_view name, std::optional<TagSet> tag = std::nullopt, bool resetWhenCollect = true);
  ValueRecorder(MonitorInstance &monitor,
                std::string_view name,
                std::optional<TagSet> tag = std::nullopt,
                bool resetWhenCollect = true)
      : ValueRecorder(monitor, name, tag, true, resetWhenCollect) {}
  ValueRecorder(ValueRecorder &parent, const TagSet &tag)
      : ValueRecorder(parent.monitor_, parent.name(), tag, false, parent.resetWhenCollect_) {}
  ~ValueRecorder() override { unregisterRecorder(); }

  void collectAndClean(std::vector<Sample> &samples, bool cleanInactive) override;
  void collect(std::vector<Sample> &samples) override;
  void set(int64_t val) { val_.store(val, std::memory_order_release); }
  void set(int64_t val, const TagSet &tag);
  auto value() { return val_.load(); }

  TagRef<ValueRecorder> getRecorderWithTag(const TagSet &tag) { return Recorder::getRecorderWithTag<ValueRecorder>(tag); }

 private:
  ValueRecorder(MonitorInstance &monitor,
                std::string_view name,
                std::optional<TagSet> tag,
                bool needRegister,
                bool resetWhenCollect)
      : Recorder(name, tag, monitor),
        resetWhenCollect_(resetWhenCollect) {
    if (needRegister) {
      registerRecorder();
    }
  }

  std::atomic<int64_t> val_{0};
  bool resetWhenCollect_;
};

class LambdaRecorder : public Recorder {
 public:
  LambdaRecorder(std::string_view name, std::optional<TagSet> tag = std::nullopt);
  ~LambdaRecorder() override {
    unregisterRecorder();
    reset();
  }

  void setLambda(auto &&f) {
    auto lock = std::unique_lock(mutex_);
    getter_ = std::forward<decltype(f)>(f);
  }

  void reset() {
    auto lock = std::unique_lock(mutex_);
    getter_ = [] { return 0; };
  }

  void collect(std::vector<Sample> &samples) override;

 private:
  std::mutex mutex_;
  std::function<int64_t()> getter_ = [] { return 0; };
};

}  // namespace hf3fs::monitor

#pragma once

#include <algorithm>
#include <atomic>
#include <cassert>
#include <cerrno>
#include <cstddef>
#include <cstdint>
#include <fmt/core.h>
#include <fmt/format.h>
#include <folly/IPAddressV4.h>
#include <folly/Likely.h>
#include <folly/Synchronized.h>
#include <folly/Utility.h>
#include <folly/detail/IPAddressSource.h>
#include <folly/logging/xlog.h>
#include <gtest/gtest_prod.h>
#include <infiniband/verbs.h>
#include <map>
#include <memory>
#include <mutex>
#include <optional>
#include <queue>
#include <set>
#include <shared_mutex>
#include <span>
#include <stdexcept>
#include <string>
#include <string_view>
#include <utility>
#include <vector>

#include "common/net/EventLoop.h"
#include "common/net/IfAddrs.h"
#include "common/utils/Address.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/Duration.h"
#include "common/utils/MagicEnum.hpp"
#include "common/utils/Result.h"
#include "common/utils/StrongType.h"

namespace hf3fs::net {

class TestIBNotInitialized;
class TestIBDevice;

class IBConfig : public ConfigBase<IBConfig> {
 public:
  static constexpr std::string_view kUnknownZone = "UNKNOWN";
  class Network : public folly::CIDRNetworkV4 {
   public:
    static Result<Network> from(std::string_view str);
    Network(folly::CIDRNetworkV4 network)
        : folly::CIDRNetworkV4(network) {}
    folly::IPAddressV4 ip() const { return first; }
    uint8_t mask() const { return second; }
    std::string toString() const { return fmt::format("{}/{}", ip().str(), mask()); }
  };

  class Subnet : public ConfigBase<Subnet> {
    CONFIG_ITEM(subnet, Network({}));
    CONFIG_ITEM(network_zones, std::vector<std::string>{std::string(kUnknownZone)}, [](auto &v) { return !v.empty(); });
  };

  CONFIG_ITEM(allow_no_usable_devices, false);
  CONFIG_ITEM(skip_inactive_ports, true);
  CONFIG_ITEM(skip_unusable_device, true);

  CONFIG_ITEM(allow_unknown_zone, true);
  CONFIG_ITEM(device_filter, std::vector<std::string>());
  CONFIG_ITEM(default_network_zone, std::string(kUnknownZone), [](auto &v) { return !v.empty(); });
  CONFIG_ITEM(subnets, std::vector<Subnet>());
  CONFIG_ITEM(fork_safe, true);
  CONFIG_ITEM(default_pkey_index, uint16_t(0));
  CONFIG_ITEM(default_roce_pkey_index, uint16_t(0));  // for RoCE, should just use default value 0
  // CONFIG_ITEM(default_gid_index, uint8_t(0));      // for RoCE
  CONFIG_ITEM(default_traffic_class, uint8_t(0));  // for RoCE
  CONFIG_ITEM(prefer_ibdevice, true);              // prefer use ibdevice instead of RoCE device.
};

class IBPort;
class IBManager;

class IBDevice : public std::enable_shared_from_this<IBDevice> {
 public:
  using Ptr = std::shared_ptr<IBDevice>;
  using Config = IBConfig;
  using IB2NetMap = std::map<std::pair<std::string, uint8_t>, std::string>;

  struct Port {
    std::vector<IfAddrs::Addr> addrs;
    std::set<std::string> zones;
    mutable folly::Synchronized<ibv_port_attr> attr;
  };

  struct Deleter {
    void operator()(ibv_pd *pd) const {
      auto ret = ibv_dealloc_pd(pd);
      XLOGF_IF(CRITICAL, ret != 0, "ibv_dealloc_pd failed {}", ret);
    }
    void operator()(ibv_context *context) const {
      auto ret = ibv_close_device(context);
      XLOGF_IF(CRITICAL, ret != 0, "ibv_close_device failed {}", ret);
    }
    void operator()(ibv_cq *cq) const {
      auto ret = ibv_destroy_cq(cq);
      XLOGF_IF(CRITICAL, ret != 0, "ibv_destroy_cq failed {}", ret);
    }
    void operator()(ibv_qp *qp) const {
      auto ret = ibv_destroy_qp(qp);
      XLOGF_IF(CRITICAL, ret != 0, "ibv_destroy_qp failed {}", ret);
    }
    void operator()(ibv_comp_channel *channel) const {
      auto ret = ibv_destroy_comp_channel(channel);
      XLOGF_IF(CRITICAL, ret != 0, "ibv_destroy_comp_channel failed {}", ret);
    }
  };

  static constexpr size_t kMaxDeviceCnt = 4;
  static const std::vector<IBDevice::Ptr> &all();
  static IBDevice::Ptr get(uint8_t devId) {
    if (all().size() < devId) {
      return nullptr;
    }
    return all().at(devId);
  }

  uint8_t id() const { return devId_; }
  const std::string &name() const { return name_; }
  const ibv_device_attr &attr() const { return attr_; }
  ibv_context *context() const { return context_.get(); }
  ibv_pd *pd() const { return pd_.get(); }
  const std::map<uint8_t, Port> &ports() const { return ports_; }
  Result<IBPort> openPort(size_t portNum) const;

  ibv_mr *regMemory(void *addr, size_t length, int access) const;
  int deregMemory(ibv_mr *mr) const;

 private:
  friend class IBManager;
  class BackgroundRunner;
  class AsyncEventHandler;

  static Result<std::vector<IBDevice::Ptr>> openAll(const IBConfig &config);
  static Result<IBDevice::Ptr> open(ibv_device *dev,
                                    uint8_t devId,
                                    std::map<std::pair<std::string, uint8_t>, std::string> ib2net,
                                    std::multimap<std::string, IfAddrs::Addr> ifaddrs,
                                    const IBConfig &config);

  FRIEND_TEST(TestIBDevice, IBConfig);
  static std::vector<IfAddrs::Addr> getIBPortAddrs(const IB2NetMap &ibdev2netdev,
                                                   const IfAddrs::Map &ifaddrs,
                                                   const std::string &devName,
                                                   uint8_t portNum);
  static std::set<std::string> getZonesByAddrs(std::vector<IfAddrs::Addr> addrs,
                                               const IBConfig &config,
                                               const std::string &devName,
                                               uint8_t portNum);

  void checkAsyncEvent() const;
  Result<Void> updatePort(size_t portNum) const;

  uint8_t devId_ = 0;
  std::string name_;
  std::unique_ptr<ibv_context, Deleter> context_;
  std::unique_ptr<ibv_pd, Deleter> pd_;
  ibv_device_attr attr_;
  std::map<uint8_t, Port> ports_;
};

class IBPort {
 public:
  IBPort(std::shared_ptr<const IBDevice> dev = {},
         uint8_t portNum = 0,
         ibv_port_attr attr = {},
         std::optional<std::pair<ibv_gid, uint8_t>> rocev2Gid = {});

  uint8_t portNum() const { return portNum_; }
  const IBDevice *dev() const { return dev_.get(); }
  const ibv_port_attr attr() const { return attr_; }
  const std::vector<IfAddrs::Addr> &addrs() const { return dev_->ports().at(portNum_).addrs; }
  const std::set<std::string> &zones() const { return dev_->ports().at(portNum_).zones; }

  bool isRoCE() const { return attr().link_layer == IBV_LINK_LAYER_ETHERNET; }
  bool isInfiniband() const { return attr().link_layer == IBV_LINK_LAYER_INFINIBAND; }
  bool isActive() const { return attr().state == IBV_PORT_ACTIVE || attr().state == IBV_PORT_ACTIVE_DEFER; }
  Result<ibv_gid> queryGid(uint8_t index) const;
  std::pair<ibv_gid, uint8_t> getRoCEv2Gid() const {
    XLOGF_IF(FATAL, !isRoCE(), "port is not RoCE");
    return *rocev2Gid_;
  }

  operator bool() const { return dev_ && portNum_ != 0; }

 private:
  std::shared_ptr<const IBDevice> dev_;
  uint8_t portNum_;
  ibv_port_attr attr_;
  std::optional<std::pair<ibv_gid, uint8_t>> rocev2Gid_;
};

class IBSocket;
class IBSocketManager;

class IBManager {
 public:
  static IBManager &instance() {
    static IBManager instance;
    return instance;
  }
  static Result<Void> start(IBConfig config) { return instance().startImpl(config); }
  static void stop() { return instance().reset(); }
  static void close(std::unique_ptr<IBSocket> socket);
  static bool initialized() { return instance().inited_; }

  ~IBManager();

  const std::vector<IBDevice::Ptr> &allDevices() const { return devices_; }
  const std::multimap<std::string, std::pair<IBDevice::Ptr, uint8_t>> zone2port() const { return zone2port_; }
  const auto &config() const { return config_; }

 private:
  IBManager();

  void reset();

  Result<Void> startImpl(IBConfig config);
  void stopImpl() { return reset(); }
  void closeImpl(std::unique_ptr<IBSocket> socket);

  friend class IBSocketManager;

  IBConfig config_;
  std::atomic<bool> inited_ = false;
  std::vector<IBDevice::Ptr> devices_;
  std::multimap<std::string, std::pair<IBDevice::Ptr, uint8_t>> zone2port_;
  std::shared_ptr<EventLoop> eventLoop_;
  std::shared_ptr<IBSocketManager> socketManager_;
  std::shared_ptr<IBDevice::BackgroundRunner> devBgRunner_;
  std::vector<std::shared_ptr<IBDevice::AsyncEventHandler>> devEventHandlers_;
};
}  // namespace hf3fs::net

FMT_BEGIN_NAMESPACE

inline std::string_view ibvLinklayerName(int linklayer) {
  switch (linklayer) {
    case IBV_LINK_LAYER_INFINIBAND:
      return "INFINIBAND";
    case IBV_LINK_LAYER_ETHERNET:
      return "ETHERNET";
    default:
      return "UNSPECIFIED";
  }
}

inline std::string_view ibvPortStateName(ibv_port_state state) {
  switch (state) {
    case IBV_PORT_NOP:
      return "NOP";
    case IBV_PORT_DOWN:
      return "DOWN";
    case IBV_PORT_INIT:
      return "INIT";
    case IBV_PORT_ARMED:
      return "ARMED";
    case IBV_PORT_ACTIVE:
      return "ACTIVE";
    case IBV_PORT_ACTIVE_DEFER:
      return "ACTIVE_DEFER";
    default:
      return "UNKNOWN";
  }
}

template <>
struct formatter<ibv_gid> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(const ibv_gid &gid, FormatContext &ctx) const {
    std::string str = fmt::format("{:x}", fmt::join(&gid.raw[0], &gid.raw[sizeof(gid.raw)], ":"));
    return formatter<std::string_view>::format(str, ctx);
  }
};

template <>
struct formatter<ibv_port_attr> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(const ibv_port_attr &attr, FormatContext &ctx) const {
    std::string str = fmt::format("{{linklayer: {}, state: {}, lid: {}, mtu: {}}}",
                                  ibvLinklayerName(attr.link_layer),
                                  ibvPortStateName(attr.state),
                                  attr.lid,
                                  magic_enum::enum_name(attr.active_mtu));
    return formatter<std::string_view>::format(str, ctx);
  }
};

template <>
struct formatter<hf3fs::net::IBPort> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(const hf3fs::net::IBPort &port, FormatContext &ctx) const {
    return fmt::format_to(ctx.out(),
                          "{{{}:{}, ifaddrs [{}], zones [{}], {}, {}}}",
                          port.dev()->name(),
                          port.portNum(),
                          fmt::join(port.addrs().begin(), port.addrs().end(), ";"),
                          fmt::join(port.zones().begin(), port.zones().end(), ";"),
                          ibvLinklayerName(port.attr().link_layer),
                          ibvPortStateName(port.attr().state));
  }
};

template <>
struct formatter<hf3fs::net::IBDevice::Config::Network> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(const hf3fs::net::IBDevice::Config::Network &network, FormatContext &ctx) const {
    return fmt::format_to(ctx.out(), "{}/{}", network.ip().str(), network.mask());
  }
};

FMT_END_NAMESPACE

#pragma once
#include <arrow/io/file.h>
#include <optional>
#include <parquet/stream_reader.h>
#include <type_traits>
#include <utility>
#include <variant>

#include "SerdeObjectVisitor.h"
#include "SerdeSchemaBuilder.h"
#include "common/serde/Serde.h"
#include "common/utils/Nameof.hpp"
#include "common/utils/StrongType.h"
#include "common/utils/TypeTraits.h"

namespace hf3fs::analytics {

template <serde::SerdeType SerdeType>
class SerdeObjectReader : public BaseObjectVisitor<SerdeObjectReader<SerdeType>> {
 public:
  SerdeObjectReader(parquet::StreamReader &&reader)
      : reader_(std::move(reader)) {}

  static std::shared_ptr<SerdeObjectReader> open(const Path path) {
    // open file
    auto openStream = arrow::io::ReadableFile::Open(path.string());

    if (!openStream.ok()) {
      XLOGF(ERR, "Failed to open file input stream: {}, error: {}", path.string(), openStream.status().message());
      return nullptr;
    }

    std::shared_ptr<arrow::io::ReadableFile> infile;
    PARQUET_ASSIGN_OR_THROW(infile, openStream);

    try {
      parquet::StreamReader streamReader{parquet::ParquetFileReader::Open(infile)};
      return std::make_shared<SerdeObjectReader>(std::move(streamReader));
    } catch (const std::exception &ex) {
      XLOGF(ERR, "Failed to create stream reader: {}, error: {}", path.string(), ex.what());
      return nullptr;
    }
  }

  SerdeObjectReader &operator>>(SerdeType &v) {
    eof_ = eof();
    if (!bool(*this)) return *this;

    try {
      visit("", v);
      reader_ >> parquet::EndRow;
    } catch (const parquet::ParquetException &ex) {
      XLOGF(CRITICAL, "Failed to read from parquet file, error: {}", ex.what());
      isOk_ = false;
    }

    return *this;
  }

  operator bool() const { return ok() && !eof_; }

  bool ok() const { return isOk_; }

  bool eof() const { return reader_.eof(); }

  size_t numRows() const { return reader_.num_rows(); }

 public:
  // default
  template <typename T>
  void visit(std::string_view k, T &v) = delete;

  template <typename T>
  requires std::is_arithmetic_v<T>
  void visit(std::string_view k, T &v) {
    reader_ >> v;
    XLOGF(DBG3, "arithmetic visit({}): {}", k, v);
  }

  template <typename T>
  requires std::is_enum_v<T>
  void visit(std::string_view k, T &v) {
    int32_t n;
    reader_ >> n;
    XLOGF(DBG3, "enum visit({}): {}", k, n);
    auto result = magic_enum::enum_cast<T>(n);
    if (result) {
      v = *result;
    } else {
      XLOGF(CRITICAL, "Failed to parse enum {} from value: {}", nameof::nameof_short_type<T>(), n);
    }
  }

  template <serde::ConvertibleToString T>
  void visit(std::string_view k, T &v) {
    reader_ >> v;
    XLOGF(DBG3, "string visit({}): {}", k, v);
  }

  template <StrongTyped T>
  void visit(std::string_view k, T &v) {
    XLOGF(DBG3, "strongtyped visit({})", k);
    BaseObjectVisitor<SerdeObjectReader>::visit(k, v);
  }

  template <serde::WithReadableSerdeMethod T>
  void visit(std::string_view k, T &val) {
    XLOGF(DBG3, "WithReadableSerdeMethod visit({})", k);
    typename serde::SerdeToReadableReturnType<T> serialized;
    visit(k, serialized);
    auto result = serde::SerdeMethod<T>::serdeFromReadable(serialized);
    if (result) {
      val = *result;
    } else {
      XLOGF(CRITICAL, "Failed to parse {} from value: {}", nameof::nameof_short_type<T>(), serialized);
    }
  }

  template <serde::WithSerdeMethod T>
  void visit(std::string_view k, T &val) {
    XLOGF(DBG3, "WithSerdeMethod visit({})", k);
    typename serde::SerdeToReturnType<T> serialized;
    visit(k, serialized);
    auto result = serde::SerdeMethod<T>::serdeFrom(serialized);
    if (result) {
      val = *result;
    } else {
      XLOGF(CRITICAL, "Failed to parse {} from value: {}", nameof::nameof_short_type<T>(), serialized);
    }
  }

  template <serde::WithReadableSerdeMemberMethod T>
  void visit(std::string_view k, T &val) {
    XLOGF(DBG3, "WithReadableSerdeMemberMethod visit({})", k);
    serde::SerdeToReadableMemberMethodReturnType<T> serialized;
    visit(k, serialized);
    auto result = T::serdeFromReadable(serialized);
    if (result) {
      val = *result;
    } else {
      XLOGF(CRITICAL, "Failed to parse {} from value: {}", nameof::nameof_short_type<T>(), serialized);
    }
  }

  template <serde::WithSerdeMemberMethod T>
  void visit(std::string_view k, T &val) {
    XLOGF(DBG3, "WithSerdeMemberMethod visit({})", k);
    serde::SerdeToReadableMemberMethodReturnType<T> serialized;
    visit(k, serialized);
    auto result = T::serdeFromReadable(serialized);
    if (result) {
      val = *result;
    } else {
      XLOGF(CRITICAL, "Failed to parse {} from value: {}", nameof::nameof_short_type<T>(), serialized);
    }
  }

  template <serde::SerdeTypeWithoutSpecializedSerdeMethod T>
  void visit(std::string_view k, T &val) {
    XLOGF(DBG3, "serdetype visit({})", k);
    BaseObjectVisitor<SerdeObjectReader>::visit(k, val);
  }

  template <typename T>
  requires is_specialization_of_v<T, folly::Expected>
  void visit(std::string_view k, T &val) {
    XLOGF(DBG3, "result visit({})", k);
    std::string errorColumnName = std::string{k} + kResultErrorTypeColumnSuffix;

    Status status(StatusCode::kOK);
    typename T::value_type value;
    visit<typename T::error_type>(errorColumnName, status);
    visit<typename T::value_type>(k, value);

    if (status.isOK()) {
      val = std::move(value);
    } else {
      val = makeError(status);
    }
  }

  template <typename T>
  requires is_variant_v<T>
  void visit(std::string_view k, T &val) {
    XLOGF(DBG3, "variant visit({})", k);
    // get the index of value
    std::string valIdxColumnName = std::string{k} + kVariantValueIndexColumnSuffix;
    uint32_t valIdx = 0;
    visit<uint32_t>(valIdxColumnName, valIdx);

    // read and set the value
    uint32_t altIdx = 0;
    visitVariant(val, [&](std::string_view typeName, auto &&v) {
      std::string altTypeName = std::string{k} + std::string{typeName};
      std::remove_reference_t<decltype(v)> alt;
      visit(altTypeName, alt);
      if (altIdx == valIdx) val = std::move(alt);
      altIdx++;
    });
  }

  template <typename T>
  requires is_vector_v<T> || is_set_v<T>
  void visit(std::string_view k, T &val) {
    std::string str;
    reader_ >> str;
    XLOGF(DBG3, "container visit({}): {}", k, str);
    auto result = serde::fromJsonString(val, str);
    if (!result) {
      XLOGF(CRITICAL, "Failed to parse {} from json string: {}", nameof::nameof_short_type<T>(), str);
    }
  }

  template <typename T>
  requires is_optional_v<T>
  void visit(std::string_view k, T &val) {
    std::string str;
    reader_ >> str;
    XLOGF(DBG3, "container visit({}): {}", k, str);
    if (str.empty()) {
      val = std::nullopt;
    } else {
      using ValueType = typename T::value_type;
      val = ValueType();
      auto result = serde::fromJsonString(*val, str);
      if (!result) {
        XLOGF(CRITICAL, "Failed to parse {} from json string: {}", nameof::nameof_short_type<ValueType>(), str);
      }
    }
  }

 private:
  parquet::StreamReader reader_;
  bool isOk_{true};
  bool eof_{false};
};

}  // namespace hf3fs::analytics

#pragma once

#include <cstring>
#include <folly/io/async/Request.h>
#include <memory>
#include <mutex>

#include "String.h"
#include "TracingEvent.h"
#include "UtcTime.h"

#define SCOPE_SET_TRACING_POINTS(points) hf3fs::tracing::ScopeGuard FB_ANONYMOUS_VARIABLE(guard)(points)

#define TRACING_ADD_SCOPE_EVENT(...) hf3fs::tracing::ScopeTracer FB_ANONYMOUS_VARIABLE(guard)(__VA_ARGS__)

#define TRACING_ADD_EVENT(...)                                       \
  do {                                                               \
    auto *proxy = hf3fs::tracing::PointsProxy::get();                \
    if (proxy) {                                                     \
      proxy->points().addPoint(hf3fs::UtcClock::now(), __VA_ARGS__); \
    }                                                                \
  } while (false)

namespace hf3fs::tracing {
struct Point {
  Point(UtcTime ts, uint64_t e, std::string_view msg = {})
      : timestamp_(ts),
        event_(e),
        message_(msg) {}

  ~Point() = default;

  Point(const Point &) = delete;
  Point(Point &&other) = default;

  Point &operator=(const Point &) = delete;
  Point &operator=(Point &&other) = default;

  void setMessage(std::string_view msg) { message_ = msg; }

  void clearMessage() { message_.clear(); }

  std::string_view message() const { return message_; }

  uint64_t event() const { return event_; }

  UtcTime timestamp() const { return timestamp_; }

 private:
  UtcTime timestamp_;
  uint64_t event_;
  String message_;
};

class Points {
 public:
  void addPoint(UtcTime ts, uint64_t event, std::string_view msg = {}) {
    std::unique_lock lock(mu_);
    points_.emplace_back(ts, event, msg);
  }

  std::vector<Point> extractAll() {
    std::unique_lock lock(mu_);
    return std::move(points_);
  }

 private:
  std::mutex mu_;
  std::vector<Point> points_;
};

class PointsProxy : public folly::RequestData {
 public:
  static PointsProxy *get() {
    auto requestContext = folly::RequestContext::try_get();
    if (LIKELY(requestContext == nullptr)) {
      return nullptr;
    }
    return dynamic_cast<PointsProxy *>(requestContext->getContextData(token()));
  }

  static const folly::RequestToken &token() {
    static const folly::RequestToken t(kTokenName);
    return t;
  }

  explicit PointsProxy(Points &points)
      : points_(points) {}

  Points &points() { return points_; }

  bool hasCallback() override { return false; }

 private:
  static constexpr auto kTokenName = "hf3fs::tracing::Points";

  Points &points_;
};

class ScopeGuard {
 public:
  explicit ScopeGuard(Points &points) { guard_.emplace(PointsProxy::token(), std::make_unique<PointsProxy>(points)); }

 private:
  std::optional<folly::ShallowCopyRequestContextScopeGuard> guard_;
};

class ScopeTracer {
 public:
  ScopeTracer(uint64_t event, std::string_view msg = {})
      : event_(event) {
    pointsProxy_ = PointsProxy::get();
    if (pointsProxy_) {
      pointsProxy_->points().addPoint(UtcClock::now(), getBeginEvent(event_), msg);
    }
  }

  ~ScopeTracer() {
    if (pointsProxy_) {
      pointsProxy_->points().addPoint(UtcClock::now(), getEndEvent(event_));
    }
  }

 private:
  PointsProxy *pointsProxy_;
  uint64_t event_;
};

}  // namespace hf3fs::tracing

#pragma once

#include <span>
#include <vector>

#include "StorageClient.h"
#include "StorageMessenger.h"
#include "UpdateChannelAllocator.h"
#include "common/net/Client.h"
#include "common/utils/Address.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Result.h"
#include "fbs/storage/Common.h"

namespace hf3fs::storage::client {

class ClientRequestContext;

class StorageClientImpl : public StorageClient {
 public:
  StorageClientImpl(const ClientId &clientId, const Config &config, hf3fs::client::ICommonMgmtdClient &mgmtdClient);

  ~StorageClientImpl() override;

  Result<Void> start() override;

  hf3fs::client::ICommonMgmtdClient &getMgmtdClient() override { return mgmtdClient_; }
  void stop() override;

  CoTryTask<void> batchRead(std::span<ReadIO> readIOs,
                            const flat::UserInfo &userInfo,
                            const ReadOptions &options = ReadOptions(),
                            std::vector<ReadIO *> *failedIOs = nullptr) override;

  CoTryTask<void> batchWrite(std::span<WriteIO> writeIOs,
                             const flat::UserInfo &userInfo,
                             const WriteOptions &options = WriteOptions(),
                             std::vector<WriteIO *> *failedIOs = nullptr) override;

  CoTryTask<void> read(ReadIO &readIO,
                       const flat::UserInfo &userInfo,
                       const ReadOptions &options = ReadOptions()) override;

  CoTryTask<void> write(WriteIO &writeIO,
                        const flat::UserInfo &userInfo,
                        const WriteOptions &options = WriteOptions()) override;

  CoTryTask<void> queryLastChunk(std::span<QueryLastChunkOp> ops,
                                 const flat::UserInfo &userInfo,
                                 const ReadOptions &options = ReadOptions(),
                                 std::vector<QueryLastChunkOp *> *failedOps = nullptr) override;

  CoTryTask<void> removeChunks(std::span<RemoveChunksOp> ops,
                               const flat::UserInfo &userInfo,
                               const WriteOptions &options = WriteOptions(),
                               std::vector<RemoveChunksOp *> *failedOps = nullptr) override;

  CoTryTask<void> truncateChunks(std::span<TruncateChunkOp> ops,
                                 const flat::UserInfo &userInfo,
                                 const WriteOptions &options = WriteOptions(),
                                 std::vector<TruncateChunkOp *> *failedOps = nullptr) override;

  CoTryTask<SpaceInfoRsp> querySpaceInfo(NodeId nodeId) override;

  CoTryTask<CreateTargetRsp> createTarget(NodeId nodeId, const CreateTargetReq &req) override;

  CoTryTask<OfflineTargetRsp> offlineTarget(NodeId nodeId, const OfflineTargetReq &req) override;

  CoTryTask<RemoveTargetRsp> removeTarget(NodeId nodeId, const RemoveTargetReq &req) override;

  CoTryTask<std::vector<Result<QueryChunkRsp>>> queryChunk(const QueryChunkReq &req) override;

  CoTryTask<ChunkMetaVector> getAllChunkMetadata(const ChainId &chainId, const TargetId &targetId) override;

 private:
  CoTryTask<void> batchReadWithRetry(ClientRequestContext &requestCtx,
                                     const std::vector<ReadIO *> &readIOs,
                                     const flat::UserInfo &userInfo,
                                     const ReadOptions &options,
                                     std::vector<ReadIO *> &failedIOs);

  CoTryTask<void> batchReadWithoutRetry(ClientRequestContext &requestCtx,
                                        const std::vector<ReadIO *> &readIOs,
                                        const flat::UserInfo &userInfo,
                                        const ReadOptions &options);

  CoTryTask<void> batchWriteWithRetry(ClientRequestContext &requestCtx,
                                      const std::vector<WriteIO *> &writeIOs,
                                      const flat::UserInfo &userInfo,
                                      const WriteOptions &options,
                                      std::vector<WriteIO *> &failedIOs);

  CoTryTask<void> batchWriteWithoutRetry(ClientRequestContext &requestCtx,
                                         const std::vector<WriteIO *> &writeIOs,
                                         const flat::UserInfo &userInfo,
                                         const WriteOptions &options);

  CoTryTask<void> sendWriteRequest(ClientRequestContext &requestCtx,
                                   WriteIO *writeIO,
                                   const hf3fs::flat::NodeInfo &nodeInfo,
                                   const flat::UserInfo &userInfo,
                                   const WriteOptions &options);

  CoTryTask<void> sendWriteRequestsSequentially(ClientRequestContext &requestCtx,
                                                const std::vector<WriteIO *> &writeIOs,
                                                std::shared_ptr<hf3fs::client::RoutingInfo const> routingInfo,
                                                NodeId nodeId,
                                                const flat::UserInfo &userInfo,
                                                const WriteOptions &options);

  CoTryTask<void> queryLastChunkWithoutRetry(ClientRequestContext &requestCtx,
                                             const std::vector<QueryLastChunkOp *> &ops,
                                             const flat::UserInfo &userInfo,
                                             const ReadOptions &options);

  CoTryTask<void> removeChunksWithoutRetry(ClientRequestContext &requestCtx,
                                           const std::vector<RemoveChunksOp *> &ops,
                                           const flat::UserInfo &userInfo,
                                           const WriteOptions &options);

  CoTryTask<void> truncateChunksWithoutRetry(ClientRequestContext &requestCtx,
                                             const std::vector<TruncateChunkOp *> &ops,
                                             const flat::UserInfo &userInfo,
                                             const WriteOptions &options);

 private:
  void setCurrentRoutingInfo(std::shared_ptr<hf3fs::client::RoutingInfo const> latestRoutingInfo);

  std::shared_ptr<hf3fs::client::RoutingInfo const> getCurrentRoutingInfo() { return currentRoutingInfo_.load(); }

  StorageMessenger &getStorageMessengerForUpdates() {
    return config_.create_net_client_for_updates() ? messengerForUpdates_ : messenger_;
  }

  template <typename Req, typename Rsp, auto Method>
  CoTryTask<Rsp> callMessengerMethod(StorageMessenger &messenger,
                                     ClientRequestContext &requestCtx,
                                     const hf3fs::flat::NodeInfo &nodeInfo,
                                     const Req &request);

  template <typename Op, typename BatchReq, typename BatchRsp, auto Method>
  CoTryTask<BatchRsp> sendBatchRequest(StorageMessenger &messenger,
                                       ClientRequestContext &requestCtx,
                                       std::shared_ptr<hf3fs::client::RoutingInfo const> routingInfo,
                                       const NodeId &nodeId,
                                       const BatchReq &batchReq,
                                       const std::vector<Op *> &ops);

 private:
  class OperationConcurrencyLimit {
   public:
    OperationConcurrencyLimit(size_t maxConcurrentRequests, size_t maxConcurrentRequestsPerServer)
        : maxConcurrentRequests_(maxConcurrentRequests),
          maxConcurrentRequestsPerServer_(maxConcurrentRequestsPerServer),
          concurrencySemaphore_(maxConcurrentRequests_) {}

    OperationConcurrencyLimit(const OperationConcurrency &config)
        : OperationConcurrencyLimit(config.max_concurrent_requests(), config.max_concurrent_requests_per_server()) {}

    virtual ~OperationConcurrencyLimit() = default;

    hf3fs::Semaphore &getConcurrencySemaphore() { return concurrencySemaphore_; }

    virtual std::shared_ptr<hf3fs::Semaphore> getPerServerSemaphore(const NodeId &nodeId) {
      return getPerServerSemaphore(nodeId, maxConcurrentRequestsPerServer_);
    }

   protected:
    std::shared_ptr<hf3fs::Semaphore> getPerServerSemaphore(const NodeId &nodeId, size_t initTokens) {
      {
        std::shared_lock rlock(perServerSemaphoreMutex_);
        auto iter = perServerSemaphore_.find(nodeId);
        if (iter != perServerSemaphore_.end()) {
          return iter->second;
        }
      }

      std::unique_lock wlock(perServerSemaphoreMutex_);

      auto iter = perServerSemaphore_.find(nodeId);
      if (iter != perServerSemaphore_.end()) {
        return iter->second;
      }

      auto semaphore = std::make_shared<hf3fs::Semaphore>(initTokens);
      perServerSemaphore_.emplace(nodeId, semaphore);
      return semaphore;
    }

   protected:
    const size_t maxConcurrentRequests_;
    const size_t maxConcurrentRequestsPerServer_;
    hf3fs::Semaphore concurrencySemaphore_;
    std::shared_mutex perServerSemaphoreMutex_;  // protect the following maps
    std::unordered_map<NodeId, std::shared_ptr<hf3fs::Semaphore>> perServerSemaphore_;
  };

  class HotLoadOperationConcurrencyLimit : public OperationConcurrencyLimit {
   public:
    HotLoadOperationConcurrencyLimit(const HotLoadOperationConcurrency &config)
        : OperationConcurrencyLimit(config.max_concurrent_requests(), config.max_concurrent_requests_per_server()),
          config_(config),
          onConfigUpdated_(config_.addCallbackGuard([this]() { updateUsableTokens(); })) {}

    std::shared_ptr<hf3fs::Semaphore> getPerServerSemaphore(const NodeId &nodeId) override {
      return OperationConcurrencyLimit::getPerServerSemaphore(nodeId, config_.max_concurrent_requests_per_server());
    }

   private:
    void updateUsableTokens() {
      concurrencySemaphore_.changeUsableTokens(config_.max_concurrent_requests());

      std::shared_lock rlock(perServerSemaphoreMutex_);
      for (auto &[_, semaphore] : perServerSemaphore_) {
        semaphore->changeUsableTokens(config_.max_concurrent_requests_per_server());
      }
    }

   private:
    const HotLoadOperationConcurrency &config_;
    std::unique_ptr<ConfigCallbackGuard> onConfigUpdated_;
  };

 private:
  bool clientStarted_;
  hf3fs::client::ICommonMgmtdClient &mgmtdClient_;
  StorageMessenger messenger_;
  StorageMessenger messengerForUpdates_;
  UpdateChannelAllocator chanAllocator_;
  folly::atomic_shared_ptr<hf3fs::client::RoutingInfo const> currentRoutingInfo_;

  HotLoadOperationConcurrencyLimit readConcurrencyLimit_;
  OperationConcurrencyLimit writeConcurrencyLimit_;
  HotLoadOperationConcurrencyLimit queryConcurrencyLimit_;
  OperationConcurrencyLimit removeConcurrencyLimit_;
  OperationConcurrencyLimit truncateConcurrencyLimit_;
};

}  // namespace hf3fs::storage::client

#pragma once

#include <cassert>
#include <folly/ScopeGuard.h>
#include <folly/logging/xlog.h>
#include <functional>
#include <memory>
#include <optional>
#include <type_traits>
#include <utility>

#include "common/kv/ITransaction.h"
#include "common/monitor/Recorder.h"
#include "common/monitor/Sample.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Duration.h"
#include "common/utils/Result.h"
#include "common/utils/UtcTime.h"
#include "fbs/meta/Common.h"
#include "fbs/meta/Service.h"
#include "fbs/meta/Utils.h"
#include "fdb/FDBRetryStrategy.h"
#include "fdb/FDBTransaction.h"
#include "meta/components/AclCache.h"
#include "meta/components/GcManager.h"
#include "meta/components/SessionManager.h"
#include "meta/event/Event.h"
#include "meta/store/Idempotent.h"
#include "meta/store/MetaStore.h"

#define OPERATION_TAGS(reqName)                                                       \
  std::string_view name() const override { return MetaSerde<>::getRpcName(reqName); } \
  flat::Uid user() const override { return reqName.user.uid; }

#define CHECK_REQUEST(reqName)                                              \
  do {                                                                      \
    if (auto result = reqName.valid(); UNLIKELY(result.hasError())) {       \
      auto rpcName = MetaSerde<>::getRpcName(reqName);                      \
      XLOGF(WARN, "{} get invalid req, error {}", rpcName, result.error()); \
      CO_RETURN_ERROR(result);                                              \
    }                                                                       \
  } while (0)

namespace hf3fs::meta::server {

template <typename Rsp>
class Operation : public IOperation<Rsp> {
 public:
  Operation(MetaStore &meta)
      : meta_(meta) {}

  bool isReadOnly() override { return false; }
  CoTryTask<Rsp> run(IReadWriteTransaction &) override = 0;

  void retry(const Status &) override { clearEvents(); }

  void finish(const Result<Rsp> &result) override {
    if (!result.hasError()) {
      // success
      for (const auto &event : events_) {
        event.log();
      }

      for (const auto &trace : traces_) {
        auto &traceLog = meta_.getEventTraceLog();
        traceLog.append(trace);
      }
    }
  }

 protected:
  const Config &config() const { return meta_.config_; }

  InodeIdAllocator &inodeIdAlloc() { return *meta_.inodeAlloc_; }
  ChainAllocator &chainAlloc() { return *meta_.chainAlloc_; }
  FileHelper &fileHelper() { return *meta_.fileHelper_; }
  SessionManager &sessionManager() { return *meta_.sessionManager_; }
  GcManager &gcManager() { return *meta_.gcManager_; }
  AclCache &aclCache() { return meta_.aclCache_; }
  Distributor &distributor() { return *meta_.distributor_; }

  UtcTime now() const { return UtcClock::now().castGranularity(config().time_granularity()); }

  PathResolveOp resolve(IReadOnlyTransaction &txn, const UserInfo &user, Path *path = nullptr) {
    return PathResolveOp(txn,
                         aclCache(),
                         user,
                         path,
                         config().max_symlink_count(),
                         config().max_symlink_depth(),
                         config().acl_cache_time());
  }

  CoTryTask<InodeId> allocateInodeId(IReadWriteTransaction &txn, bool newChunkEngine) {
    auto newId = co_await inodeIdAlloc().allocate();
    CO_RETURN_ON_ERROR(newId);

    if (newChunkEngine) {
      newId = InodeId::withNewChunkEngine(*newId);
    }

    if (config().inodeId_check_unique()) {
      auto loadResult = co_await Inode::load(txn, *newId);
      CO_RETURN_ON_ERROR(loadResult);
      if (loadResult->has_value()) {
        XLOGF_IF(FATAL,
                 config().inodeId_abort_on_duplicate(),
                 "InodeIdAllocator get duplicated InodeId {}",
                 newId.value());
        XLOGF(DFATAL, "InodeIdAllocator get duplicated InodeId {}", newId.value());
        co_return makeError(MetaCode::kInodeIdAllocFailed);
      }
    } else {
      XLOGF_EVERY_MS(WARN, (300 * 1000), "inodeId_check_unique is disabled");
    }

    co_return newId;
  }

  void clearEvents() { events_.clear(); }
  Event &addEvent(Event::Type type) {
    events_.emplace_back(type);
    return *events_.rbegin();
  }
  void addEvent(Event event) { events_.emplace_back(std::move(event)); }

  void addTrace(MetaEventTrace &&trace) { traces_.emplace_back(std::move(trace)); }

  MetaStore &meta_;
  std::vector<Event> events_;
  std::vector<MetaEventTrace> traces_;
};

template <typename Rsp>
class ReadOnlyOperation : public Operation<Rsp> {
 public:
  ReadOnlyOperation(MetaStore &meta)
      : Operation<Rsp>(meta) {}

  bool isReadOnly() final { return true; }

  virtual CoTryTask<Rsp> run(IReadOnlyTransaction &) = 0;
  CoTryTask<Rsp> run(IReadWriteTransaction &txn) final {
    co_return co_await run(static_cast<IReadOnlyTransaction &>(txn));
  }
};

template <typename Rsp, typename ReqInfo>
class OperationDriver {
 public:
  OperationDriver(MetaStore::Op<Rsp> &operation, const ReqInfo &req, std::optional<SteadyTime> deadline = std::nullopt)
      : operation_(operation),
        req_(req),
        deadline_(deadline) {}

  CoTryTask<Rsp> run(std::unique_ptr<kv::IReadWriteTransaction> txn,
                     kv::FDBRetryStrategy::Config config,
                     bool readonly,
                     bool enableGrvCache) {
    config.retryMaybeCommitted = operation_.retryMaybeCommitted();
    kv::FDBRetryStrategy strategy(config);
    CO_RETURN_ON_ERROR(strategy.init(txn.get()));

    OperationRecorder::Guard recorder(OperationRecorder::server(), operation_.name(), operation_.user());

    if (readonly && !operation_.isReadOnly()) {
      co_return makeError(StatusCode::kReadOnlyMode, "FileSystem is in readonly mode.");
    }

    auto grvCache = operation_.isReadOnly() && enableGrvCache;
    if (grvCache && dynamic_cast<kv::FDBTransaction *>(txn.get())) {
      auto fdbTxn = dynamic_cast<kv::FDBTransaction *>(txn.get());
      CO_RETURN_ON_ERROR(fdbTxn->setOption(FDBTransactionOption::FDB_TR_OPTION_USE_GRV_CACHE, {}));
    }

    Result<Rsp> result = makeError(MetaCode::kOperationTimeout);
    auto duplicate = false;
    while (true) {
      // check timeout
      if (deadline_ && deadline_.value() <= SteadyClock::now()) {
        XLOGF(ERR, "Request {} timeout, return error {}", describe(), result);
        break;
      }
      // run operation
      result = co_await runAndCommit(*txn, operation_, duplicate);
      if (ErrorHandling::success(result)) {
        break;
      }
      // retry
      XLOGF(WARN, "Request {} failed, error {}", describe(), result.error());
      operation_.retry(result.error());
      auto retry = co_await strategy.onError(txn.get(), result.error());
      if (retry.hasError()) {
        result = makeError(retry.error());
        break;
      }
      recorder.retry()++;
    }

    if (result.hasError() && result.error().code() == StatusCode::kOK) {
      XLOGF(DFATAL, "Has error but error code is kOK, {}, {}", describe(), result);
      result = makeError(MetaCode::kFoundBug);
    }

    recorder.finish(result, duplicate);
    operation_.finish(result);
    co_return result;
  }

 private:
#define IDEMPOTENT_CHECK()                                                           \
  do {                                                                               \
    auto idemCheck = co_await Idempotent::load<Rsp>(txn, clientId, requestId, req_); \
    CO_RETURN_ON_ERROR(idemCheck);                                                   \
    duplicate = idemCheck->has_value();                                              \
    if (duplicate) {                                                                 \
      co_return idemCheck->value();                                                  \
    }                                                                                \
  } while (0)

  template <typename Handler>
  std::invoke_result_t<Handler, IReadWriteTransaction &> runAndCommit(IReadWriteTransaction &txn,
                                                                      Handler &&handler,
                                                                      bool &duplicate) {
    Uuid clientId, requestId;
    auto readonly = handler.isReadOnly();
    auto idem = !readonly && operation_.needIdempotent(clientId, requestId);
    if (idem) {
      OperationRecorder::server().addIdempotentCount();
      IDEMPOTENT_CHECK();
      auto result = co_await handler(txn);
      if (result) {
        CO_RETURN_ON_ERROR(co_await Idempotent::store(txn, clientId, requestId, result));
        CO_RETURN_ON_ERROR(co_await txn.commit());
      } else if (ErrorHandling::success(result) || !ErrorHandling::retryable(result.error())) {
        // this is final result, discard other modifications and save result
        txn.reset();
        IDEMPOTENT_CHECK();
        CO_RETURN_ON_ERROR(co_await Idempotent::store(txn, clientId, requestId, result));
        CO_RETURN_ON_ERROR(co_await txn.commit());
      }
      co_return result;
    } else {
      auto result = co_await handler(txn);
      if (!result.hasError() && !readonly) {
        CO_RETURN_ON_ERROR(co_await txn.commit());
      }
      co_return result;
    }
  }

  std::string describe() const {
    if constexpr (std::is_base_of_v<ReqBase, ReqInfo>) {
      return fmt::format("{}{}", operation_.name(), req_);
    } else {
      return std::string(operation_.name());
    }
  }

  MetaStore::Op<Rsp> &operation_;
  const ReqInfo &req_;
  std::optional<SteadyTime> deadline_;
};

}  // namespace hf3fs::meta::server

#pragma once

#include <algorithm>
#include <atomic>
#include <chrono>
#include <cstddef>
#include <deque>
#include <fmt/core.h>
#include <folly/Executor.h>
#include <folly/Likely.h>
#include <folly/Range.h>
#include <folly/Synchronized.h>
#include <folly/ThreadLocal.h>
#include <folly/Utility.h>
#include <folly/experimental/coro/Baton.h>
#include <folly/experimental/coro/BoundedQueue.h>
#include <folly/experimental/coro/CurrentExecutor.h>
#include <folly/experimental/coro/Mutex.h>
#include <folly/experimental/coro/Promise.h>
#include <folly/experimental/coro/Sleep.h>
#include <folly/experimental/coro/Timeout.h>
#include <folly/fibers/BatchSemaphore.h>
#include <folly/io/async/Request.h>
#include <folly/logging/xlog.h>
#include <list>
#include <memory>
#include <mutex>
#include <optional>
#include <queue>
#include <string>
#include <string_view>
#include <utility>

#include "common/kv/IKVEngine.h"
#include "common/monitor/Recorder.h"
#include "common/utils/Coroutine.h"
#include "common/utils/FaultInjection.h"
#include "common/utils/IdAllocator.h"
#include "common/utils/Result.h"
#include "fbs/meta/Common.h"
#include "fdb/FDBRetryStrategy.h"

namespace hf3fs::meta::server {

/**
 * Generate InodeId range from 0x00000000_00001000 to 0x01ffffff_ffffffff.
 *
 * Generated InodeId format: [ high 52bits: generated by IdAllocator ][ low 12 bits: local generated ].
 * InodeIdAllocator first use IdAllocator to generate a 52bits value, then left shift 12 bits and generate lower 12 bits
 * locally. So it only need to access the FoundationDB after generate 4096 InodeIds.
 */
class InodeIdAllocator : public std::enable_shared_from_this<InodeIdAllocator> {
  // These values are used in FoundationDB
  static std::string kAllocatorKeyPrefix;
  static constexpr size_t kAllocatorShard = 32;    // avoid txn conflictation
  static constexpr uint64_t kAllocatorShift = 12;  // shift 12 bit
  static constexpr uint64_t kAllocatorBit =
      64 - kAllocatorShift;  // IdAllocator generated values only have 52bits valid.
  static constexpr uint64_t kAllocatorMask = (1ULL << kAllocatorBit) - 1;
  static constexpr uint64_t kAllocateBatch = 1 << kAllocatorShift;

  struct Tag {};

 public:
  InodeIdAllocator(Tag, std::shared_ptr<kv::IKVEngine> kvEngine)
      : engine_(std::move(kvEngine)),
        allocator_(*engine_, createRetryStrategy(), kAllocatorKeyPrefix, kAllocatorShard),
        allocating_(false),
        queue_(2 * kAllocateBatch) {}

  static std::shared_ptr<InodeIdAllocator> create(std::shared_ptr<kv::IKVEngine> kvEngine) {
    return std::make_shared<InodeIdAllocator>(Tag{}, std::move(kvEngine));
  }

  CoTryTask<InodeId> allocate(std::chrono::microseconds timeout = std::chrono::seconds(2)) {
    static monitor::CountRecorder failed("meta_inodeid_alloc_failed");

    auto id = queue_.try_dequeue();
    if (LIKELY(id.has_value())) {
      if (queue_.size() < kAllocateBatch / 2) {
        tryStartAllocateTask(co_await folly::coro::co_current_executor);
      }
      co_return id.value();
    }
    auto result = co_await allocateSlow(timeout);
    if (result.hasError()) {
      failed.addSample(1);
      co_return result;
    }
    if (result->u64() >= InodeId::kNewChunkEngineMask) {
      failed.addSample(1);
      XLOGF(DFATAL, "InodeId {} is larger than", *result, InodeId(InodeId::kNewChunkEngineMask));
      co_return makeError(MetaCode::kInodeIdAllocFailed, "InodeId too large, shouldn't happen");
    }
    co_return result;
  }

 private:
  static kv::FDBRetryStrategy createRetryStrategy() { return kv::FDBRetryStrategy({.retryMaybeCommitted = true}); }

  static CoTask<void> allocateTask(std::weak_ptr<InodeIdAllocator> weak,
                                   std::optional<folly::Duration> delay = std::nullopt) {
    if (delay.has_value()) {
      co_await folly::coro::sleep(delay.value());
    }

    auto ptr = weak.lock();
    if (ptr) {
      co_await ptr->allocateFromDB();
    }
    co_return;
  }

  void tryStartAllocateTask(folly::Executor *exec) {
    if (!allocating_.exchange(true)) {
      startAllocateTask(exec);
    }
  }

  void startAllocateTask(folly::Executor *exec) {
    folly::RequestContextScopeGuard guard;
    allocateTask(weak_from_this()).scheduleOn(exec).start();
  }

  CoTryTask<InodeId> allocateSlow(std::chrono::microseconds timeout);
  CoTask<void> allocateFromDB();

  std::shared_ptr<kv::IKVEngine> engine_;
  IdAllocator<kv::FDBRetryStrategy> allocator_;
  std::atomic<bool> allocating_;
  folly::coro::BoundedQueue<InodeId, false, false> queue_;
};

}  // namespace hf3fs::meta::server

#pragma once

#include <fmt/core.h>
#include <folly/IPAddressV4.h>
#include <folly/SocketAddress.h>
#include <folly/hash/Hash.h>
#include <scn/scn.h>

#include "common/utils/MagicEnum.hpp"
#include "common/utils/Result.h"

namespace hf3fs::net {

constexpr const char *kUnixDomainSocketPrefix[] = {nullptr, "/tmp/domain_socket."};

struct Address {
  uint32_t ip{};
  uint16_t port{};
  enum Type : uint16_t { TCP, RDMA, IPoIB, LOCAL, UNIX };
  Type type = Type::TCP;
  using is_serde_copyable = void;

  explicit Address(uint64_t addr = 0) { *reinterpret_cast<uint64_t *>(this) = addr; }
  Address(uint32_t ip, uint16_t port, Type type)
      : ip(ip),
        port(port),
        type(type) {}

  bool isTCP() const { return type == Type::TCP || type == Type::IPoIB || type == Type::LOCAL || type == Type::UNIX; }
  bool isRDMA() const { return type == Type::RDMA; }
  bool isUNIX() const { return type == Type::UNIX; }

  bool operator==(const Address &other) const { return uint64_t(*this) == uint64_t(other); }

  Address tcp() { return Address{ip, port, Type::TCP}; }

  operator uint64_t() const { return *reinterpret_cast<const uint64_t *>(this); }
  std::string str() const {
    auto &arr = *reinterpret_cast<const std::array<uint8_t, 4> *>(this);
    return fmt::format("{}://{}.{}.{}.{}:{}", magic_enum::enum_name(type), arr[0], arr[1], arr[2], arr[3], port);
  }
  std::string toString() const { return str(); }
  std::string serdeToReadable() const { return toString(); }
  std::string ipStr() const {
    auto &arr = *reinterpret_cast<const std::array<uint8_t, 4> *>(this);
    return fmt::format("{}.{}.{}.{}", arr[0], arr[1], arr[2], arr[3]);
  }
  Result<std::string> domainSocketPath() const {
    if (UNLIKELY(type != Type::UNIX)) {
      return makeError(StatusCode::kInvalidFormat, "invalid type: {} != UNIX", type);
    }
    if (UNLIKELY(ip == 0 || ip >= sizeof(kUnixDomainSocketPrefix) / sizeof(kUnixDomainSocketPrefix[0]))) {
      return makeError(StatusCode::kInvalidFormat, "invalid ip: {}", ip);
    }
    return fmt::format("{}{}", kUnixDomainSocketPrefix[ip], port);
  }

  folly::SocketAddress toFollyAddress() const {
    if (UNLIKELY(type == Type::UNIX)) {
      folly::SocketAddress ret;
      auto result = domainSocketPath();
      if (LIKELY(bool(result))) {
        ret.setFromPath(*result);
      }
      return ret;
    }
    return folly::SocketAddress{folly::IPAddressV4::fromLong(ip).toAddr(), port};
  }
  folly::IPAddressV4 toFollyIP() const { return folly::IPAddressV4::fromLong(ip); }

  static Address fromFollyAddress(const folly::SocketAddress &follyAddr, Type type) {
    auto IPAddr = follyAddr.getIPAddress();
    return Address{IPAddr.isV4() ? IPAddr.asV4().toLong() : 0, follyAddr.getPort(), type};
  }

  static Address fromString(std::string_view sv, Type type) {
    Address addr(0, 0, type);
    auto &arr = *reinterpret_cast<std::array<uint8_t, 4> *>(&addr);
    auto r = scn::scan(sv, "{}.{}.{}.{}:{}", arr[0], arr[1], arr[2], arr[3], addr.port);
    return r ? addr : Address{};
  }

  static Address fromString(std::string_view sv) {
    constexpr std::string_view delimiter = "://";
    auto pos = sv.find(delimiter);
    if (pos != sv.npos) {
      auto addressType = sv.substr(0, pos);
      auto opt = magic_enum::enum_cast<Type>(addressType, magic_enum::case_insensitive);
      if (opt) {
        return fromString(sv.substr(pos + delimiter.size()), *opt);
      }
    }
    return fromString(sv, Type::TCP);
  }

  static Result<Address> from(std::string_view sv) {
    auto addr = fromString(sv);
    if (UNLIKELY(!addr)) {
      return makeError(StatusCode::kInvalidFormat, "invalid address: {}", sv);
    }
    return addr;
  }

  static Result<Address> serdeFromReadable(std::string_view sv) { return from(sv); }
};
static_assert(sizeof(Address) == sizeof(uint64_t), "sizeof(Address) != sizeof(uint64_t)");

}  // namespace hf3fs::net

template <>
struct std::hash<hf3fs::net::Address> {
  size_t operator()(const hf3fs::net::Address &addr) const { return folly::hash::twang_mix64(addr); }
};

FMT_BEGIN_NAMESPACE

template <>
struct formatter<hf3fs::net::Address> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(const hf3fs::net::Address &address, FormatContext &ctx) const {
    return formatter<std::string_view>::format(address.str(), ctx);
  }
};

FMT_END_NAMESPACE

/*
 * rapidcsv.h
 *
 * URL:      https://github.com/d99kris/rapidcsv
 * Version:  8.69
 *
 * Copyright (C) 2017-2022 Kristofer Berggren
 * All rights reserved.
 *
 * rapidcsv is distributed under the BSD 3-Clause license, see LICENSE for details.
 *
 */

#pragma once

#include <algorithm>
#include <cassert>
#include <cmath>
#ifdef HAS_CODECVT
#include <codecvt>
#include <locale>
#endif
#include <fstream>
#include <functional>
#include <iostream>
#include <limits>
#include <map>
#include <sstream>
#include <string>
#include <typeinfo>
#include <vector>

#if defined(_MSC_VER)
#include <BaseTsd.h>
typedef SSIZE_T ssize_t;
#endif

namespace rapidcsv {
#if defined(_MSC_VER)
static const bool sPlatformHasCR = true;
#else
static const bool sPlatformHasCR = false;
#endif

/**
 * @brief     Datastructure holding parameters controlling how invalid numbers (including
 *            empty strings) should be handled.
 */
struct ConverterParams {
  /**
   * @brief   Constructor
   * @param   pHasDefaultConverter  specifies if conversion of non-numerical strings shall be
   *                                converted to a default numerical value, instead of causing
   *                                an exception to be thrown (default).
   * @param   pDefaultFloat         floating-point default value to represent invalid numbers.
   * @param   pDefaultInteger       integer default value to represent invalid numbers.
   * @param   pNumericLocale        specifies whether to honor LC_NUMERIC locale (default
   *                                true).
   */
  explicit ConverterParams(const bool pHasDefaultConverter = false,
                           const long double pDefaultFloat = std::numeric_limits<long double>::signaling_NaN(),
                           const long long pDefaultInteger = 0,
                           const bool pNumericLocale = true)
      : mHasDefaultConverter(pHasDefaultConverter),
        mDefaultFloat(pDefaultFloat),
        mDefaultInteger(pDefaultInteger),
        mNumericLocale(pNumericLocale) {}

  /**
   * @brief   specifies if conversion of non-numerical strings shall be converted to a default
   *          numerical value, instead of causing an exception to be thrown (default).
   */
  bool mHasDefaultConverter;

  /**
   * @brief   floating-point default value to represent invalid numbers.
   */
  long double mDefaultFloat;

  /**
   * @brief   integer default value to represent invalid numbers.
   */
  long long mDefaultInteger;

  /**
   * @brief   specifies whether to honor LC_NUMERIC locale.
   */
  bool mNumericLocale;
};

/**
 * @brief     Exception thrown when attempting to access Document data in a datatype which
 *            is not supported by the Converter class.
 */
class no_converter : public std::exception {
  /**
   * @brief   Provides details about the exception
   * @returns an explanatory string
   */
  virtual const char *what() const throw() { return "unsupported conversion datatype"; }
};

/**
 * @brief     Class providing conversion to/from numerical datatypes and strings. Only
 *            intended for rapidcsv internal usage, but exposed externally to allow
 *            specialization for custom datatype conversions.
 */
template <typename T>
class Converter {
 public:
  /**
   * @brief   Constructor
   * @param   pConverterParams      specifies how conversion of non-numerical values to
   *                                numerical datatype shall be handled.
   */
  Converter(const ConverterParams &pConverterParams)
      : mConverterParams(pConverterParams) {}

  /**
   * @brief   Converts numerical value to string representation.
   * @param   pVal                  numerical value
   * @param   pStr                  output string
   */
  void ToStr(const T &pVal, std::string &pStr) const {
    if (typeid(T) == typeid(int) || typeid(T) == typeid(long) || typeid(T) == typeid(long long) ||
        typeid(T) == typeid(unsigned) || typeid(T) == typeid(unsigned long) ||
        typeid(T) == typeid(unsigned long long) || typeid(T) == typeid(float) || typeid(T) == typeid(double) ||
        typeid(T) == typeid(long double) || typeid(T) == typeid(char)) {
      std::ostringstream out;
      out << pVal;
      pStr = out.str();
    } else {
      throw no_converter();
    }
  }

  /**
   * @brief   Converts string holding a numerical value to numerical datatype representation.
   * @param   pVal                  numerical value
   * @param   pStr                  output string
   */
  void ToVal(const std::string &pStr, T &pVal) const {
    try {
      if (typeid(T) == typeid(int)) {
        pVal = static_cast<T>(std::stoi(pStr));
        return;
      } else if (typeid(T) == typeid(long)) {
        pVal = static_cast<T>(std::stol(pStr));
        return;
      } else if (typeid(T) == typeid(long long)) {
        pVal = static_cast<T>(std::stoll(pStr));
        return;
      } else if (typeid(T) == typeid(unsigned)) {
        pVal = static_cast<T>(std::stoul(pStr));
        return;
      } else if (typeid(T) == typeid(unsigned long)) {
        pVal = static_cast<T>(std::stoul(pStr));
        return;
      } else if (typeid(T) == typeid(unsigned long long)) {
        pVal = static_cast<T>(std::stoull(pStr));
        return;
      }
    } catch (...) {
      if (!mConverterParams.mHasDefaultConverter) {
        throw;
      } else {
        pVal = static_cast<T>(mConverterParams.mDefaultInteger);
        return;
      }
    }

    try {
      if (mConverterParams.mNumericLocale) {
        if (typeid(T) == typeid(float)) {
          pVal = static_cast<T>(std::stof(pStr));
          return;
        } else if (typeid(T) == typeid(double)) {
          pVal = static_cast<T>(std::stod(pStr));
          return;
        } else if (typeid(T) == typeid(long double)) {
          pVal = static_cast<T>(std::stold(pStr));
          return;
        }
      } else {
        if ((typeid(T) == typeid(float)) || (typeid(T) == typeid(double)) || (typeid(T) == typeid(long double))) {
          std::istringstream iss(pStr);
          iss >> pVal;
          if (iss.fail() || iss.bad() || !iss.eof()) {
            throw std::invalid_argument("istringstream: no conversion");
          }
          return;
        }
      }
    } catch (...) {
      if (!mConverterParams.mHasDefaultConverter) {
        throw;
      } else {
        pVal = static_cast<T>(mConverterParams.mDefaultFloat);
        return;
      }
    }

    if (typeid(T) == typeid(char)) {
      pVal = static_cast<T>(pStr[0]);
      return;
    } else {
      throw no_converter();
    }
  }

 private:
  const ConverterParams &mConverterParams;
};

/**
 * @brief     Specialized implementation handling string to string conversion.
 * @param     pVal                  string
 * @param     pStr                  string
 */
template <>
inline void Converter<std::string>::ToStr(const std::string &pVal, std::string &pStr) const {
  pStr = pVal;
}

/**
 * @brief     Specialized implementation handling string to string conversion.
 * @param     pVal                  string
 * @param     pStr                  string
 */
template <>
inline void Converter<std::string>::ToVal(const std::string &pStr, std::string &pVal) const {
  pVal = pStr;
}

template <typename T>
using ConvFunc = std::function<void(const std::string &pStr, T &pVal)>;

/**
 * @brief     Datastructure holding parameters controlling which row and column should be
 *            treated as labels.
 */
struct LabelParams {
  /**
   * @brief   Constructor
   * @param   pColumnNameIdx        specifies the zero-based row index of the column labels, setting
   *                                it to -1 prevents column lookup by label name, and gives access
   *                                to all rows as document data. Default: 0
   * @param   pRowNameIdx           specifies the zero-based column index of the row labels, setting
   *                                it to -1 prevents row lookup by label name, and gives access
   *                                to all columns as document data. Default: -1
   */
  explicit LabelParams(const ssize_t pColumnNameIdx = 0, const ssize_t pRowNameIdx = -1)
      : mColumnNameIdx(pColumnNameIdx),
        mRowNameIdx(pRowNameIdx) {
    if (mColumnNameIdx < -1) {
      const std::string errStr = "invalid column name index " + std::to_string(mColumnNameIdx) + " < -1";
      throw std::out_of_range(errStr);
    }

    if (mRowNameIdx < -1) {
      const std::string errStr = "invalid row name index " + std::to_string(mRowNameIdx) + " < -1";
      throw std::out_of_range(errStr);
    }
  }

  /**
   * @brief   specifies the zero-based row index of the column labels.
   */
  ssize_t mColumnNameIdx;

  /**
   * @brief   specifies the zero-based column index of the row labels.
   */
  ssize_t mRowNameIdx;
};

/**
 * @brief     Datastructure holding parameters controlling how the CSV data fields are separated.
 */
struct SeparatorParams {
  /**
   * @brief   Constructor
   * @param   pSeparator            specifies the column separator (default ',').
   * @param   pTrim                 specifies whether to trim leading and trailing spaces from
   *                                cells read (default false).
   * @param   pHasCR                specifies whether a new document (i.e. not an existing document read)
   *                                should use CR/LF instead of only LF (default is to use standard
   *                                behavior of underlying platforms - CR/LF for Win, and LF for others).
   * @param   pQuotedLinebreaks     specifies whether to allow line breaks in quoted text (default false)
   * @param   pAutoQuote            specifies whether to automatically dequote data during read, and add
   *                                quotes during write (default true).
   * @param   pQuoteChar            specifies the quote character (default '\"').
   */
  explicit SeparatorParams(const char pSeparator = ',',
                           const bool pTrim = false,
                           const bool pHasCR = sPlatformHasCR,
                           const bool pQuotedLinebreaks = false,
                           const bool pAutoQuote = true,
                           const char pQuoteChar = '"')
      : mSeparator(pSeparator),
        mTrim(pTrim),
        mHasCR(pHasCR),
        mQuotedLinebreaks(pQuotedLinebreaks),
        mAutoQuote(pAutoQuote),
        mQuoteChar(pQuoteChar) {}

  /**
   * @brief   specifies the column separator.
   */
  char mSeparator;

  /**
   * @brief   specifies whether to trim leading and trailing spaces from cells read.
   */
  bool mTrim;

  /**
   * @brief   specifies whether new documents should use CR/LF instead of LF.
   */
  bool mHasCR;

  /**
   * @brief   specifies whether to allow line breaks in quoted text.
   */
  bool mQuotedLinebreaks;

  /**
   * @brief   specifies whether to automatically dequote cell data.
   */
  bool mAutoQuote;

  /**
   * @brief   specifies the quote character.
   */
  char mQuoteChar;
};

/**
 * @brief     Datastructure holding parameters controlling how special line formats should be
 *            treated.
 */
struct LineReaderParams {
  /**
   * @brief   Constructor
   * @param   pSkipCommentLines     specifies whether to skip lines prefixed with
   *                                mCommentPrefix. Default: false
   * @param   pCommentPrefix        specifies which prefix character to indicate a comment
   *                                line. Default: #
   * @param   pSkipEmptyLines       specifies whether to skip empty lines. Default: false
   */
  explicit LineReaderParams(const bool pSkipCommentLines = false,
                            const char pCommentPrefix = '#',
                            const bool pSkipEmptyLines = false)
      : mSkipCommentLines(pSkipCommentLines),
        mCommentPrefix(pCommentPrefix),
        mSkipEmptyLines(pSkipEmptyLines) {}

  /**
   * @brief   specifies whether to skip lines prefixed with mCommentPrefix.
   */
  bool mSkipCommentLines;

  /**
   * @brief   specifies which prefix character to indicate a comment line.
   */
  char mCommentPrefix;

  /**
   * @brief   specifies whether to skip empty lines.
   */
  bool mSkipEmptyLines;
};

/**
 * @brief     Class representing a CSV document.
 */
class Document {
 public:
  /**
   * @brief   Constructor
   * @param   pPath                 specifies the path of an existing CSV-file to populate the Document
   *                                data with.
   * @param   pLabelParams          specifies which row and column should be treated as labels.
   * @param   pSeparatorParams      specifies which field and row separators should be used.
   * @param   pConverterParams      specifies how invalid numbers (including empty strings) should be
   *                                handled.
   * @param   pLineReaderParams     specifies how special line formats should be treated.
   */
  explicit Document(const std::string &pPath = std::string(),
                    const LabelParams &pLabelParams = LabelParams(),
                    const SeparatorParams &pSeparatorParams = SeparatorParams(),
                    const ConverterParams &pConverterParams = ConverterParams(),
                    const LineReaderParams &pLineReaderParams = LineReaderParams())
      : mPath(pPath),
        mLabelParams(pLabelParams),
        mSeparatorParams(pSeparatorParams),
        mConverterParams(pConverterParams),
        mLineReaderParams(pLineReaderParams),
        mData(),
        mColumnNames(),
        mRowNames() {
    if (!mPath.empty()) {
      ReadCsv();
    }
  }

  /**
   * @brief   Constructor
   * @param   pStream               specifies a binary input stream to read CSV data from.
   * @param   pLabelParams          specifies which row and column should be treated as labels.
   * @param   pSeparatorParams      specifies which field and row separators should be used.
   * @param   pConverterParams      specifies how invalid numbers (including empty strings) should be
   *                                handled.
   * @param   pLineReaderParams     specifies how special line formats should be treated.
   */
  explicit Document(std::istream &pStream,
                    const LabelParams &pLabelParams = LabelParams(),
                    const SeparatorParams &pSeparatorParams = SeparatorParams(),
                    const ConverterParams &pConverterParams = ConverterParams(),
                    const LineReaderParams &pLineReaderParams = LineReaderParams())
      : mPath(),
        mLabelParams(pLabelParams),
        mSeparatorParams(pSeparatorParams),
        mConverterParams(pConverterParams),
        mLineReaderParams(pLineReaderParams),
        mData(),
        mColumnNames(),
        mRowNames() {
    ReadCsv(pStream);
  }

  /**
   * @brief   Read Document data from file.
   * @param   pPath                 specifies the path of an existing CSV-file to populate the Document
   *                                data with.
   * @param   pLabelParams          specifies which row and column should be treated as labels.
   * @param   pSeparatorParams      specifies which field and row separators should be used.
   * @param   pConverterParams      specifies how invalid numbers (including empty strings) should be
   *                                handled.
   * @param   pLineReaderParams     specifies how special line formats should be treated.
   */
  void Load(const std::string &pPath,
            const LabelParams &pLabelParams = LabelParams(),
            const SeparatorParams &pSeparatorParams = SeparatorParams(),
            const ConverterParams &pConverterParams = ConverterParams(),
            const LineReaderParams &pLineReaderParams = LineReaderParams()) {
    mPath = pPath;
    mLabelParams = pLabelParams;
    mSeparatorParams = pSeparatorParams;
    mConverterParams = pConverterParams;
    mLineReaderParams = pLineReaderParams;
    ReadCsv();
  }

  /**
   * @brief   Read Document data from stream.
   * @param   pStream               specifies a binary input stream to read CSV data from.
   * @param   pLabelParams          specifies which row and column should be treated as labels.
   * @param   pSeparatorParams      specifies which field and row separators should be used.
   * @param   pConverterParams      specifies how invalid numbers (including empty strings) should be
   *                                handled.
   * @param   pLineReaderParams     specifies how special line formats should be treated.
   */
  void Load(std::istream &pStream,
            const LabelParams &pLabelParams = LabelParams(),
            const SeparatorParams &pSeparatorParams = SeparatorParams(),
            const ConverterParams &pConverterParams = ConverterParams(),
            const LineReaderParams &pLineReaderParams = LineReaderParams()) {
    mPath = "";
    mLabelParams = pLabelParams;
    mSeparatorParams = pSeparatorParams;
    mConverterParams = pConverterParams;
    mLineReaderParams = pLineReaderParams;
    ReadCsv(pStream);
  }

  /**
   * @brief   Write Document data to file.
   * @param   pPath                 optionally specifies the path where the CSV-file will be created
   *                                (if not specified, the original path provided when creating or
   *                                loading the Document data will be used).
   */
  void Save(const std::string &pPath = std::string()) {
    if (!pPath.empty()) {
      mPath = pPath;
    }
    WriteCsv();
  }

  /**
   * @brief   Write Document data to stream.
   * @param   pStream               specifies a binary output stream to write the data to.
   */
  void Save(std::ostream &pStream) { WriteCsv(pStream); }

  /**
   * @brief   Clears loaded Document data.
   *
   */
  void Clear() {
    mData.clear();
    mColumnNames.clear();
    mRowNames.clear();
#ifdef HAS_CODECVT
    mIsUtf16 = false;
    mIsLE = false;
#endif
  }

  /**
   * @brief   Get column index by name.
   * @param   pColumnName           column label name.
   * @returns zero-based column index.
   */
  ssize_t GetColumnIdx(const std::string &pColumnName) const {
    if (mLabelParams.mColumnNameIdx >= 0) {
      if (mColumnNames.find(pColumnName) != mColumnNames.end()) {
        return static_cast<ssize_t>(mColumnNames.at(pColumnName)) - (mLabelParams.mRowNameIdx + 1);
      }
    }
    return -1;
  }

  /**
   * @brief   Get column by index.
   * @param   pColumnIdx            zero-based column index.
   * @returns vector of column data.
   */
  template <typename T>
  std::vector<T> GetColumn(const size_t pColumnIdx) const {
    const size_t dataColumnIdx = GetDataColumnIndex(pColumnIdx);
    std::vector<T> column;
    Converter<T> converter(mConverterParams);
    for (auto itRow = mData.begin(); itRow != mData.end(); ++itRow) {
      if (std::distance(mData.begin(), itRow) > mLabelParams.mColumnNameIdx) {
        if (dataColumnIdx < itRow->size()) {
          T val;
          converter.ToVal(itRow->at(dataColumnIdx), val);
          column.push_back(val);
        } else {
          const std::string errStr =
              "requested column index " + std::to_string(pColumnIdx) +
              " >= " + std::to_string(itRow->size() - GetDataColumnIndex(0)) + " (number of columns on row index " +
              std::to_string(std::distance(mData.begin(), itRow) - (mLabelParams.mColumnNameIdx + 1)) + ")";
          throw std::out_of_range(errStr);
        }
      }
    }
    return column;
  }

  /**
   * @brief   Get column by index.
   * @param   pColumnIdx            zero-based column index.
   * @param   pToVal                conversion function.
   * @returns vector of column data.
   */
  template <typename T>
  std::vector<T> GetColumn(const size_t pColumnIdx, ConvFunc<T> pToVal) const {
    const size_t dataColumnIdx = GetDataColumnIndex(pColumnIdx);
    std::vector<T> column;
    for (auto itRow = mData.begin(); itRow != mData.end(); ++itRow) {
      if (std::distance(mData.begin(), itRow) > mLabelParams.mColumnNameIdx) {
        T val;
        pToVal(itRow->at(dataColumnIdx), val);
        column.push_back(val);
      }
    }
    return column;
  }

  /**
   * @brief   Get column by name.
   * @param   pColumnName           column label name.
   * @returns vector of column data.
   */
  template <typename T>
  std::vector<T> GetColumn(const std::string &pColumnName) const {
    const ssize_t columnIdx = GetColumnIdx(pColumnName);
    if (columnIdx < 0) {
      throw std::out_of_range("column not found: " + pColumnName);
    }
    return GetColumn<T>(static_cast<size_t>(columnIdx));
  }

  /**
   * @brief   Get column by name.
   * @param   pColumnName           column label name.
   * @param   pToVal                conversion function.
   * @returns vector of column data.
   */
  template <typename T>
  std::vector<T> GetColumn(const std::string &pColumnName, ConvFunc<T> pToVal) const {
    const ssize_t columnIdx = GetColumnIdx(pColumnName);
    if (columnIdx < 0) {
      throw std::out_of_range("column not found: " + pColumnName);
    }
    return GetColumn<T>(static_cast<size_t>(columnIdx), pToVal);
  }

  /**
   * @brief   Set column by index.
   * @param   pColumnIdx            zero-based column index.
   * @param   pColumn               vector of column data.
   */
  template <typename T>
  void SetColumn(const size_t pColumnIdx, const std::vector<T> &pColumn) {
    const size_t dataColumnIdx = GetDataColumnIndex(pColumnIdx);

    while (GetDataRowIndex(pColumn.size()) > GetDataRowCount()) {
      std::vector<std::string> row;
      row.resize(GetDataColumnCount());
      mData.push_back(row);
    }

    if ((dataColumnIdx + 1) > GetDataColumnCount()) {
      for (auto itRow = mData.begin(); itRow != mData.end(); ++itRow) {
        itRow->resize(GetDataColumnIndex(dataColumnIdx + 1));
      }
    }

    Converter<T> converter(mConverterParams);
    for (auto itRow = pColumn.begin(); itRow != pColumn.end(); ++itRow) {
      std::string str;
      converter.ToStr(*itRow, str);
      mData.at(static_cast<size_t>(std::distance(pColumn.begin(), itRow) + mLabelParams.mColumnNameIdx + 1))
          .at(dataColumnIdx) = str;
    }
  }

  /**
   * @brief   Set column by name.
   * @param   pColumnName           column label name.
   * @param   pColumn               vector of column data.
   */
  template <typename T>
  void SetColumn(const std::string &pColumnName, const std::vector<T> &pColumn) {
    const ssize_t columnIdx = GetColumnIdx(pColumnName);
    if (columnIdx < 0) {
      throw std::out_of_range("column not found: " + pColumnName);
    }
    SetColumn<T>(static_cast<size_t>(columnIdx), pColumn);
  }

  /**
   * @brief   Remove column by index.
   * @param   pColumnIdx            zero-based column index.
   */
  void RemoveColumn(const size_t pColumnIdx) {
    const size_t dataColumnIdx = GetDataColumnIndex(pColumnIdx);
    for (auto itRow = mData.begin(); itRow != mData.end(); ++itRow) {
      itRow->erase(itRow->begin() + static_cast<ssize_t>(dataColumnIdx));
    }

    UpdateColumnNames();
  }

  /**
   * @brief   Remove column by name.
   * @param   pColumnName           column label name.
   */
  void RemoveColumn(const std::string &pColumnName) {
    ssize_t columnIdx = GetColumnIdx(pColumnName);
    if (columnIdx < 0) {
      throw std::out_of_range("column not found: " + pColumnName);
    }

    RemoveColumn(static_cast<size_t>(columnIdx));
  }

  /**
   * @brief   Insert column at specified index.
   * @param   pColumnIdx            zero-based column index.
   * @param   pColumn               vector of column data (optional argument).
   * @param   pColumnName           column label name (optional argument).
   */
  template <typename T>
  void InsertColumn(const size_t pColumnIdx,
                    const std::vector<T> &pColumn = std::vector<T>(),
                    const std::string &pColumnName = std::string()) {
    const size_t dataColumnIdx = GetDataColumnIndex(pColumnIdx);

    std::vector<std::string> column;
    if (pColumn.empty()) {
      column.resize(GetDataRowCount());
    } else {
      column.resize(GetDataRowIndex(pColumn.size()));
      Converter<T> converter(mConverterParams);
      for (auto itRow = pColumn.begin(); itRow != pColumn.end(); ++itRow) {
        std::string str;
        converter.ToStr(*itRow, str);
        const size_t rowIdx =
            static_cast<size_t>(std::distance(pColumn.begin(), itRow) + (mLabelParams.mColumnNameIdx + 1));
        column.at(rowIdx) = str;
      }
    }

    while (column.size() > GetDataRowCount()) {
      std::vector<std::string> row;
      const size_t columnCount =
          std::max<size_t>(static_cast<size_t>(mLabelParams.mColumnNameIdx + 1), GetDataColumnCount());
      row.resize(columnCount);
      mData.push_back(row);
    }

    for (auto itRow = mData.begin(); itRow != mData.end(); ++itRow) {
      const size_t rowIdx = static_cast<size_t>(std::distance(mData.begin(), itRow));
      itRow->insert(itRow->begin() + static_cast<ssize_t>(dataColumnIdx), column.at(rowIdx));
    }

    if (!pColumnName.empty()) {
      SetColumnName(pColumnIdx, pColumnName);
    }

    UpdateColumnNames();
  }

  /**
   * @brief   Get number of data columns (excluding label columns).
   * @returns column count.
   */
  size_t GetColumnCount() const {
    const ssize_t count =
        static_cast<ssize_t>((mData.size() > 0) ? mData.at(0).size() : 0) - (mLabelParams.mRowNameIdx + 1);
    return (count >= 0) ? static_cast<size_t>(count) : 0;
  }

  /**
   * @brief   Get row index by name.
   * @param   pRowName              row label name.
   * @returns zero-based row index.
   */
  ssize_t GetRowIdx(const std::string &pRowName) const {
    if (mLabelParams.mRowNameIdx >= 0) {
      if (mRowNames.find(pRowName) != mRowNames.end()) {
        return static_cast<ssize_t>(mRowNames.at(pRowName)) - (mLabelParams.mColumnNameIdx + 1);
      }
    }
    return -1;
  }

  /**
   * @brief   Get row by index.
   * @param   pRowIdx               zero-based row index.
   * @returns vector of row data.
   */
  template <typename T>
  std::vector<T> GetRow(const size_t pRowIdx) const {
    const size_t dataRowIdx = GetDataRowIndex(pRowIdx);
    std::vector<T> row;
    Converter<T> converter(mConverterParams);
    for (auto itCol = mData.at(dataRowIdx).begin(); itCol != mData.at(dataRowIdx).end(); ++itCol) {
      if (std::distance(mData.at(dataRowIdx).begin(), itCol) > mLabelParams.mRowNameIdx) {
        T val;
        converter.ToVal(*itCol, val);
        row.push_back(val);
      }
    }
    return row;
  }

  /**
   * @brief   Get row by index.
   * @param   pRowIdx               zero-based row index.
   * @param   pToVal                conversion function.
   * @returns vector of row data.
   */
  template <typename T>
  std::vector<T> GetRow(const size_t pRowIdx, ConvFunc<T> pToVal) const {
    const size_t dataRowIdx = GetDataRowIndex(pRowIdx);
    std::vector<T> row;
    Converter<T> converter(mConverterParams);
    for (auto itCol = mData.at(dataRowIdx).begin(); itCol != mData.at(dataRowIdx).end(); ++itCol) {
      if (std::distance(mData.at(dataRowIdx).begin(), itCol) > mLabelParams.mRowNameIdx) {
        T val;
        pToVal(*itCol, val);
        row.push_back(val);
      }
    }
    return row;
  }

  /**
   * @brief   Get row by name.
   * @param   pRowName              row label name.
   * @returns vector of row data.
   */
  template <typename T>
  std::vector<T> GetRow(const std::string &pRowName) const {
    ssize_t rowIdx = GetRowIdx(pRowName);
    if (rowIdx < 0) {
      throw std::out_of_range("row not found: " + pRowName);
    }
    return GetRow<T>(static_cast<size_t>(rowIdx));
  }

  /**
   * @brief   Get row by name.
   * @param   pRowName              row label name.
   * @param   pToVal                conversion function.
   * @returns vector of row data.
   */
  template <typename T>
  std::vector<T> GetRow(const std::string &pRowName, ConvFunc<T> pToVal) const {
    ssize_t rowIdx = GetRowIdx(pRowName);
    if (rowIdx < 0) {
      throw std::out_of_range("row not found: " + pRowName);
    }
    return GetRow<T>(static_cast<size_t>(rowIdx), pToVal);
  }

  /**
   * @brief   Set row by index.
   * @param   pRowIdx               zero-based row index.
   * @param   pRow                  vector of row data.
   */
  template <typename T>
  void SetRow(const size_t pRowIdx, const std::vector<T> &pRow) {
    const size_t dataRowIdx = GetDataRowIndex(pRowIdx);

    while ((dataRowIdx + 1) > GetDataRowCount()) {
      std::vector<std::string> row;
      row.resize(GetDataColumnCount());
      mData.push_back(row);
    }

    if (pRow.size() > GetDataColumnCount()) {
      for (auto itRow = mData.begin(); itRow != mData.end(); ++itRow) {
        itRow->resize(GetDataColumnIndex(pRow.size()));
      }
    }

    Converter<T> converter(mConverterParams);
    for (auto itCol = pRow.begin(); itCol != pRow.end(); ++itCol) {
      std::string str;
      converter.ToStr(*itCol, str);
      mData.at(dataRowIdx).at(static_cast<size_t>(std::distance(pRow.begin(), itCol) + mLabelParams.mRowNameIdx + 1)) =
          str;
    }
  }

  /**
   * @brief   Set row by name.
   * @param   pRowName              row label name.
   * @param   pRow                  vector of row data.
   */
  template <typename T>
  void SetRow(const std::string &pRowName, const std::vector<T> &pRow) {
    ssize_t rowIdx = GetRowIdx(pRowName);
    if (rowIdx < 0) {
      throw std::out_of_range("row not found: " + pRowName);
    }
    return SetRow<T>(static_cast<size_t>(rowIdx), pRow);
  }

  /**
   * @brief   Remove row by index.
   * @param   pRowIdx               zero-based row index.
   */
  void RemoveRow(const size_t pRowIdx) {
    const size_t dataRowIdx = GetDataRowIndex(pRowIdx);
    mData.erase(mData.begin() + static_cast<ssize_t>(dataRowIdx));
    UpdateRowNames();
  }

  /**
   * @brief   Remove row by name.
   * @param   pRowName              row label name.
   */
  void RemoveRow(const std::string &pRowName) {
    ssize_t rowIdx = GetRowIdx(pRowName);
    if (rowIdx < 0) {
      throw std::out_of_range("row not found: " + pRowName);
    }

    RemoveRow(static_cast<size_t>(rowIdx));
  }

  /**
   * @brief   Insert row at specified index.
   * @param   pRowIdx               zero-based row index.
   * @param   pRow                  vector of row data (optional argument).
   * @param   pRowName              row label name (optional argument).
   */
  template <typename T>
  void InsertRow(const size_t pRowIdx,
                 const std::vector<T> &pRow = std::vector<T>(),
                 const std::string &pRowName = std::string()) {
    const size_t rowIdx = GetDataRowIndex(pRowIdx);

    std::vector<std::string> row;
    if (pRow.empty()) {
      row.resize(GetDataColumnCount());
    } else {
      row.resize(GetDataColumnIndex(pRow.size()));
      Converter<T> converter(mConverterParams);
      for (auto itCol = pRow.begin(); itCol != pRow.end(); ++itCol) {
        std::string str;
        converter.ToStr(*itCol, str);
        row.at(static_cast<size_t>(std::distance(pRow.begin(), itCol) + mLabelParams.mRowNameIdx + 1)) = str;
      }
    }

    while (rowIdx > GetDataRowCount()) {
      std::vector<std::string> tempRow;
      tempRow.resize(GetDataColumnCount());
      mData.push_back(tempRow);
    }

    mData.insert(mData.begin() + static_cast<ssize_t>(rowIdx), row);

    if (!pRowName.empty()) {
      SetRowName(pRowIdx, pRowName);
    }

    UpdateRowNames();
  }

  /**
   * @brief   Get number of data rows (excluding label rows).
   * @returns row count.
   */
  size_t GetRowCount() const {
    const ssize_t count = static_cast<ssize_t>(mData.size()) - (mLabelParams.mColumnNameIdx + 1);
    return (count >= 0) ? static_cast<size_t>(count) : 0;
  }

  /**
   * @brief   Get cell by index.
   * @param   pColumnIdx            zero-based column index.
   * @param   pRowIdx               zero-based row index.
   * @returns cell data.
   */
  template <typename T>
  T GetCell(const size_t pColumnIdx, const size_t pRowIdx) const {
    const size_t dataColumnIdx = GetDataColumnIndex(pColumnIdx);
    const size_t dataRowIdx = GetDataRowIndex(pRowIdx);

    T val;
    Converter<T> converter(mConverterParams);
    converter.ToVal(mData.at(dataRowIdx).at(dataColumnIdx), val);
    return val;
  }

  /**
   * @brief   Get cell by index.
   * @param   pColumnIdx            zero-based column index.
   * @param   pRowIdx               zero-based row index.
   * @param   pToVal                conversion function.
   * @returns cell data.
   */
  template <typename T>
  T GetCell(const size_t pColumnIdx, const size_t pRowIdx, ConvFunc<T> pToVal) const {
    const size_t dataColumnIdx = GetDataColumnIndex(pColumnIdx);
    const size_t dataRowIdx = GetDataRowIndex(pRowIdx);

    T val;
    pToVal(mData.at(dataRowIdx).at(dataColumnIdx), val);
    return val;
  }

  /**
   * @brief   Get cell by name.
   * @param   pColumnName           column label name.
   * @param   pRowName              row label name.
   * @returns cell data.
   */
  template <typename T>
  T GetCell(const std::string &pColumnName, const std::string &pRowName) const {
    const ssize_t columnIdx = GetColumnIdx(pColumnName);
    if (columnIdx < 0) {
      throw std::out_of_range("column not found: " + pColumnName);
    }

    const ssize_t rowIdx = GetRowIdx(pRowName);
    if (rowIdx < 0) {
      throw std::out_of_range("row not found: " + pRowName);
    }

    return GetCell<T>(static_cast<size_t>(columnIdx), static_cast<size_t>(rowIdx));
  }

  /**
   * @brief   Get cell by name.
   * @param   pColumnName           column label name.
   * @param   pRowName              row label name.
   * @param   pToVal                conversion function.
   * @returns cell data.
   */
  template <typename T>
  T GetCell(const std::string &pColumnName, const std::string &pRowName, ConvFunc<T> pToVal) const {
    const ssize_t columnIdx = GetColumnIdx(pColumnName);
    if (columnIdx < 0) {
      throw std::out_of_range("column not found: " + pColumnName);
    }

    const ssize_t rowIdx = GetRowIdx(pRowName);
    if (rowIdx < 0) {
      throw std::out_of_range("row not found: " + pRowName);
    }

    return GetCell<T>(static_cast<size_t>(columnIdx), static_cast<size_t>(rowIdx), pToVal);
  }

  /**
   * @brief   Get cell by column name and row index.
   * @param   pColumnName           column label name.
   * @param   pRowIdx               zero-based row index.
   * @returns cell data.
   */
  template <typename T>
  T GetCell(const std::string &pColumnName, const size_t pRowIdx) const {
    const ssize_t columnIdx = GetColumnIdx(pColumnName);
    if (columnIdx < 0) {
      throw std::out_of_range("column not found: " + pColumnName);
    }

    return GetCell<T>(static_cast<size_t>(columnIdx), pRowIdx);
  }

  /**
   * @brief   Get cell by column name and row index.
   * @param   pColumnName           column label name.
   * @param   pRowIdx               zero-based row index.
   * @param   pToVal                conversion function.
   * @returns cell data.
   */
  template <typename T>
  T GetCell(const std::string &pColumnName, const size_t pRowIdx, ConvFunc<T> pToVal) const {
    const ssize_t columnIdx = GetColumnIdx(pColumnName);
    if (columnIdx < 0) {
      throw std::out_of_range("column not found: " + pColumnName);
    }

    return GetCell<T>(static_cast<size_t>(columnIdx), pRowIdx, pToVal);
  }

  /**
   * @brief   Get cell by column index and row name.
   * @param   pColumnIdx            zero-based column index.
   * @param   pRowName              row label name.
   * @returns cell data.
   */
  template <typename T>
  T GetCell(const size_t pColumnIdx, const std::string &pRowName) const {
    const ssize_t rowIdx = GetRowIdx(pRowName);
    if (rowIdx < 0) {
      throw std::out_of_range("row not found: " + pRowName);
    }

    return GetCell<T>(pColumnIdx, static_cast<size_t>(rowIdx));
  }

  /**
   * @brief   Get cell by column index and row name.
   * @param   pColumnIdx            zero-based column index.
   * @param   pRowName              row label name.
   * @param   pToVal                conversion function.
   * @returns cell data.
   */
  template <typename T>
  T GetCell(const size_t pColumnIdx, const std::string &pRowName, ConvFunc<T> pToVal) const {
    const ssize_t rowIdx = GetRowIdx(pRowName);
    if (rowIdx < 0) {
      throw std::out_of_range("row not found: " + pRowName);
    }

    return GetCell<T>(pColumnIdx, static_cast<size_t>(rowIdx), pToVal);
  }

  /**
   * @brief   Set cell by index.
   * @param   pRowIdx               zero-based row index.
   * @param   pColumnIdx            zero-based column index.
   * @param   pCell                 cell data.
   */
  template <typename T>
  void SetCell(const size_t pColumnIdx, const size_t pRowIdx, const T &pCell) {
    const size_t dataColumnIdx = GetDataColumnIndex(pColumnIdx);
    const size_t dataRowIdx = GetDataRowIndex(pRowIdx);

    while ((dataRowIdx + 1) > GetDataRowCount()) {
      std::vector<std::string> row;
      row.resize(GetDataColumnCount());
      mData.push_back(row);
    }

    if ((dataColumnIdx + 1) > GetDataColumnCount()) {
      for (auto itRow = mData.begin(); itRow != mData.end(); ++itRow) {
        itRow->resize(dataColumnIdx + 1);
      }
    }

    std::string str;
    Converter<T> converter(mConverterParams);
    converter.ToStr(pCell, str);
    mData.at(dataRowIdx).at(dataColumnIdx) = str;
  }

  /**
   * @brief   Set cell by name.
   * @param   pColumnName           column label name.
   * @param   pRowName              row label name.
   * @param   pCell                 cell data.
   */
  template <typename T>
  void SetCell(const std::string &pColumnName, const std::string &pRowName, const T &pCell) {
    const ssize_t columnIdx = GetColumnIdx(pColumnName);
    if (columnIdx < 0) {
      throw std::out_of_range("column not found: " + pColumnName);
    }

    const ssize_t rowIdx = GetRowIdx(pRowName);
    if (rowIdx < 0) {
      throw std::out_of_range("row not found: " + pRowName);
    }

    SetCell<T>(static_cast<size_t>(columnIdx), static_cast<size_t>(rowIdx), pCell);
  }

  /**
   * @brief   Get column name
   * @param   pColumnIdx            zero-based column index.
   * @returns column name.
   */
  std::string GetColumnName(const size_t pColumnIdx) {
    const size_t dataColumnIdx = GetDataColumnIndex(pColumnIdx);
    if (mLabelParams.mColumnNameIdx < 0) {
      throw std::out_of_range("column name row index < 0: " + std::to_string(mLabelParams.mColumnNameIdx));
    }

    return mData.at(static_cast<size_t>(mLabelParams.mColumnNameIdx)).at(dataColumnIdx);
  }

  /**
   * @brief   Set column name
   * @param   pColumnIdx            zero-based column index.
   * @param   pColumnName           column name.
   */
  void SetColumnName(size_t pColumnIdx, const std::string &pColumnName) {
    if (mLabelParams.mColumnNameIdx < 0) {
      throw std::out_of_range("column name row index < 0: " + std::to_string(mLabelParams.mColumnNameIdx));
    }

    const size_t dataColumnIdx = GetDataColumnIndex(pColumnIdx);
    mColumnNames[pColumnName] = dataColumnIdx;

    // increase table size if necessary:
    const size_t rowIdx = static_cast<size_t>(mLabelParams.mColumnNameIdx);
    if (rowIdx >= mData.size()) {
      mData.resize(rowIdx + 1);
    }
    auto &row = mData[rowIdx];
    if (dataColumnIdx >= row.size()) {
      row.resize(dataColumnIdx + 1);
    }

    mData.at(static_cast<size_t>(mLabelParams.mColumnNameIdx)).at(dataColumnIdx) = pColumnName;
  }

  /**
   * @brief   Get column names
   * @returns vector of column names.
   */
  std::vector<std::string> GetColumnNames() {
    if (mLabelParams.mColumnNameIdx >= 0) {
      return std::vector<std::string>(
          mData.at(static_cast<size_t>(mLabelParams.mColumnNameIdx)).begin() + (mLabelParams.mRowNameIdx + 1),
          mData.at(static_cast<size_t>(mLabelParams.mColumnNameIdx)).end());
    }

    return std::vector<std::string>();
  }

  /**
   * @brief   Get row name
   * @param   pRowIdx               zero-based column index.
   * @returns row name.
   */
  std::string GetRowName(const size_t pRowIdx) {
    const size_t dataRowIdx = GetDataRowIndex(pRowIdx);
    if (mLabelParams.mRowNameIdx < 0) {
      throw std::out_of_range("row name column index < 0: " + std::to_string(mLabelParams.mRowNameIdx));
    }

    return mData.at(dataRowIdx).at(static_cast<size_t>(mLabelParams.mRowNameIdx));
  }

  /**
   * @brief   Set row name
   * @param   pRowIdx               zero-based row index.
   * @param   pRowName              row name.
   */
  void SetRowName(size_t pRowIdx, const std::string &pRowName) {
    const size_t dataRowIdx = GetDataRowIndex(pRowIdx);
    mRowNames[pRowName] = dataRowIdx;
    if (mLabelParams.mRowNameIdx < 0) {
      throw std::out_of_range("row name column index < 0: " + std::to_string(mLabelParams.mRowNameIdx));
    }

    // increase table size if necessary:
    if (dataRowIdx >= mData.size()) {
      mData.resize(dataRowIdx + 1);
    }
    auto &row = mData[dataRowIdx];
    if (mLabelParams.mRowNameIdx >= static_cast<ssize_t>(row.size())) {
      row.resize(static_cast<size_t>(mLabelParams.mRowNameIdx) + 1);
    }

    mData.at(dataRowIdx).at(static_cast<size_t>(mLabelParams.mRowNameIdx)) = pRowName;
  }

  /**
   * @brief   Get row names
   * @returns vector of row names.
   */
  std::vector<std::string> GetRowNames() {
    std::vector<std::string> rownames;
    if (mLabelParams.mRowNameIdx >= 0) {
      for (auto itRow = mData.begin(); itRow != mData.end(); ++itRow) {
        if (std::distance(mData.begin(), itRow) > mLabelParams.mColumnNameIdx) {
          rownames.push_back(itRow->at(static_cast<size_t>(mLabelParams.mRowNameIdx)));
        }
      }
    }
    return rownames;
  }

 private:
  void ReadCsv() {
    std::ifstream stream;
    stream.exceptions(std::ifstream::failbit | std::ifstream::badbit);
    stream.open(mPath, std::ios::binary);
    ReadCsv(stream);
  }

  void ReadCsv(std::istream &pStream) {
    Clear();
    pStream.seekg(0, std::ios::end);
    std::streamsize length = pStream.tellg();
    pStream.seekg(0, std::ios::beg);

#ifdef HAS_CODECVT
    std::vector<char> bom2b(2, '\0');
    if (length >= 2) {
      pStream.read(bom2b.data(), 2);
      pStream.seekg(0, std::ios::beg);
    }

    static const std::vector<char> bomU16le = {'\xff', '\xfe'};
    static const std::vector<char> bomU16be = {'\xfe', '\xff'};
    if ((bom2b == bomU16le) || (bom2b == bomU16be)) {
      mIsUtf16 = true;
      mIsLE = (bom2b == bomU16le);

      std::wifstream wstream;
      wstream.exceptions(std::wifstream::failbit | std::wifstream::badbit);
      wstream.open(mPath, std::ios::binary);
      if (mIsLE) {
        wstream.imbue(std::locale(
            wstream.getloc(),
            new std::codecvt_utf16<wchar_t,
                                   0x10ffff,
                                   static_cast<std::codecvt_mode>(std::consume_header | std::little_endian)>));
      } else {
        wstream.imbue(std::locale(wstream.getloc(), new std::codecvt_utf16<wchar_t, 0x10ffff, std::consume_header>));
      }
      std::wstringstream wss;
      wss << wstream.rdbuf();
      std::string utf8 = ToString(wss.str());
      std::stringstream ss(utf8);
      ParseCsv(ss, static_cast<std::streamsize>(utf8.size()));
    } else
#endif
    {
      // check for UTF-8 Byte order mark and skip it when found
      if (length >= 3) {
        std::vector<char> bom3b(3, '\0');
        pStream.read(bom3b.data(), 3);
        static const std::vector<char> bomU8 = {'\xef', '\xbb', '\xbf'};
        if (bom3b != bomU8) {
          // file does not start with a UTF-8 Byte order mark
          pStream.seekg(0, std::ios::beg);
        } else {
          // file did start with a UTF-8 Byte order mark, simply skip it
          length -= 3;
        }
      }

      ParseCsv(pStream, length);
    }
  }

  void ParseCsv(std::istream &pStream, std::streamsize p_FileLength) {
    const std::streamsize bufLength = 64 * 1024;
    std::vector<char> buffer(bufLength);
    std::vector<std::string> row;
    std::string cell;
    bool quoted = false;
    int cr = 0;
    int lf = 0;

    while (p_FileLength > 0) {
      const std::streamsize toReadLength = std::min<std::streamsize>(p_FileLength, bufLength);
      pStream.read(buffer.data(), toReadLength);

      // With user-specified istream opened in non-binary mode on windows, we may have a
      // data length mismatch, so ensure we don't parse outside actual data length read.
      const std::streamsize readLength = pStream.gcount();
      if (readLength <= 0) {
        break;
      }

      for (size_t i = 0; i < static_cast<size_t>(readLength); ++i) {
        if (buffer[i] == mSeparatorParams.mQuoteChar) {
          if (cell.empty() || (cell[0] == mSeparatorParams.mQuoteChar)) {
            quoted = !quoted;
          }
          cell += buffer[i];
        } else if (buffer[i] == mSeparatorParams.mSeparator) {
          if (!quoted) {
            row.push_back(Unquote(Trim(cell)));
            cell.clear();
          } else {
            cell += buffer[i];
          }
        } else if (buffer[i] == '\r') {
          if (mSeparatorParams.mQuotedLinebreaks && quoted) {
            cell += buffer[i];
          } else {
            ++cr;
          }
        } else if (buffer[i] == '\n') {
          if (mSeparatorParams.mQuotedLinebreaks && quoted) {
            cell += buffer[i];
          } else {
            ++lf;
            if (mLineReaderParams.mSkipEmptyLines && row.empty() && cell.empty()) {
              // skip empty line
            } else {
              row.push_back(Unquote(Trim(cell)));

              if (mLineReaderParams.mSkipCommentLines && !row.at(0).empty() &&
                  (row.at(0)[0] == mLineReaderParams.mCommentPrefix)) {
                // skip comment line
              } else {
                mData.push_back(row);
              }

              cell.clear();
              row.clear();
              quoted = false;
            }
          }
        } else {
          cell += buffer[i];
        }
      }
      p_FileLength -= readLength;
    }

    // Handle last line without linebreak
    if (!cell.empty() || !row.empty()) {
      row.push_back(Unquote(Trim(cell)));
      cell.clear();
      mData.push_back(row);
      row.clear();
    }

    // Assume CR/LF if at least half the linebreaks have CR
    mSeparatorParams.mHasCR = (cr > (lf / 2));

    // Set up column labels
    UpdateColumnNames();

    // Set up row labels
    UpdateRowNames();
  }

  void WriteCsv() const {
#ifdef HAS_CODECVT
    if (mIsUtf16) {
      std::stringstream ss;
      WriteCsv(ss);
      std::string utf8 = ss.str();
      std::wstring wstr = ToWString(utf8);

      std::wofstream wstream;
      wstream.exceptions(std::wofstream::failbit | std::wofstream::badbit);
      wstream.open(mPath, std::ios::binary | std::ios::trunc);

      if (mIsLE) {
        wstream.imbue(
            std::locale(wstream.getloc(),
                        new std::codecvt_utf16<wchar_t, 0x10ffff, static_cast<std::codecvt_mode>(std::little_endian)>));
      } else {
        wstream.imbue(std::locale(wstream.getloc(), new std::codecvt_utf16<wchar_t, 0x10ffff>));
      }

      wstream << static_cast<wchar_t>(0xfeff);
      wstream << wstr;
    } else
#endif
    {
      std::ofstream stream;
      stream.exceptions(std::ofstream::failbit | std::ofstream::badbit);
      stream.open(mPath, std::ios::binary | std::ios::trunc);
      WriteCsv(stream);
    }
  }

  void WriteCsv(std::ostream &pStream) const {
    for (auto itr = mData.begin(); itr != mData.end(); ++itr) {
      for (auto itc = itr->begin(); itc != itr->end(); ++itc) {
        if (mSeparatorParams.mAutoQuote &&
            ((itc->find(mSeparatorParams.mSeparator) != std::string::npos) || (itc->find(' ') != std::string::npos))) {
          // escape quotes in string
          std::string str = *itc;
          const std::string quoteCharStr = std::string(1, mSeparatorParams.mQuoteChar);
          ReplaceString(str, quoteCharStr, quoteCharStr + quoteCharStr);

          pStream << quoteCharStr << str << quoteCharStr;
        } else {
          pStream << *itc;
        }

        if (std::distance(itc, itr->end()) > 1) {
          pStream << mSeparatorParams.mSeparator;
        }
      }
      pStream << (mSeparatorParams.mHasCR ? "\r\n" : "\n");
    }
  }

  size_t GetDataRowCount() const { return mData.size(); }

  size_t GetDataColumnCount() const { return (mData.size() > 0) ? mData.at(0).size() : 0; }

  inline size_t GetDataRowIndex(const size_t pRowIdx) const {
    return pRowIdx + static_cast<size_t>(mLabelParams.mColumnNameIdx + 1);
  }

  inline size_t GetDataColumnIndex(const size_t pColumnIdx) const {
    return pColumnIdx + static_cast<size_t>(mLabelParams.mRowNameIdx + 1);
  }

  std::string Trim(const std::string &pStr) {
    if (mSeparatorParams.mTrim) {
      std::string str = pStr;

      // ltrim
      str.erase(str.begin(), std::find_if(str.begin(), str.end(), [](int ch) { return !isspace(ch); }));

      // rtrim
      str.erase(std::find_if(str.rbegin(), str.rend(), [](int ch) { return !isspace(ch); }).base(), str.end());

      return str;
    } else {
      return pStr;
    }
  }

  std::string Unquote(const std::string &pStr) {
    if (mSeparatorParams.mAutoQuote && (pStr.size() >= 2) && (pStr.front() == mSeparatorParams.mQuoteChar) &&
        (pStr.back() == mSeparatorParams.mQuoteChar)) {
      // remove start/end quotes
      std::string str = pStr.substr(1, pStr.size() - 2);

      // unescape quotes in string
      const std::string quoteCharStr = std::string(1, mSeparatorParams.mQuoteChar);
      ReplaceString(str, quoteCharStr + quoteCharStr, quoteCharStr);

      return str;
    } else {
      return pStr;
    }
  }

  void UpdateColumnNames() {
    mColumnNames.clear();
    if ((mLabelParams.mColumnNameIdx >= 0) && (static_cast<ssize_t>(mData.size()) > mLabelParams.mColumnNameIdx)) {
      size_t i = 0;
      for (auto &columnName : mData[static_cast<size_t>(mLabelParams.mColumnNameIdx)]) {
        mColumnNames[columnName] = i++;
      }
    }
  }

  void UpdateRowNames() {
    mRowNames.clear();
    if ((mLabelParams.mRowNameIdx >= 0) && (static_cast<ssize_t>(mData.size()) > (mLabelParams.mColumnNameIdx + 1))) {
      size_t i = 0;
      for (auto &dataRow : mData) {
        if (static_cast<ssize_t>(dataRow.size()) > mLabelParams.mRowNameIdx) {
          mRowNames[dataRow[static_cast<size_t>(mLabelParams.mRowNameIdx)]] = i++;
        }
      }
    }
  }

#ifdef HAS_CODECVT
#if defined(_MSC_VER)
#pragma warning(push)
#pragma warning(disable : 4996)
#endif
  static std::string ToString(const std::wstring &pWStr) {
    return std::wstring_convert<std::codecvt_utf8<wchar_t>, wchar_t>{}.to_bytes(pWStr);
  }

  static std::wstring ToWString(const std::string &pStr) {
    return std::wstring_convert<std::codecvt_utf8<wchar_t>, wchar_t>{}.from_bytes(pStr);
  }
#if defined(_MSC_VER)
#pragma warning(pop)
#endif
#endif

  static void ReplaceString(std::string &pStr, const std::string &pSearch, const std::string &pReplace) {
    size_t pos = 0;

    while ((pos = pStr.find(pSearch, pos)) != std::string::npos) {
      pStr.replace(pos, pSearch.size(), pReplace);
      pos += pReplace.size();
    }
  }

 private:
  std::string mPath;
  LabelParams mLabelParams;
  SeparatorParams mSeparatorParams;
  ConverterParams mConverterParams;
  LineReaderParams mLineReaderParams;
  std::vector<std::vector<std::string>> mData;
  std::map<std::string, size_t> mColumnNames;
  std::map<std::string, size_t> mRowNames;
#ifdef HAS_CODECVT
  bool mIsUtf16 = false;
  bool mIsLE = false;
#endif
};
}  // namespace rapidcsv

#pragma once

#include <forward_list>
#include <shared_mutex>

#include "PerProcTable.h"
#include "client/storage/StorageClient.h"
#include "common/utils/Path.h"
#include "fbs/meta/Schema.h"

namespace hf3fs::lib {

struct IorAttrs {
  int priority = 1;
  Duration timeout{Duration::zero()};
  uint64_t flags = 0;
};

struct ShmBuf {
  ShmBuf(const Path &p, size_t sz, size_t bsz, int numa, meta::Uid u, int pid, int ppid);
  ShmBuf(const Path &p, off_t o, size_t sz, size_t bsz, Uuid u);
  ~ShmBuf();

  CoTask<void> registerForIO(folly::Executor::KeepAlive<> exec,
                             storage::client::StorageClient &sc,
                             std::function<void()> &&recordMetrics);
  CoTask<std::shared_ptr<storage::client::IOBuffer>> memh(size_t off);

  bool checkId(const Uuid &uid) const { return id == uid; }

  CoTask<void> deregisterForIO();
  void unmapBuf();
  bool maybeUnlinkShm() {
    if (owner_) {
      shm_unlink(path.c_str());
    }

    return owner_;
  }

  Path path;
  uint8_t *bufStart{nullptr};
  size_t size{0};
  size_t blockSize{0};

  // for client lib
  off_t off;

  // for global shm to do acl
  meta::Uid user{0};
  // for global shm to be freed after owning process is gone
  int pid{0};
  int ppid{0};
  Uuid id;

  // for fuse
  std::string key;
  int iorIndex = -1;
  bool isIoRing = false;
  bool forRead = true;
  int ioDepth = 0;
  std::optional<IorAttrs> iora;

 private:
  void mapBuf();

 private:
  bool owner_;
  int numaNode_;
  //  int fd_;

  // for client agent
  std::vector<folly::atomic_shared_ptr<storage::client::IOBuffer>> memhs_;
  folly::coro::Baton memhBaton_;
  std::atomic<bool> regging_;
};

class ShmBufForIO {
 public:
  ShmBufForIO(std::shared_ptr<ShmBuf> buf, off_t off)
      : buf_(std::move(buf)),
        off_(off) {}
  uint8_t *ptr() const {
    XLOGF(DBG, "buf start {} off {} ptr {}", (void *)buf_->bufStart, off_, (void *)(buf_->bufStart + off_));
    return buf_->bufStart + off_;
  }
  CoTryTask<storage::client::IOBuffer *> memh(size_t len) const {
    XLOGF(DBG, "shm buf for io off {} buf ptr {}", off_, (void *)buf_.get());
    XLOGF(DBG, "shm block size {}", buf_->blockSize);
    if (len && off_ / buf_->blockSize != (off_ + len - 1) / buf_->blockSize) {
      co_return makeError(StatusCode::kInvalidArg);
    }
    co_return (co_await buf_->memh(off_)).get();
  }

 private:
  std::shared_ptr<ShmBuf> buf_;
  off_t off_;
};

using ProcShmBuf = AllProcMap<ShmBuf, StatusCode::kInvalidArg, StatusCode::kNotEnoughMemory>;
}  // namespace hf3fs::lib

#pragma once

#include <array>
#include <atomic>
#include <cstdint>
#include <cstring>
#include <folly/Synchronized.h>
#include <folly/lang/Bits.h>
#include <folly/logging/xlog.h>
#include <limits>
#include <map>
#include <mutex>
#include <optional>
#include <queue>
#include <shared_mutex>
#include <string>
#include <unordered_set>
#include <utility>
#include <vector>

#include "common/kv/ITransaction.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Result.h"
#include "common/utils/StatusCode.h"
#include "common/utils/String.h"

namespace hf3fs::kv::mem {

inline int32_t memKvDefaultLimit = 25;

class MemKV : public folly::MoveOnly {
 public:
  struct VersionstampedKV {
    std::string key;
    std::string value;
    bool onKey;
    uint32_t offset;

    static VersionstampedKV versionstampedKey(std::string key, uint32_t offset, std::string value) {
      return {key, value, true, offset};
    }

    static VersionstampedKV versionstampedValue(std::string key, std::string value, uint32_t offset) {
      return {key, value, false, offset};
    }
  };

  Result<std::optional<String>> get(std::string_view key, int64_t readVersion) const {
    std::shared_lock<std::shared_mutex> lock(mutex_);

    auto it = map_.find(String(key));
    if (it == map_.end()) {
      return std::optional<String>();
    }

    auto version_it = it->second.upper_bound(readVersion);
    if (version_it != it->second.begin()) {
      --version_it;
      if (version_it->second.has_value()) {
        return std::make_optional(version_it->second.value());
      }
    }
    return std::optional<String>();
  }

  Result<IReadOnlyTransaction::GetRangeResult> getRange(const IReadOnlyTransaction::KeySelector &begin,
                                                        const IReadOnlyTransaction::KeySelector &end,
                                                        int32_t limit,
                                                        int64_t readVersion) const {
    if (limit < 1) {
      limit = memKvDefaultLimit;
    }
    std::shared_lock<std::shared_mutex> lock(mutex_);

    std::vector<IReadOnlyTransaction::KeyValue> kvs;

    if (begin.key > end.key || (begin.key == end.key && !(begin.inclusive && end.inclusive))) {
      return IReadOnlyTransaction::GetRangeResult(std::move(kvs), false);
    }

    auto begin_it = begin.inclusive ? map_.lower_bound(begin.key) : map_.upper_bound(begin.key);
    auto end_it = end.inclusive ? map_.upper_bound(end.key) : map_.lower_bound(end.key);
    while (begin_it != end_it && kvs.size() < (size_t)limit) {
      auto version_it = begin_it->second.upper_bound(readVersion);
      if (version_it != begin_it->second.begin()) {
        --version_it;
        if (version_it->second.has_value()) {
          kvs.emplace_back(begin_it->first, version_it->second.value());
        }
      }
      ++begin_it;
    }

    return IReadOnlyTransaction::GetRangeResult(std::move(kvs), begin_it != end_it);
  }

  Result<int64_t> commit(int64_t readVersion,
                         const std::unordered_set<String> &readKeys,
                         const std::vector<std::pair<String, String>> readRanges,
                         const std::vector<std::pair<String, std::optional<String>>> changes,
                         const std::vector<VersionstampedKV> versionstampedChanges,
                         const std::set<String> writeConflicts) {
    // if transaction contains no updates, don't need commit.
    if (changes.empty() && versionstampedChanges.empty() && writeConflicts.empty()) {
      return -1;  // return a invalid version
    }
    if (readVersion < 0) {
      return makeError(TransactionCode::kFailed, "MemKV transaction invalid version!");
    }

    std::unique_lock<std::shared_mutex> lock(mutex_);

    // check conflict
    for (auto &key : readKeys) {
      auto it = map_.find(key);
      if (it != map_.end()) {
        auto version_it = it->second.upper_bound(readVersion);
        if (version_it != it->second.end()) {
          return makeError(TransactionCode::kConflict, "MemKV transaction conflicts!");
        }
      }
    }
    for (auto &range : readRanges) {
      auto it = map_.lower_bound(range.first);
      while (it != map_.end() && it->first < range.second) {
        auto version_it = it->second.upper_bound(readVersion);
        if (version_it != it->second.end()) {
          return makeError(TransactionCode::kConflict, "MemKV transaction conflicts!");
        }
        it++;
      }
    }

    auto version = ++version_;
    uint16_t seq = 0;
    for (auto change : versionstampedChanges) {
      Versionstamp(version, seq++).update(change);
      XLOGF_IF(FATAL, change.onKey && map_.contains(change.key), "shouldn't happen");
      map_[change.key][version] = change.value;
    }
    for (auto &change : changes) {
      if (change.second.has_value()) {
        map_[change.first][version] = change.second.value();
      } else {
        map_[change.first][version] = std::nullopt;
      }
    }
    for (const auto &conflict : writeConflicts) {
      if (map_.contains(conflict)) {
        auto value = map_[conflict].rbegin()->second;
        map_[conflict][version] = value;
      } else {
        map_[conflict][version] = std::nullopt;
      }
    }
    return version;
  }

  int64_t version() const {
    std::shared_lock<std::shared_mutex> lock(mutex_);
    return version_;
  }

  void backup() { states_.lock()->push(version()); }

  void restore() {
    auto guard = states_.lock();
    auto version = guard->front();
    guard->pop();
    restore(version);
  }

 private:
  struct Versionstamp {
    std::array<uint8_t, 8> version;
    std::array<uint8_t, 2> seq;

    Versionstamp(uint64_t version, uint16_t seq)
        : version(folly::bit_cast<std::array<uint8_t, 8>>(folly::Endian::big64(version))),
          seq(folly::bit_cast<std::array<uint8_t, 2>>(folly::Endian::big16(seq))) {}

    void update(VersionstampedKV &kv) const {
      auto &str = kv.onKey ? kv.key : kv.value;
      auto offset = kv.offset;
      XLOGF_IF(FATAL, offset + 10 > str.size(), "{} {}", offset, str.size());
      memcpy(str.data() + offset, (char *)this, 10);
    }
  };
  static_assert(sizeof(Versionstamp) == 10);

  void restore(int64_t version) {
    std::unique_lock<std::shared_mutex> lock(mutex_);
    version_ = version;

    // remove all modification after version
    auto iter = map_.begin();
    while (iter != map_.end()) {
      auto &values = iter->second;
      auto pos = values.lower_bound(version_ + 1);
      if (pos != values.end()) {
        values.erase(pos, values.end());
      }
      if (values.empty()) {
        map_.erase(iter++);
      } else {
        iter++;
      }
    }
  }

  void clear() {
    std::unique_lock lock(mutex_);
    map_.clear();
    version_ = 0;
  }

 private:
  mutable std::shared_mutex mutex_;
  folly::Synchronized<std::queue<int64_t>, std::mutex> states_;
  std::map<String, std::map<int64_t, std::optional<String>>, std::less<>> map_;
  int64_t version_ = 0;
};

}  // namespace hf3fs::kv::mem

#pragma once

#include <algorithm>
#include <cassert>
#include <folly/Likely.h>
#include <folly/Synchronized.h>
#include <folly/experimental/coro/Sleep.h>
#include <folly/logging/xlog.h>
#include <functional>
#include <iostream>
#include <limits>
#include <mutex>
#include <optional>
#include <string>
#include <string_view>
#include <unordered_set>
#include <utility>
#include <vector>

#include "common/kv/ITransaction.h"
#include "common/kv/mem/MemKV.h"
#include "common/utils/Coroutine.h"
#include "common/utils/FaultInjection.h"
#include "common/utils/RandomUtils.h"
#include "common/utils/Result.h"
#include "common/utils/Status.h"
#include "common/utils/String.h"

#define FAULT_INJECTION_ON_GET(op_name)                                                                              \
  do {                                                                                \
    if (FAULT_INJECTION()) {                                                                                \
      auto delayMs = folly::Random::rand32(10, 100);                                                                 \
      auto errCode =                                                                                \
          hf3fs::RandomUtils::randomSelect(std::vector<int>{TransactionCode::kThrottled, TransactionCode::kTooOld}); \
      XLOGF(WARN, "Inject fault on " #op_name ", errCode: {}, delayMs: {}.", errCode, delayMs);                      \
      co_await folly::coro::sleep(std::chrono::milliseconds(delayMs));                                               \
      co_return makeError(errCode, "Inject fault on " #op_name);                                                     \
    }                                                                                \
  } while (0)

#define FAULT_INJECTION_ON_COMMIT(db_name, injectMaybeCommitted)                                                     \
  do {                                                                                \
    if (FAULT_INJECTION()) {                                                                                \
      auto delayMs = folly::Random::rand32(10, 100);                                                                 \
      auto errCode = hf3fs::RandomUtils::randomSelect(                                                               \
          std::vector<int>{TransactionCode::kConflict, TransactionCode::kTooOld, TransactionCode::kMaybeCommitted}); \
      XLOGF(WARN, "Inject fault on commit, errCode: {}, delayMs: {}.", errCode, delayMs);                            \
      if (errCode == TransactionCode::kMaybeCommitted && folly::Random::oneIn(2)) {                                  \
        injectMaybeCommitted = true;                                                                                \
        XLOGF(WARN, "Inject maybeCommitted on commit, commit transaction in " #db_name ".");                         \
      } else {                                                                                \
        co_return makeError(errCode, "Inject fault on commit.");                                                     \
      }                                                                                \
    }                                                                                \
  } while (0)

namespace hf3fs::kv {

class MemTransaction : public IReadWriteTransaction {
 public:
  MemTransaction(mem::MemKV &mem)
      : mem_(mem),
        readVersion_(mem.version()) {}

  CoTryTask<std::optional<String>> snapshotGet(std::string_view key) override {
    FAULT_INJECTION_ON_GET(snapshotGet);
    co_return getImpl(key, true);
  }

  CoTryTask<GetRangeResult> snapshotGetRange(const KeySelector &begin, const KeySelector &end, int32_t limit) override {
    FAULT_INJECTION_ON_GET(snapshotGetRange);
    co_return getRangeImpl(begin, end, limit, true);
  }

  CoTryTask<void> cancel() override {
    std::scoped_lock<std::mutex> guard(mutex_);
    canceled_ = true;
    co_return Void{};
  }

  CoTryTask<void> addReadConflict(std::string_view key) override {
    std::scoped_lock<std::mutex> guard(mutex_);
    if (canceled_) {
      co_return makeError(TransactionCode::kCanceled, "Canceled transaction!");
    }
    if (changes_.contains(key)) {
      co_return Void{};
    }
    readKeys_.insert(String(key));
    co_return Void{};
  }

  CoTryTask<void> addReadConflictRange(std::string_view begin, std::string_view end) override {
    std::scoped_lock<std::mutex> guard(mutex_);
    if (canceled_) {
      co_return makeError(TransactionCode::kCanceled, "Canceled transaction!");
    }
    readRanges_.push_back({std::string(begin), std::string(end)});
    co_return Void{};
  }

  CoTryTask<std::optional<String>> get(std::string_view key) override {
    FAULT_INJECTION_ON_GET(get);
    co_return getImpl(key, false);
  }

  CoTryTask<GetRangeResult> getRange(const KeySelector &begin, const KeySelector &end, int32_t limit) override {
    FAULT_INJECTION_ON_GET(getRange);
    co_return getRangeImpl(begin, end, limit, false);
  }

  CoTryTask<void> set(std::string_view key, std::string_view value) override {
    std::scoped_lock<std::mutex> guard(mutex_);
    if (canceled_) {
      co_return makeError(TransactionCode::kCanceled, "Canceled transaction!");
    }
    changes_[String(key)] = value;
    co_return Void{};
  }

  CoTryTask<void> setVersionstampedKey(std::string_view key, uint32_t offset, std::string_view value) override {
    std::scoped_lock<std::mutex> guard(mutex_);
    if (canceled_) {
      co_return makeError(TransactionCode::kCanceled, "Canceled transaction!");
    }
    if (offset + sizeof(kv::Versionstamp) > key.size()) {
      co_return makeError(
          StatusCode::kInvalidArg,
          fmt::format("setVersionstampedKey: {} + sizeof(kv::Versionstamp) > key.size {}", offset, key.size()));
    }
    versionstampedChanges_.push_back(
        mem::MemKV::VersionstampedKV::versionstampedKey(std::string(key), offset, std::string(value)));
    co_return Void{};
  }

  CoTryTask<void> setVersionstampedValue(std::string_view key, std::string_view value, uint32_t offset) override {
    std::scoped_lock<std::mutex> guard(mutex_);
    if (canceled_) {
      co_return makeError(TransactionCode::kCanceled, "Canceled transaction!");
    }
    if (offset + sizeof(kv::Versionstamp) > value.size()) {
      co_return makeError(
          StatusCode::kInvalidArg,
          fmt::format("setVersionstampedValue: {} + sizeof(kv::Versionstamp) > value.size {}", offset, value.size()));
    }
    versionstampedChanges_.push_back(
        mem::MemKV::VersionstampedKV::versionstampedValue(std::string(key), std::string(value), offset));
    co_return Void{};
  }

  // Check given keys are in transaction's conflict set.
  bool checkConflictSet(const std::vector<String> &readConflict,
                        const std::vector<String> &writeConflict,
                        bool exactly = false) {
    std::scoped_lock<std::mutex> guard(mutex_);
    for (auto &key : readConflict) {
      if (!readKeys_.contains(key)) {
        XLOGF(ERR, "Read conflict set doesn't contains key {}", key);
        return false;
      }
    }
    for (auto &key : writeConflict) {
      auto iter = std::find_if(changes_.begin(), changes_.end(), [&](auto &iter) -> bool { return iter.first == key; });
      if (iter == changes_.end()) {
        XLOGF(ERR, "Write conflict set doesn't contains key {}", key);
        return false;
      }
    }
    if (!exactly) {
      return true;
    }

    // transaction's conflict set should only given keys
    for (auto &key : readKeys_) {
      if (std::find(readConflict.begin(), readConflict.end(), key) == readConflict.end()) {
        XLOGF(ERR, "Read conflict contains unexpected key {}", key);
        return false;
      }
    }
    for (auto &change : changes_) {
      if (std::find(writeConflict.begin(), writeConflict.end(), change.first) == writeConflict.end()) {
        XLOGF(ERR, "Write conflict contains unexpected key {}", change.first);
        return false;
      }
    }
    return true;
  }

  CoTryTask<void> clear(std::string_view key) override {
    std::scoped_lock<std::mutex> guard(mutex_);
    if (canceled_) {
      co_return makeError(TransactionCode::kCanceled, "Canceled transaction!");
    }
    changes_[String(key)] = std::optional<String>();
    co_return Void{};
  }

  CoTryTask<void> commit() override {
    bool injectMaybeCommitted = false;
    FAULT_INJECTION_ON_COMMIT(MemKV, injectMaybeCommitted);

    std::scoped_lock<std::mutex> guard(mutex_);
    if (canceled_) {
      co_return makeError(TransactionCode::kCanceled, "Canceled transaction!");
    }

    std::vector<std::pair<String, std::optional<String>>> vec;
    vec.reserve(changes_.size());
    for (auto iter : changes_) {
      vec.emplace_back(iter.first, iter.second);
    }
    auto result = mem_.commit(readVersion(), readKeys_, readRanges_, vec, versionstampedChanges_, writeConflicts_);
    if (UNLIKELY(injectMaybeCommitted)) {
      XLOGF(ERR,
            "Inject mayebeCommitted error after commit, MemKV commit result is {}.",
            result.hasError() ? result.error().describe() : "OK");
      co_return makeError(TransactionCode::kMaybeCommitted, "Fault injection commit unknown result.");
    }
    CO_RETURN_ON_ERROR(result);
    commitVersion_ = *result;
    co_return Void{};
  }

  void setReadVersion(int64_t version) override {
    std::scoped_lock<std::mutex> guard(mutex_);
    if (version >= 0) {
      readVersion_ = version;
    }
  }

  int64_t getCommittedVersion() override {
    std::scoped_lock<std::mutex> guard(mutex_);
    return commitVersion_;
  };

  void reset() override {
    std::scoped_lock<std::mutex> guard(mutex_);
    readVersion_ = -1;
    commitVersion_ = -1;
    canceled_ = false;
    readKeys_.clear();
    readRanges_.clear();
    changes_.clear();
    versionstampedChanges_.clear();
    writeConflicts_.clear();
  }

  // check txn1 updated txn2's read set
  static bool checkConflict(MemTransaction &txn1, MemTransaction &txn2) {
    std::scoped_lock<std::mutex> guard1(txn1.mutex_);
    std::scoped_lock<std::mutex> guard2(txn2.mutex_);

    for (auto &change : txn1.changes_) {
      auto &key = change.first;
      if (std::find(txn2.readKeys_.begin(), txn2.readKeys_.end(), key) != txn2.readKeys_.end()) {
        return true;
      }
      for (auto &range : txn2.readRanges_) {
        if (key >= range.first && key < range.second) {
          return true;
        }
      }
    }
    return false;
  }

  mem::MemKV &kv() { return mem_; }

 private:
  int64_t readVersion() {
    if (readVersion_ < 0) {
      readVersion_ = mem_.version();
    }
    return readVersion_;
  }

  Result<std::optional<String>> getImpl(std::string_view key, bool snapshot) {
    std::scoped_lock<std::mutex> guard(mutex_);
    if (canceled_) {
      return makeError(TransactionCode::kCanceled, "Canceled transaction!");
    }

    if (changes_.count(key)) {
      return changes_.at(String(key));
    }
    if (!snapshot) {
      readKeys_.insert(String(key));
    }
    return mem_.get(key, readVersion());
  }

  Result<GetRangeResult> getRangeImpl(const KeySelector &begin, const KeySelector &end, int32_t limit, bool snapshot) {
    std::scoped_lock<std::mutex> guard(mutex_);
    if (canceled_) {
      return makeError(TransactionCode::kCanceled, "Canceled transaction!");
    }
    // always get all kvs in range
    auto result = mem_.getRange(begin, end, std::numeric_limits<int32_t>::max(), readVersion());
    RETURN_ON_ERROR(result);

    auto changeIt = changes_.lower_bound(begin.key);
    if (changeIt != changes_.end() && !begin.inclusive) ++changeIt;

    if (changeIt != changes_.end()) {
      std::vector<KeyValue> newKvs;
      auto originIt = result->kvs.begin();
      auto originEnded = [&] { return originIt == result->kvs.end(); };
      auto changeEnded = [&] {
        if (changeIt == changes_.end()) return true;
        if (end.inclusive && changeIt->first > end.key) return true;
        if (!end.inclusive && changeIt->first >= end.key) return true;
        if (result->hasMore) {
          assert(!result->kvs.empty());
          if (changeIt->first > result->kvs.back().key) return true;
        }
        return false;
      };
      auto pushChange = [&] {
        if (changeIt->second.has_value()) newKvs.emplace_back(changeIt->first, *changeIt->second);
      };
      for (;;) {
        if (originEnded() || changeEnded()) break;
        if (originIt->key < changeIt->first) {
          newKvs.push_back(*originIt);
          ++originIt;
        } else if (originIt->key > changeIt->first) {
          pushChange();
          ++changeIt;
        } else {
          pushChange();
          ++changeIt;
          ++originIt;
        }
      }
      for (; !originEnded(); ++originIt) {
        newKvs.push_back(*originIt);
      }
      for (; !changeEnded(); ++changeIt) {
        pushChange();
      }
      result->kvs = std::move(newKvs);
    }

    if (limit < 1) {
      limit = mem::memKvDefaultLimit;
    }
    if (result->kvs.size() > (size_t)limit) {
      result->kvs = std::vector(&result->kvs[0], &result->kvs[(size_t)limit]);
      result->hasMore = true;
      assert(result->kvs.size() == (size_t)limit);
    }

    String rangeBegin = begin.inclusive ? String(begin.key) : TransactionHelper::keyAfter(begin.key);
    String rangeEnd;
    if (result->hasMore) {
      rangeEnd = TransactionHelper::keyAfter(result->kvs.end()[-1].key);
    } else {
      rangeEnd = end.inclusive ? TransactionHelper::keyAfter(end.key) : String(end.key);
    }
    if (!snapshot) {
      readRanges_.push_back({rangeBegin, rangeEnd});
    }

    return result;
  }

  std::mutex mutex_;
  mem::MemKV &mem_;
  bool canceled_ = false;
  int64_t readVersion_ = -1;
  int64_t commitVersion_ = -1;
  std::unordered_set<String> readKeys_;
  std::vector<std::pair<String, String>> readRanges_;
  std::map<String, std::optional<String>, std::less<>> changes_;
  std::vector<mem::MemKV::VersionstampedKV> versionstampedChanges_;
  std::set<String> writeConflicts_;
};

}  // namespace hf3fs::kv

#pragma once

#include <cstddef>
#include <cstdint>
#include <fmt/core.h>
#include <folly/Random.h>
#include <folly/Synchronized.h>
#include <folly/logging/xlog.h>
#include <map>
#include <optional>
#include <utility>
#include <vector>

#include "client/mgmtd/ICommonMgmtdClient.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Result.h"
#include "fbs/meta/Schema.h"
#include "fbs/mgmtd/MgmtdTypes.h"

namespace hf3fs::meta::server {

class ChainAllocator {
 public:
  ChainAllocator(std::shared_ptr<client::ICommonMgmtdClient> mgmtdClient)
      : mgmtdClient_(std::move(mgmtdClient)) {}

  CoTryTask<void> checkLayoutValid(const Layout &layout) {
    CO_RETURN_ON_ERROR(layout.valid(true));

    if (!layout.empty()) {
      auto routing = getRoutingInfo();

      const auto &chains = layout.getChainIndexList();
      for (auto index : chains) {
        auto ref = flat::ChainRef{layout.tableId, layout.tableVersion, index};
        if (auto chain = routing->getChain(ref); !chain) {
          XLOGF(ERR, "Layout contains a not found ChainRef {}", ref);
          co_return makeError(MetaCode::kInvalidFileLayout, fmt::format("{} not found", ref));
        } else if (chain->targets.empty()) {
          XLOGF(ERR, "Chain {} has no target", chain->chainId);
          co_return makeError(MetaCode::kInvalidFileLayout, fmt::format("Chain {} has no target", chain->chainId));
        }
      }
    }
    co_return Void{};
  }

  CoTryTask<void> allocateChainsForLayout(Layout &layout) {
    co_return co_await allocateChainsForLayout(layout, [&](size_t chainCnt) {
      auto tableId = layout.tableId;
      auto stripeSize = layout.stripeSize;
      auto key = AllocType(tableId, stripeSize);
      auto guard = roundRobin_.lock();
      auto iter = guard->find(key);
      if (iter == guard->end()) {
        // start with random value
        auto initial = folly::Random::rand32(chainCnt) / stripeSize * stripeSize;
        iter = guard->insert({key, initial}).first;
      }
      auto res = (iter->second % chainCnt) + 1;
      iter->second = (iter->second + stripeSize) % chainCnt;
      return res;
    });
  }

  CoTryTask<void> allocateChainsForLayout(Layout &layout, folly::Synchronized<uint32_t> &chainAllocCounter) {
    co_return co_await allocateChainsForLayout(layout, [&](size_t chainCnt) {
      auto guard = chainAllocCounter.wlock();
      auto stripeSize = layout.stripeSize;
      if (*guard == (uint32_t)-1) {
        // start with random value
        *guard = folly::Random::rand32(chainCnt) / stripeSize * stripeSize;
      }
      // add and return.
      auto res = (*guard % chainCnt) + 1;
      *guard = (*guard + stripeSize) % chainCnt;
      return res;
    });
  }

  CoTryTask<void> allocateChainsForLayout(Layout &layout, auto &&roundRobin) {
    CO_RETURN_ON_ERROR(co_await checkLayoutValid(layout));
    if (!layout.empty()) {
      co_return Void{};
    }

    auto tableId = layout.tableId;
    auto tableVersion = layout.tableVersion;

    auto routing = getRoutingInfo();
    const auto *table = routing->raw()->getChainTable(tableId, tableVersion);
    if (!table) {
      XLOGF(ERR, "Failed to find ChainTable with {} and {}", tableId, tableVersion);
      co_return makeError(MetaCode::kInvalidFileLayout,
                          fmt::format("ChainTable with {} and {} not found", tableId, tableVersion));
    } else if (!table->chainTableVersion) {
      XLOGF(ERR, "Invalid table {} version {}", tableId, tableVersion);
      co_return makeError(MetaCode::kInvalidFileLayout,
                          fmt::format("Invalid chain table {} version {}", tableId, tableVersion));
    }
    auto chainCnt = table->chains.size();
    if (chainCnt < layout.stripeSize || chainCnt == 0) {
      XLOGF(ERR,
            "Failed to allocate for layout {}, chain table {} have only {} chains.",
            layout,
            tableId.toUnderType(),
            chainCnt);
      co_return makeError(
          MetaCode::kInvalidFileLayout,
          fmt::format("try to allocate {} chains from {}, found {}", layout.stripeSize, tableId, chainCnt));
    }
    auto chainBegin = roundRobin(chainCnt);
    layout.tableVersion = table->chainTableVersion;
    layout.chains = Layout::ChainRange(chainBegin, Layout::ChainRange::STD_SHUFFLE_MT19937, folly::Random::rand64());
    if (auto valid = layout.valid(false); valid.hasError()) {
      XLOGF(DFATAL, "Layout is not valid after alloc {}, error {}", layout, valid.error());
      CO_RETURN_ERROR(valid);
    }

    co_return Void{};
  }

 private:
  std::shared_ptr<client::RoutingInfo> getRoutingInfo() { return mgmtdClient_->getRoutingInfo(); }

  using AllocType = std::pair<flat::ChainTableId, size_t>;
  folly::Synchronized<std::map<AllocType, uint32_t>, std::mutex> roundRobin_;
  std::shared_ptr<client::ICommonMgmtdClient> mgmtdClient_;
};

}  // namespace hf3fs::meta::server

#pragma once

#include <folly/Random.h>
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <folly/experimental/coro/Collect.h>
#include <folly/experimental/coro/Sleep.h>

#include "common/utils/BoundedQueue.h"
#include "common/utils/CPUExecutorGroup.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/Coroutine.h"

namespace hf3fs {

struct CoroutinesPoolBase {
  class Config : public ConfigBase<Config> {
    CONFIG_ITEM(coroutines_num, 64u);
    CONFIG_ITEM(queue_size, 1024u);
    CONFIG_ITEM(enable_work_stealing, false);

   public:
    Config() = default;
    Config(uint32_t coroutinesNum, uint32_t queueSize, bool enableWorkStealing = false) {
      set_coroutines_num(coroutinesNum);
      set_queue_size(queueSize);
      set_enable_work_stealing(enableWorkStealing);
    }
  };
};

template <class Job>
class CoroutinesPool : public CoroutinesPoolBase {
 public:
  CoroutinesPool(const Config &config, std::optional<folly::CPUThreadPoolExecutor::KeepAlive<>> executor = std::nullopt)
      : config_(config),
        executor_(std::move(executor)) {
    if (config_.enable_work_stealing()) {
      auto coroutinesNum = config_.coroutines_num();
      auto queueSizePerCoroutine = (config_.queue_size() + coroutinesNum - 1) / coroutinesNum;
      for (auto i = 0u; i < coroutinesNum; ++i) {
        funcQueues_.push_back(std::make_unique<Queue>(queueSizePerCoroutine));
      }
    } else {
      funcQueues_.push_back(std::make_unique<Queue>(config_.queue_size()));
    }
  }
  ~CoroutinesPool() { stopAndJoin(); }

  using Handler = std::function<CoTask<void>(Job job)>;
  Result<Void> start(Handler handler) {
    if (UNLIKELY(!executor_.has_value())) {
      auto msg = fmt::format("CoroutinesPool@{} does not have an executor", fmt::ptr(this));
      XLOG(ERR, msg);
      return makeError(StatusCode::kInvalidConfig, std::move(msg));
    }
    auto coroutinesNum = config_.coroutines_num();
    for (auto i = 0u; i < coroutinesNum; ++i) {
      futures_.push_back(run(handler, i).scheduleOn(*executor_).start());
    }
    return Void{};
  }

  Result<Void> start(Handler handler, CPUExecutorGroup &executor) {
    auto coroutinesNum = config_.coroutines_num();
    for (auto i = 0u; i < coroutinesNum; ++i) {
      futures_.push_back(run(handler, i).scheduleOn(&executor.pickNext()).start());
    }
    return Void{};
  }

  void stopAndJoin() {
    if (stopped_.test_and_set()) {
      XLOGF(DBG, "coroutines pool {} is already stopped.", fmt::ptr(this));
      return;
    }
    cancel_.requestCancellation();
    for (auto &future : futures_) {
      future.wait();
    }
    futures_.clear();
    if (executor_.has_value()) {
      executor_->reset();
    }
    XLOGF(INFO, "coroutines pool {} is stopped", fmt::ptr(this));
  }

  void enqueueSync(Job job) { pickQueue().enqueue(std::move(job)); }
  CoTask<void> enqueue(Job job) { co_await pickQueue().co_enqueue(std::move(job)); }

 protected:
  using Queue = BoundedQueue<Job>;
  using QueuePtr = std::unique_ptr<Queue>;

  Queue &pickQueue() {
    auto n = folly::Random::rand32(0, funcQueues_.size());
    return *funcQueues_[n];
  }

  struct WorkStealingJobTaker {
    WorkStealingJobTaker(bool enableWorkStealing,
                         size_t selfIndex,
                         size_t totalRunners,
                         const std::vector<QueuePtr> &qs)
        : self(enableWorkStealing ? selfIndex : 0),
          queues(qs) {
      if (enableWorkStealing) {
        for (size_t i = 0; i < totalRunners; ++i) {
          if (i != selfIndex) {
            others.push_back(i);
          }
        }
        std::shuffle(others.begin(), others.end(), folly::ThreadLocalPRNG{});
      }
    }

    CoTask<Job> take() {
      if (others.empty()) {
        co_return co_await queues[self]->co_dequeue();
      }

      while (true) {
        if (auto maybeJob = co_await timedWait(*queues[self]); maybeJob) {
          co_return std::move(*maybeJob);
        }

        auto otherIndex = others[nextIndex++ % others.size()];
        if (auto maybeJob = queues[otherIndex]->try_dequeue(); maybeJob) {
          co_return std::move(*maybeJob);
        }
      }
      __builtin_unreachable();
    }

    CoTask<std::optional<Job>> timedWait(Queue &queue) {
      static constexpr auto timeout = std::chrono::milliseconds(5);
      auto [sleepResult, dequeueResult] =
          co_await folly::coro::collectAnyNoDiscard(folly::coro::sleep(timeout), queue.co_dequeue());
      if (!dequeueResult.hasValue() && !dequeueResult.hasException()) {
        co_return std::nullopt;
      }
      co_return std::move(*dequeueResult);
    }

    size_t self;
    size_t nextIndex = 0;
    std::vector<size_t> others;
    const std::vector<QueuePtr> &queues;
  };

  CoTask<void> run(Handler handler, size_t queueIndex) {
    WorkStealingJobTaker taker(config_.enable_work_stealing(), queueIndex, config_.coroutines_num(), funcQueues_);
    while (true) {
      auto result = co_await co_awaitTry(co_withCancellation(cancel_.getToken(), taker.take()));
      if (UNLIKELY(result.template hasException<OperationCancelled>())) {
        XLOGF(DBG, "a coroutine in pool {} is stopped", fmt::ptr(this));
        break;
      } else if (result.hasException()) {
        XLOGF(ERR, "a coroutine in pool {} is stopped (exception: {})", fmt::ptr(this), result.exception().what());
        break;
      }
      co_await handler(std::move(*result));
    }
  }

 private:
  const Config &config_;
  std::optional<folly::CPUThreadPoolExecutor::KeepAlive<>> executor_;
  std::vector<QueuePtr> funcQueues_;

  CancellationSource cancel_;
  std::vector<folly::SemiFuture<Void>> futures_;
  std::atomic_flag stopped_{false};
};

}  // namespace hf3fs

#pragma once

#include <array>
#include <chrono>
#include <cstring>
#include <folly/Overload.h>
#include <folly/container/BitIterator.h>
#include <folly/experimental/coro/Mutex.h>
#include <folly/lang/Bits.h>
#include <functional>
#include <infiniband/verbs.h>
#include <map>
#include <memory>
#include <string>
#include <string_view>
#include <utility>
#include <variant>
#include <vector>

#include "common/net/ib/IBDevice.h"
#include "common/serde/Serde.h"
#include "common/utils/Result.h"

namespace hf3fs::net {

struct IBDeviceInfo {
 public:
  struct PortInfo {
    SERDE_STRUCT_FIELD(port, uint8_t(0));
    SERDE_STRUCT_FIELD(zones, std::vector<std::string>());
    SERDE_STRUCT_FIELD(link_layer, uint8_t(IBV_LINK_LAYER_UNSPECIFIED));
  };

  SERDE_STRUCT_FIELD(dev, uint8_t(0));
  SERDE_STRUCT_FIELD(dev_name, std::string());
  SERDE_STRUCT_FIELD(ports, std::vector<PortInfo>());
};

struct IBQueryReq {
  SERDE_STRUCT_FIELD(dummy, Void{});
};

struct IBQueryRsp {
  SERDE_STRUCT_FIELD(devices, std::vector<IBDeviceInfo>());

 public:
  struct Match {
    uint8_t remoteDev;
    std::string remoteDevName;
    uint8_t remotePort;
    IBDevice::Ptr localDev;
    uint8_t localPort;
    std::string zone;

    std::string describe() const {
      if (zone.empty()) {
        return fmt::format("{{local {}:{} remote {}:{}}}", localDev->name(), localPort, remoteDevName, remotePort);
      } else {
        return fmt::format("{{local {}:{}, remote {}:{}, zone {}}}",
                           localDev->name(),
                           localPort,
                           remoteDevName,
                           remotePort,
                           zone);
      }
    }
  };
  Result<Match> selectDevice(const std::string &describe, uint64_t counter);
  void findMatchDevices(const std::string &describe,
                        std::vector<IBQueryRsp::Match> &ibMatches,
                        std::vector<IBQueryRsp::Match> &roceMatches,
                        bool checkZone);
};

struct IBConnectConfig {
  SERDE_STRUCT_FIELD(sl, uint8_t(0));
  SERDE_STRUCT_FIELD(traffic_class, uint8_t(0));
  SERDE_STRUCT_FIELD(gid_index, uint8_t(0));  // unused
  SERDE_STRUCT_FIELD(pkey_index, uint16_t(0));

  SERDE_STRUCT_FIELD(start_psn, uint32_t(0));
  SERDE_STRUCT_FIELD(min_rnr_timer, uint8_t(0));
  SERDE_STRUCT_FIELD(timeout, uint8_t(0));
  SERDE_STRUCT_FIELD(retry_cnt, uint8_t(0));
  SERDE_STRUCT_FIELD(rnr_retry, uint8_t(0));

  SERDE_STRUCT_FIELD(max_sge, uint32_t(0));
  SERDE_STRUCT_FIELD(max_rdma_wr, uint32_t(0));
  SERDE_STRUCT_FIELD(max_rdma_wr_per_post, uint32_t(0));
  SERDE_STRUCT_FIELD(max_rd_atomic, uint32_t(0));

  SERDE_STRUCT_FIELD(buf_size, uint32_t(0));
  SERDE_STRUCT_FIELD(send_buf_cnt, uint32_t(0));
  SERDE_STRUCT_FIELD(buf_ack_batch, uint32_t(0));
  SERDE_STRUCT_FIELD(buf_signal_batch, uint32_t(0));
  SERDE_STRUCT_FIELD(event_ack_batch, uint32_t(0));

  SERDE_STRUCT_FIELD(record_latency_per_peer, false);

 public:
  uint32_t qpAckBufs() const { return (send_buf_cnt + buf_ack_batch - 1) / buf_ack_batch + 4; }
  uint32_t qpMaxSendWR() const {
    return send_buf_cnt + qpAckBufs() + max_rdma_wr + 1 /* slot to send close msg */ + 1 /* slot to send check msg. */;
  }
  uint32_t qpMaxRecvWR() const { return send_buf_cnt + qpAckBufs() + 1 /* slot to recv close msg */; }
  uint32_t qpMaxCQEntries() const { return qpMaxSendWR() + qpMaxRecvWR(); }
};

struct IBConnectIBInfo {
  SERDE_STRUCT_FIELD(lid, uint16_t(0));

 public:
  static constexpr auto kType = IBV_LINK_LAYER_INFINIBAND;
};

struct IBConnectRoCEInfo {
  SERDE_STRUCT_FIELD(gid, ibv_gid{});

 public:
  static constexpr auto kType = IBV_LINK_LAYER_ETHERNET;
};

struct IBConnectInfo {
  SERDE_STRUCT_FIELD(hostname, std::string());
  SERDE_STRUCT_FIELD(dev, uint8_t(0));
  SERDE_STRUCT_FIELD(dev_name, std::string());
  SERDE_STRUCT_FIELD(port, uint8_t(0));

  SERDE_STRUCT_FIELD(mtu, uint8_t(0));
  SERDE_STRUCT_FIELD(qp_num, uint32_t(0));
  SERDE_STRUCT_FIELD(linklayer, (std::variant<IBConnectIBInfo, IBConnectRoCEInfo>()));

 public:
  auto getLinkerLayerType() const {
    return folly::variant_match(linklayer, [](auto &v) { return std::decay_t<decltype(v)>::kType; });
  }
};

struct IBConnectReq : IBConnectInfo {
  SERDE_STRUCT_FIELD(target_dev, uint8_t(0));
  SERDE_STRUCT_FIELD(target_devname, std::string());
  SERDE_STRUCT_FIELD(target_port, uint8_t(0));
  SERDE_STRUCT_FIELD(config, IBConnectConfig());

 public:
  IBConnectReq()
      : IBConnectReq({}, 0, "", 0, {}) {}
  IBConnectReq(IBConnectInfo info,
               uint8_t target_dev,
               std::string target_devname,
               uint8_t target_port,
               IBConnectConfig config)
      : IBConnectInfo(info),
        target_dev(target_dev),
        target_devname(std::move(target_devname)),
        target_port(target_port),
        config(config) {}
};
static_assert(serde::SerializableToBytes<IBConnectReq> && serde::SerializableToJson<IBConnectReq>);

struct IBConnectRsp : IBConnectInfo {
  SERDE_STRUCT_FIELD(dummy, Void{});

 public:
  IBConnectRsp(IBConnectInfo info = {})
      : IBConnectInfo(info) {}
};
static_assert(serde::SerializableToBytes<IBConnectRsp> && serde::SerializableToJson<IBConnectRsp>);

}  // namespace hf3fs::net

template <>
struct hf3fs::serde::SerdeMethod<ibv_gid> {
  static std::string_view serdeTo(const ibv_gid &gid) {
    return std::string_view((const char *)&gid.raw[0], sizeof(ibv_gid::raw));
  }
  static Result<ibv_gid> serdeFrom(std::string_view s) {
    if (s.length() != sizeof(ibv_gid::raw)) {
      return makeError(StatusCode::kDataCorruption);
    } else {
      ibv_gid gid;
      memcpy(gid.raw, s.data(), s.length());
      return gid;
    }
  }
};

#pragma once

#include <folly/Utility.h>
#include <folly/experimental/coro/FutureUtil.h>
#include <folly/experimental/coro/Promise.h>
#include <folly/experimental/coro/Task.h>

#include "common/kv/ITransaction.h"
#include "common/utils/String.h"
#include "foundationdb/fdb_c_options.g.h"
#include "foundationdb/fdb_c_types.h"

#define FDB_API_VERSION 710
#include <foundationdb/fdb_c.h>

namespace hf3fs::kv {

class FDBKVEngine;

namespace fdb {

using folly::coro::Task;
using KeyValue = IReadOnlyTransaction::KeyValue;

// Structs for returned values.
struct KeyRange {
  String beginKey;
  String endKey;
};

// Structs for parameters.
struct KeyRangeView {
  std::string_view beginKey;
  std::string_view endKey;
};

struct KeySelector {
  std::string_view key;  // Find the last item less than key
  bool orEqual = true;   // (or equal to key, if this is true)
  int offset = 0;        // and then move forward this many items (or backward if negative)

  KeySelector() = default;
  KeySelector(std::string_view key, bool orEqual, int offset)
      : key(std::move(key)),
        orEqual(orEqual),
        offset(offset) {}
};

struct GetRangeLimits {
  std::optional<uint32_t> rows;
  std::optional<uint32_t> bytes;

  explicit GetRangeLimits(std::optional<uint32_t> r = {}, std::optional<uint32_t> b = {})
      : rows(r),
        bytes(b) {}
};

template <class T, class V>
class Result {
 public:
  fdb_error_t error() const { return error_; };
  V &value() { return value_; }
  const V &value() const { return value_; }

 protected:
  friend class DB;
  friend class Transaction;

  static Task<T> toTask(FDBFuture *f);
  void extractValue();

 protected:
  struct FDBFutureDeleter {
    constexpr FDBFutureDeleter() noexcept = default;
    void operator()(FDBFuture *future) const { future ? fdb_future_destroy(future) : void(); }
  };
  std::unique_ptr<FDBFuture, FDBFutureDeleter> future_;
  fdb_error_t error_ = 0;
  V value_{};
};

class Int64Result : public Result<Int64Result, int64_t> {};
class KeyResult : public Result<KeyResult, String> {};
class ValueResult : public Result<ValueResult, std::optional<String>> {};
class KeyArrayResult : public Result<KeyArrayResult, std::vector<String>> {};
class StringArrayResult : public Result<StringArrayResult, std::vector<String>> {};
class KeyValueArrayResult : public Result<KeyValueArrayResult, std::pair<std::vector<KeyValue>, bool>> {};
class KeyRangeArrayResult : public Result<KeyRangeArrayResult, std::vector<KeyRange>> {};
struct EmptyValue {};
class EmptyResult : public Result<EmptyResult, EmptyValue> {};

class DB {
 public:
  static fdb_error_t selectAPIVersion(int version);
  static std::string_view errorMsg(fdb_error_t code);
  static bool evaluatePredicate(int predicate_test, fdb_error_t code);

  // network
  static fdb_error_t setNetworkOption(FDBNetworkOption option, std::string_view value = {});
  static fdb_error_t setupNetwork();
  static fdb_error_t runNetwork();
  static fdb_error_t stopNetwork();

 public:
  DB() = default;
  explicit DB(const String &clusterFilePath, bool readonly)
      : readonly_(readonly) {
    auto path = clusterFilePath.empty() ? nullptr : clusterFilePath.c_str();
    FDBDatabase *db;
    error_ = fdb_create_database(path, &db);
    if (error_ == 0) {
      db_.reset(db);
    }
  }

  fdb_error_t error() const { return error_; }
  explicit operator bool() const { return error() == 0; }
  operator FDBDatabase *() const { return db_.get(); }

  fdb_error_t setOption(FDBDatabaseOption option, std::string_view value = {});

  Task<Int64Result> rebootWorker(std::string_view address, bool check = false, int duration = 0);
  Task<EmptyResult> forceRecoveryWithDataLoss(std::string_view dcid);
  Task<EmptyResult> createSnapshot(std::string_view uid, std::string_view snapCommand);
  Task<KeyResult> purgeBlobGranules(const KeyRangeView &range, int64_t purgeVersion, fdb_bool_t force);
  Task<EmptyResult> waitPurgeGranulesComplete(std::string_view purgeKey);

  bool readonly() const { return readonly_; }

 private:
  friend class hf3fs::kv::FDBKVEngine;

  struct FDBDatabaseDeleter {
    constexpr FDBDatabaseDeleter() noexcept = default;
    void operator()(FDBDatabase *db) const { db ? fdb_database_destroy(db) : void(); }
  };
  std::unique_ptr<FDBDatabase, FDBDatabaseDeleter> db_;
  bool readonly_ = false;
  fdb_error_t error_ = 1;
};

class Transaction final {
 public:
  Transaction(DB &db)
      : readonly_(db.readonly()) {
    FDBTransaction *tr;
    error_ = fdb_database_create_transaction(db, &tr);
    if (error_ == 0) {
      tr_.reset(tr);
    }
  }

  fdb_error_t error() const { return error_; }
  explicit operator bool() const { return error_ == 0 && tr_; }

  fdb_error_t setOption(FDBTransactionOption option, std::string_view value = {});

  // Read transaction
  void setReadVersion(int64_t version);
  Task<Int64Result> getReadVersion();

  Task<ValueResult> get(std::string_view key, fdb_bool_t snapshot = false);
  Task<KeyResult> getKey(const KeySelector &selector, fdb_bool_t snapshot = false);
  Task<EmptyResult> watch(std::string_view key);

  Task<StringArrayResult> getAddressesForKey(std::string_view key);
  Task<KeyValueArrayResult> getRange(const KeySelector &begin,
                                     const KeySelector &end,
                                     GetRangeLimits limits = GetRangeLimits(),
                                     int iteration = 0,
                                     bool snapshot = false,
                                     bool reverse = false,
                                     FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL);

  Task<Int64Result> getEstimatedRangeSizeBytes(const KeyRangeView &range);
  Task<KeyArrayResult> getRangeSplitPoints(const KeyRangeView &range, int64_t chunkSize);

  Task<EmptyResult> onError(fdb_error_t err);

  void cancel();
  void reset();

  // Write transaction
  fdb_error_t addConflictRange(const KeyRangeView &range, FDBConflictRangeType type);

  void atomicOp(std::string_view key, std::string_view param, FDBMutationType operationType);
  void set(std::string_view key, std::string_view value);
  void clear(std::string_view key);
  void clearRange(const KeyRangeView &range);

  Task<EmptyResult> commit();
  fdb_error_t getCommittedVersion(int64_t *outVersion);
  Task<Int64Result> getApproximateSize();
  Task<KeyResult> getVersionstamp();

 private:
  struct FDBTransactionDeleter {
    constexpr FDBTransactionDeleter() noexcept = default;
    void operator()(FDBTransaction *tr) const { tr ? fdb_transaction_destroy(tr) : void(); }
  };
  std::unique_ptr<FDBTransaction, FDBTransactionDeleter> tr_;
  bool readonly_ = false;
  fdb_error_t error_ = 1;
};
}  // namespace fdb
}  // namespace hf3fs::kv

#pragma once
#include <cassert>
#include <fmt/format.h>
#include <string_view>
#include <type_traits>
#include <utility>

#include "common/utils/Conversion.h"
#include "common/utils/Result.h"
#include "common/utils/String.h"

namespace hf3fs {

namespace detail {
template <typename T, typename... Ts>
constexpr inline size_t totalSize(const T &, const Ts &...others) {
  size_t total = sizeof(T);
  if constexpr (sizeof...(others) > 0) {
    total += totalSize(others...);
  }
  return total;
}
};  // namespace detail

struct Serializer {
  explicit Serializer(String &s)
      : s_(s) {}

  template <typename... Args>
  static String serRawArgs(Args &&...args) {
    String buf;
    buf.reserve(detail::totalSize(std::forward<Args>(args)...));
    Serializer ser(buf);
    ser.put(std::forward<Args>(args)...);
    return buf;
  }

  Serializer &putChar(char c) {
    s_.push_back(c);
    return *this;
  }

  template <typename T>
  Serializer &putIntAsChar(T v) {
    assert(isSafeConvertTo<char>(v));
    return putChar(static_cast<char>(v));
  }

  // for strings whose length is less than 128.
  Serializer &putShortString(std::string_view s) {
    putIntAsChar(s.size());
    s_.append(s.data(), s.size());
    return *this;
  }

  template <typename T>
  Serializer &put(const T &v) {
    static_assert(std::is_trivial_v<T>);
    return putRaw(&v, sizeof(T));
  }

  template <typename T, typename... Ts>
  Serializer &put(const T &v, const Ts &...others) {
    put(v);
    if constexpr (sizeof...(others) > 0) {
      put(others...);
    }
    return *this;
  }

  Serializer &putRaw(const void *data, size_t length) {
    s_.append(reinterpret_cast<const char *>(data), length);
    return *this;
  }

  String &s_;
};

struct Deserializer {
  explicit Deserializer(std::string_view s, size_t pos = 0)
      : s_(s),
        pos_(pos) {}

  template <typename... Args>
  static Result<Void> deserRawArgs(std::string_view s, Args &...args) {
    Deserializer des(s);
    RETURN_ON_ERROR(des.get(args...));
    if (!des.reachEnd()) {
      return makeError(StatusCode::kDataCorruption,
                       fmt::format("Data Corruption when deserialize at pos {}: should reach end, remaining {}.",
                                   des.pos_,
                                   des.s_.size() - des.pos_));
    }
    return Void();
  }

#define CHECK_BUFFER_LENGTH(n)                                                                       \
  do {                                                                                \
    auto nn = (n);                                                                                \
    if (pos_ + nn > s_.size()) {                                                                     \
      String msg = fmt::format("Data Corruption when deserialize at pos {}: need {}, remaining {}.", \
                               pos_,                                                                 \
                               nn,                                                                   \
                               s_.size() - pos_);                                                    \
      return folly::makeUnexpected(Status(StatusCode::kDataCorruption, std::move(msg)));             \
    }                                                                                \
  } while (false)

  Result<char> getChar() noexcept {
    CHECK_BUFFER_LENGTH(1);
    return s_[pos_++];
  }

  template <typename T>
  Result<T> getIntFromChar() noexcept {
    return getChar().then([](char c) {
      assert(isSafeConvertTo<T>(c));
      return static_cast<T>(c);
    });
  }

  Result<std::string_view> getShortString() noexcept {
    return getIntFromChar<size_t>().then([this](size_t size) { return getRaw(size); });
  }

  template <typename T>
  Result<T> get() noexcept {
    static_assert(std::is_trivial_v<T>);
    CHECK_BUFFER_LENGTH(sizeof(T));
    T val;
    std::memcpy(&val, &s_[pos_], sizeof(T));
    pos_ += sizeof(T);
    return val;
  }

  template <typename T, typename... Ts>
  Result<Void> get(T &t, Ts &...others) {
    auto result = get<T>();
    RETURN_ON_ERROR(result);
    t = *result;
    if constexpr (sizeof...(others) > 0) {
      return get(others...);
    }
    return Void{};
  }

  Result<std::string_view> getRaw(size_t length) noexcept {
    CHECK_BUFFER_LENGTH(length);
    std::string_view res(&s_[pos_], length);
    pos_ += length;
    return res;
  }

  Result<std::string_view> getRawUntilEnd() noexcept {
    std::string_view res(&s_[pos_], s_.length() - pos_);
    pos_ = s_.length();
    return res;
  }

#undef CHECK_BUFFER_LENGTH

  bool reachEnd() const noexcept { return pos_ >= s_.size(); }

  std::string_view s_;
  size_t pos_ = 0;
};

}  // namespace hf3fs

#pragma once

#include <folly/experimental/coro/BlockingWait.h>
#include <memory>

#include "common/net/Client.h"
#include "common/net/Network.h"
#include "common/net/Server.h"
#include "common/serde/CallContext.h"
#include "common/serde/ClientContext.h"
#include "common/utils/Address.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Result.h"

namespace hf3fs::serde {

class ClientMockContext {
 public:
  enum IsMock : bool { value = true };

  template <class Service>
  static ClientMockContext create(std::unique_ptr<Service> service, net::Address::Type type = net::Address::TCP) {
    ClientMockContext ctx;
    ctx.setService(std::move(service), type);
    return ctx;
  }

  template <class Service>
  bool setService(std::unique_ptr<Service> service, net::Address::Type type = net::Address::TCP) {
    pack_ = std::make_shared<ServerPack>(type);
    auto setupResult = pack_->server_->setup();
    XLOGF_IF(FATAL, !setupResult, "setup server failed: {}", setupResult.error());

    auto addResult = pack_->server_->addSerdeService(std::move(service));
    XLOGF_IF(FATAL, !addResult, "add service failed: {}", addResult.error());

    auto startResult = pack_->server_->start();
    XLOGF_IF(FATAL, !startResult, "start server failed: {}", startResult.error());

    auto clientResult = pack_->client_->start();
    XLOGF_IF(FATAL, !clientResult, "start client failed: {}", clientResult.error());
    return true;
  }

  template <NameWrapper kServiceName,
            NameWrapper kMethodName,
            class Req,
            class Rsp,
            uint16_t ServiceID,
            uint16_t MethodID>
  CoTryTask<Rsp> call(const Req &req,
                      const net::UserRequestOptions *options = nullptr,
                      Timestamp *timestamp = nullptr) {
    auto ctx = pack_->client_->serdeCtx(pack_->server_->groups().front()->addressList().front());
    co_return co_await ctx.call<kServiceName, kMethodName, Req, Rsp, ServiceID, MethodID>(req, options, timestamp);
  }

  template <NameWrapper kServiceName,
            NameWrapper kMethodName,
            class Req,
            class Rsp,
            uint16_t ServiceID,
            uint16_t MethodID>
  Result<Rsp> callSync(const Req &req,
                       const net::UserRequestOptions *options = nullptr,
                       Timestamp *timestamp = nullptr) {
    return folly::coro::blockingWait(
        call<kServiceName, kMethodName, Req, Rsp, ServiceID, MethodID>(req, options, timestamp));
  }

 private:
  struct ServerPack {
    ServerPack(net::Address::Type type) {
      for (size_t i = 0; i < serverConfig_.groups_length(); i++) {
        serverConfig_.groups(i).set_network_type(type);
      }
      server_ = std::make_unique<net::Server>(serverConfig_);
      client_ = std::make_unique<net::Client>(clientConfig_);
    }
    ~ServerPack() {
      client_->stopAndJoin();
      server_->stopAndJoin();
    }
    net::Server::Config serverConfig_;
    std::unique_ptr<net::Server> server_;
    net::Client::Config clientConfig_;
    std::unique_ptr<net::Client> client_;
  };
  std::shared_ptr<ServerPack> pack_;
};

}  // namespace hf3fs::serde

#pragma once

#include <list>

#include "common/utils/RobinHood.h"

namespace hf3fs {

template <class K, class V>
class LruCache {
 public:
  using key_type = K;
  using mapped_type = V;
  using value_type = std::pair<key_type, mapped_type>;
  using list_type = std::list<value_type>;
  using size_type = typename list_type::size_type;
  using iterator = typename list_type::iterator;
  using const_iterator = typename list_type::const_iterator;

 public:
  LruCache(size_t capacity, bool autoRemove = true)
      : capacity_(capacity),
        autoRemove_(autoRemove) {}

  iterator begin() { return least_.begin(); }
  const_iterator begin() const { return least_.begin(); }
  iterator end() { return least_.end(); }
  const_iterator end() const { return least_.end(); }
  bool empty() const { return least_.empty(); }
  size_t size() const { return least_.size(); }
  value_type &front() { return least_.front(); }
  const value_type &front() const { return least_.front(); }
  value_type &back() { return least_.back(); }
  const value_type &back() const { return least_.back(); }
  size_t getMaxSize() const { return capacity_; }
  void setMaxSize(size_t capacity) { capacity_ = capacity; }
  void clear() { used_.clear(), least_.clear(); }

  iterator find(const key_type &key) {
    auto it = used_.find(key);
    if (it == used_.end()) {
      return end();
    }
    return it->second;
  }
  const_iterator find(const key_type &key) const {
    auto it = used_.find(key);
    if (it == used_.end()) {
      return end();
    }
    return it->second;
  }

  void promote(const_iterator it) { least_.splice(least_.begin(), least_, it); }
  void obsolete(const_iterator it) { least_.splice(least_.end(), least_, it); }

  iterator erase(const key_type &key) {
    auto it = used_.find(key);
    if (it != used_.end()) {
      return erase(it->second);
    }
    return end();
  }
  iterator erase(const_iterator it) {
    used_.erase(it->first);
    return least_.erase(it);
  }

  size_t evictObsoleted() {
    if (least_.size() <= capacity_) {
      return 0;
    }
    auto evictingSize = least_.size() - capacity_;
    for (size_t i = 0; i < evictingSize; ++i) {
      used_.erase(back().first);
      least_.pop_back();
    }
    return evictingSize;
  }
  template <class Predicate>
  size_t evictObsoletedIf(Predicate &&p) {
    if (least_.size() <= capacity_) {
      return 0;
    }
    auto evictingSize = least_.size() - capacity_;
    for (size_t i = 0; i < evictingSize && p(back().first, back().second); ++i) {
      used_.erase(back().first);
      least_.pop_back();
    }

    return evictingSize;
  }

  template <class T>
  std::pair<iterator, bool> emplace(const key_type &key, T &&value) {
    auto [it, succ] = used_.emplace(key, iterator{});
    if (succ) {
      least_.emplace_front(key, std::forward<T>(value));
      it->second = least_.begin();
      if (autoRemove_) {
        evictObsoleted();
      }
    } else {
      promote(it->second);
    }
    return std::make_pair(it->second, succ);
  }

  mapped_type &operator[](const key_type &key) { return emplace(key, mapped_type{}).first->second; }

 private:
  size_t capacity_;
  bool autoRemove_;
  list_type least_;
  robin_hood::unordered_map<key_type, iterator> used_;
};

}  // namespace hf3fs

#pragma once

#include <folly/experimental/coro/Baton.h>
#include <folly/logging/xlog.h>
#include <functional>
#include <queue>
#include <unordered_map>
#include <utility>

#include "common/utils/ConfigBase.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Shards.h"

namespace hf3fs {

class ConcurrencyLimiterConfig : public ConfigBase<ConcurrencyLimiterConfig> {
  CONFIG_HOT_UPDATED_ITEM(max_concurrency, 1u, ConfigCheckers::checkPositive);
};

template <class Key>
class ConcurrencyLimiter {
 private:
  struct State {
    uint32_t concurrency{};
    std::queue<std::reference_wrapper<folly::coro::Baton>> queue{};
  };
  using Map = std::unordered_map<Key, State>;
  using It = typename Map::iterator;
  const ConcurrencyLimiterConfig &config_;
  Shards<Map, 16> shards_;

 public:
  ConcurrencyLimiter(const ConcurrencyLimiterConfig &config)
      : config_(config) {}

  struct Guard {
    Guard(ConcurrencyLimiter &limiter, std::size_t pos, It it)
        : limiter_(&limiter),
          pos_(pos),
          it_(it) {}
    Guard(const Guard &) = delete;
    Guard(Guard &&o)
        : limiter_(std::exchange(o.limiter_, nullptr)),
          pos_(o.pos_),
          it_(o.it_) {}
    ~Guard() { limiter_ == nullptr ? void() : limiter_->unlock(pos_, it_); }

   private:
    ConcurrencyLimiter *limiter_;
    std::size_t pos_;
    It it_;
  };

  CoTask<Guard> lock(const Key &key) {
    folly::coro::Baton baton;
    auto pos = shards_.position(key);
    auto [it, succ] = shards_.withLockAt(
        [&](Map &map) {
          auto [it, succ] = map.emplace(key, State{});
          auto &state = it->second;
          if (state.concurrency < config_.max_concurrency()) {
            ++state.concurrency;
            return std::make_pair(it, true);
          }
          state.queue.push(std::ref(baton));
          return std::make_pair(it, false);
        },
        pos);
    if (!succ) {
      co_await baton;
    }
    co_return Guard{*this, pos, it};
  }

 private:
  void unlock(size_t pos, It it) {
    shards_.withLockAt(
        [&](Map &map) {
          auto &state = it->second;
          assert(state.concurrency > 0);
          if (!state.queue.empty()) {
            if (state.concurrency <= config_.max_concurrency()) {
              // wake up the next waiting baton.
              auto next = std::move(state.queue.front());
              state.queue.pop();
              next.get().post();
            } else {
              // give up.
              --state.concurrency;
            }
          } else if (--state.concurrency == 0) {
            // remove current state.
            map.erase(it);
          }
        },
        pos);
  }
};

}  // namespace hf3fs

#pragma once

#include <folly/experimental/coro/Baton.h>
#include <folly/experimental/coro/BlockingWait.h>
#include <limits>
#include <memory>
#include <mutex>
#include <tuple>
#include <utility>

#include "common/utils/Coroutine.h"
#include "common/utils/Shards.h"

namespace hf3fs {

template <std::size_t N = 256>
class CoLockManager {
  struct State {
    std::string tag;
    folly::coro::Baton &baton;
  };
  using Queue = std::queue<State>;
  using Map = std::unordered_map<std::string, Queue>;
  using It = typename Map::iterator;

 public:
  struct Guard {
    static constexpr auto kInvalidPos = std::numeric_limits<std::size_t>::max();
    Guard(CoLockManager &manager, folly::coro::Baton &baton, std::size_t pos, It it, std::string currentTag)
        : manager_(manager),
          baton_(baton),
          pos_(pos),
          it_(it),
          currentTag_(std::move(currentTag)) {}
    Guard(const Guard &) = delete;
    Guard(Guard &&o)
        : manager_(o.manager_),
          baton_(o.baton_),
          pos_(std::exchange(o.pos_, kInvalidPos)),
          it_(o.it_),
          currentTag_(std::move(o.currentTag_)) {}
    ~Guard() {
      if (pos_ != kInvalidPos) {
        if (!locked()) {
          folly::coro::blockingWait(lock());
        }
        manager_.unlock(baton_, pos_, it_);
      }
    }

    // check locked or not.
    bool locked() const { return baton_.ready(); }

    // get current locked tag.
    auto &currentTag() const { return currentTag_; }

    // lock.
    [[nodiscard]] CoTask<void> lock() { co_await baton_; }

   private:
    CoLockManager &manager_;
    folly::coro::Baton &baton_;
    std::size_t pos_;
    It it_;
    std::string currentTag_;
  };

  auto lock(folly::coro::Baton &baton, const std::string &key, const std::string &tag = "") {
    auto pos = shards_.position(key);
    auto [it, succ, currentTag] = shards_.withLockAt(
        [&](Map &map) {
          auto [it, succ] = map.emplace(key, Queue{});
          it->second.push({tag, baton});
          return std::make_tuple(it, succ, it->second.front().tag);
        },
        pos);
    if (succ) {
      baton.post();
    }
    return Guard{*this, baton, pos, it, std::move(currentTag)};
  }

  auto tryLock(folly::coro::Baton &baton, const std::string &key, const std::string &tag = "") {
    auto pos = shards_.position(key);
    auto [it, succ, currentTag] = shards_.withLockAt(
        [&](Map &map) {
          auto [it, succ] = map.emplace(key, Queue{});
          if (succ) {
            it->second.push({tag, baton});
          }
          return std::make_tuple(it, succ, succ ? std::string{} : it->second.front().tag);
        },
        pos);
    if (succ) {
      baton.post();
    }
    return Guard{*this, baton, succ ? pos : Guard::kInvalidPos, it, std::move(currentTag)};
  }

 protected:
  void unlock(folly::coro::Baton &baton, size_t pos, It it) {
    shards_.withLockAt(
        [&](Map &map) {
          auto &queue = it->second;
          if (UNLIKELY(queue.empty() || &queue.front().baton != &baton)) {
            assert(false);
          }
          queue.pop();
          if (!queue.empty()) {
            queue.front().baton.post();
          } else {
            map.erase(it);
          }
        },
        pos);
  }

 private:
  Shards<Map, N> shards_;
};

}  // namespace hf3fs

#pragma once

#include <algorithm>
#include <cassert>
#include <cstdint>
#include <folly/Executor.h>
#include <folly/Random.h>
#include <folly/Synchronized.h>
#include <folly/Utility.h>
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <folly/logging/xlog.h>
#include <gtest/gtest_prod.h>
#include <memory>
#include <optional>
#include <string>
#include <string_view>
#include <utility>
#include <variant>
#include <vector>

#include "client/mgmtd/ICommonMgmtdClient.h"
#include "client/mgmtd/MgmtdClient.h"
#include "client/storage/StorageClient.h"
#include "common/app/ApplicationBase.h"
#include "common/app/NodeId.h"
#include "common/kv/IKVEngine.h"
#include "common/kv/ITransaction.h"
#include "common/utils/BackgroundRunner.h"
#include "common/utils/CPUExecutorGroup.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/Coroutine.h"
#include "common/utils/CoroutinesPool.h"
#include "common/utils/CountDownLatch.h"
#include "common/utils/Duration.h"
#include "common/utils/PriorityCoroutinePool.h"
#include "common/utils/Result.h"
#include "common/utils/Semaphore.h"
#include "common/utils/UtcTime.h"
#include "core/user/UserStoreEx.h"
#include "fbs/core/user/User.h"
#include "fbs/meta/Common.h"
#include "fdb/FDBRetryStrategy.h"
#include "fmt/core.h"
#include "meta/base/Config.h"
#include "meta/components/InodeIdAllocator.h"
#include "meta/components/SessionManager.h"
#include "meta/event/Event.h"
#include "meta/store/DirEntry.h"
#include "meta/store/FileSession.h"
#include "scn/scan/scan.h"

namespace hf3fs::meta::server {

class FileHelper;
using hf3fs::client::ICommonMgmtdClient;

class GcManager {
 public:
  static Result<std::pair<UtcTime, InodeId>> parseGcEntry(std::string_view entry) {
    char prefix;
    uint64_t timestamp;
    uint64_t inode;
    auto ret = scn::scan(entry, "{}-{}-{:i}", prefix, timestamp, inode);
    if (!ret) {
      return makeError(StatusCode::kInvalidArg);
    }
    return std::pair<UtcTime, InodeId>{UtcTime::fromMicroseconds(timestamp), InodeId(inode)};
  }

  static std::string formatGcEntry(char prefix, UtcTime timestamp, InodeId inode) {
    return fmt::format("{}-{:020d}-{}", prefix, (uint64_t)timestamp.toMicroseconds(), inode.toHexString());
  }

  GcManager(const Config &config,
            flat::NodeId nodeId,
            analytics::StructuredTraceLog<MetaEventTrace> &metaEventTraceLog,
            std::shared_ptr<kv::IKVEngine> kvEngine,
            std::shared_ptr<ICommonMgmtdClient> mgmtd,
            std::shared_ptr<InodeIdAllocator> idAlloc,
            std::shared_ptr<FileHelper> fileHelper,
            std::shared_ptr<SessionManager> sessionManager,
            std::shared_ptr<core::UserStoreEx> userStore)
      : config_(config),
        nodeId_(nodeId),
        metaEventTraceLog_(metaEventTraceLog),
        kvEngine_(kvEngine),
        mgmtd_(mgmtd),
        idAlloc_(idAlloc),
        fileHelper_(fileHelper),
        sessionManager_(sessionManager),
        userStore_(userStore),
        concurrentGcDirSemaphore_(config_.gc().gc_directory_concurrent()),
        concurrentGcFileSemaphore_(config_.gc().gc_file_concurrent()) {
    XLOGF_IF(FATAL, !nodeId_, "invalid node id {}", nodeId_);
    guard_ = config_.gc().addCallbackGuard([&]() {
      auto dirConcurrent = config_.gc().gc_directory_concurrent();
      if (dirConcurrent != 0 && dirConcurrent != concurrentGcDirSemaphore_.getUsableTokens()) {
        XLOGF(INFO, "GcManager set gc directory concurrent to {}", dirConcurrent);
        concurrentGcDirSemaphore_.changeUsableTokens(dirConcurrent);
        XLOGF(INFO, "GcManager finished update gc directory concurrent");
      }
      auto fileConcurrent = config_.gc().gc_file_concurrent();
      if (fileConcurrent != 0 && fileConcurrent != concurrentGcFileSemaphore_.getUsableTokens()) {
        XLOGF(INFO, "GcManager set gc directory concurrent to {}", fileConcurrent);
        concurrentGcFileSemaphore_.changeUsableTokens(fileConcurrent);
        XLOGF(INFO, "GcManager finished update gc file concurrent");
      }
    });
  }

  CoTryTask<void> init();

  void start(CPUExecutorGroup &exec);
  void stopAndJoin();

  auto &getEventTraceLog() { return metaEventTraceLog_; }

  CoTryTask<void> removeEntry(IReadWriteTransaction &txn, const DirEntry &entry, Inode &inode, GcInfo gcInfo);

 private:
  template <typename T>
  FRIEND_TEST(TestRemove, GC);

  enum GcEntryType {
    DIRECTORY = 0,
    FILE_MEDIUM,
    FILE_LARGE,
    FILE_SMALL,
    MAX,
  };

  class GcDirectory;

  struct GcTask {
    std::shared_ptr<GcDirectory> gcDir;
    GcEntryType type;
    DirEntry taskEntry;

    GcTask(std::shared_ptr<GcDirectory> gcDir, GcEntryType type, DirEntry entry)
        : gcDir(std::move(gcDir)),
          type(type),
          taskEntry(std::move(entry)) {}

    static CoTryTask<flat::UserAttr> getUserAttr(GcManager &manager, flat::Uid uid);

    CoTryTask<void> run(GcManager &manager);
    CoTryTask<void> gcDirectory(GcManager &manager);
    CoTryTask<void> gcFile(GcManager &manager);

    CoTryTask<void> removeEntry(GcManager &manager,
                                IReadWriteTransaction &txn,
                                const DirEntry &entry,
                                const flat::UserAttr &user);
    CoTryTask<void> removeGcEntryAndInode(GcManager &manager, IReadWriteTransaction &txn);
    CoTryTask<DirEntry> createOrphanEntry(GcManager &manager,
                                          IReadWriteTransaction &txn,
                                          const DirEntry &entry,
                                          const Inode &inode,
                                          const flat::UserAttr &user);
  };

  class GcDirectory : folly::MoveOnly, public std::enable_shared_from_this<GcDirectory> {
   public:
    using Ptr = std::shared_ptr<GcDirectory>;

    static char prefixOf(GcEntryType type) {
      switch (type) {
        case DIRECTORY:
          return 'd';
        case FILE_MEDIUM:
          return 'f';
        case FILE_LARGE:
          return 'L';
        case FILE_SMALL:
          return 'S';
        default:
          XLOGF(FATAL, "invalid type {}", (int)type);
      }
    }

    int8_t priorityOf(GcEntryType type) {
      switch (type) {
        case DIRECTORY:
          return folly::Executor::MID_PRI;
        case FILE_MEDIUM:
          return folly::Executor::MID_PRI;
        case FILE_LARGE:
          return folly::Executor::HI_PRI;
        case FILE_SMALL:
          return folly::Executor::LO_PRI;
        default:
          XLOGF(FATAL, "invalid type {}", (int)type);
      }
    }

    static std::string nameOf(flat::NodeId nodeId, size_t idx) {
      return idx == 0 ? fmt::format("GC-Node-{}", (uint32_t)nodeId)
                      : fmt::format("GC-Node-{}.{}", (uint32_t)nodeId, idx);
    }

    GcDirectory(DirEntry entry)
        : entry_(std::move(entry)) {}
    ~GcDirectory() { stopAndJoin(); }

    void start(GcManager &manager, CPUExecutorGroup &exec);
    void stopAndJoin();

    auto dirId() const { return entry_.id; }
    std::string name() const { return entry_.name; }

    CoTryTask<void> add(auto &txn, const Inode &inode, const GcConfig &config, GcInfo gcInfo);
    void finish(const GcTask &task);

    CoTryTask<void> moveToTail(auto &txn, const GcTask &task, Duration delay);

   private:
    struct QueueState {
      folly::Synchronized<std::set<InodeId>, std::mutex> queued;
      folly::Synchronized<std::set<InodeId>, std::mutex> finished;
      std::atomic<uint64_t> counter{0};
    };

    CoTask<void> scan(GcManager &manager, GcEntryType type);
    CoTryTask<bool> scan(GcManager &manager,
                         GcEntryType type,
                         Duration delay,
                         size_t limit,
                         std::optional<std::string> &prev);
    CoTryTask<void> addFile(auto &txn, const Inode &inode, const GcConfig &config);
    CoTryTask<void> addDirectory(auto &txn, const Inode &inode, GcInfo gcInfo);

    DirEntry entry_;  // entry points to this GcDirectory
    std::array<QueueState, GcEntryType::MAX> states_;
    CancellationSource cancel_;
    CountDownLatch<> latch_;
  };

  const std::vector<GcDirectory::Ptr> &currGcDirectories() const { return currGcDirectories_; }

  GcDirectory::Ptr pickGcDirectory() {
    XLOGF_IF(FATAL, currGcDirectories_.empty(), "currGcDirectories_.empty()");
    if (config_.gc().distributed_gc()) {
      auto guard = allGcDirectories_.rlock();
      if (!guard->empty()) {
        return guard->at(folly::Random::rand64(guard->size()));
      }
    }
    if (currGcDirectories_.size() == 1) {
      return currGcDirectories_[0];
    } else {
      return currGcDirectories_[folly::Random::rand32(1, currGcDirectories_.size())];
    }
  }

  bool enableGcDelay() const;

  CoTryTask<void> checkFs();
  CoTryTask<std::shared_ptr<GcDirectory>> openGcDirectory(size_t idx, bool create);
  CoTryTask<void> scanAllGcDirectories();

  CoTask<void> runGcTask(GcTask task);

  template <typename H>
  std::invoke_result_t<H, IReadOnlyTransaction &> runReadOnly(H &&handler) {
    auto retry = kv::FDBRetryStrategy({1_s, 10, true});
    co_return co_await kv::WithTransaction(retry).run(kvEngine_->createReadonlyTransaction(), std::forward<H>(handler));
  }

  template <typename H>
  std::invoke_result_t<H, IReadWriteTransaction &> runReadWrite(H &&handler) {
    auto retry = kv::FDBRetryStrategy({1_s, 10, true});
    co_return co_await kv::WithTransaction(retry).run(kvEngine_->createReadWriteTransaction(),
                                                      std::forward<H>(handler));
  }

  const Config &config_;
  std::unique_ptr<config::ConfigCallbackGuard> guard_;
  flat::NodeId nodeId_;
  analytics::StructuredTraceLog<MetaEventTrace> &metaEventTraceLog_;
  std::shared_ptr<kv::IKVEngine> kvEngine_;
  std::shared_ptr<ICommonMgmtdClient> mgmtd_;
  std::shared_ptr<InodeIdAllocator> idAlloc_;
  std::shared_ptr<FileHelper> fileHelper_;
  std::shared_ptr<SessionManager> sessionManager_;
  std::shared_ptr<core::UserStoreEx> userStore_;

  std::vector<GcDirectory::Ptr> currGcDirectories_;
  folly::Synchronized<std::vector<GcDirectory::Ptr>> allGcDirectories_;

  std::unique_ptr<BackgroundRunner> gcRunner_;
  std::unique_ptr<PriorityCoroutinePool<GcTask>> gcWorkers_;
  Semaphore concurrentGcDirSemaphore_;
  Semaphore concurrentGcFileSemaphore_;
};

}  // namespace hf3fs::meta::server

#pragma once

#include <algorithm>
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <ctime>
#include <fmt/core.h>
#include <folly/Expected.h>
#include <folly/logging/xlog.h>
#include <memory>
#include <optional>
#include <queue>
#include <string>
#include <string_view>
#include <utility>
#include <vector>

#include "common/kv/ITransaction.h"
#include "common/monitor/Recorder.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Result.h"
#include "common/utils/SerDeser.h"
#include "fbs/core/user/User.h"
#include "fbs/meta/Common.h"
#include "fbs/meta/Schema.h"
#include "fbs/meta/Service.h"
#include "meta/store/Inode.h"

namespace hf3fs::meta::server {
using hf3fs::kv::IReadOnlyTransaction;
using hf3fs::kv::IReadWriteTransaction;

struct DirEntryList;

class DirEntry : public meta::DirEntry {
 public:
  using Base = meta::DirEntry;
  using Base::Base;

  DirEntry(Base base)
      : Base(std::move(base)) {}

  static Result<DirEntry> newUnpacked(const std::string_view key, const std::string_view value);

  static CoTryTask<std::optional<DirEntry>> snapshotLoad(IReadOnlyTransaction &txn,
                                                         InodeId parent,
                                                         std::string_view name);

  static CoTryTask<std::optional<DirEntry>> load(IReadOnlyTransaction &txn, InodeId parent, std::string_view name);

  static CoTryTask<bool> checkExist(IReadOnlyTransaction &txn, InodeId parent, std::string_view name) {
    co_return (co_await DirEntry::load(txn, parent, name)).then([](auto &v) { return v.has_value(); });
  }

  static DirEntry newFile(InodeId parent, std::string name, InodeId inode) {
    return meta::DirEntry(parent, name, {inode, InodeType::File});
  }
  static DirEntry newSymlink(InodeId parent, std::string name, InodeId inode) {
    return meta::DirEntry(parent, name, {inode, InodeType::Symlink});
  }
  static DirEntry newDirectory(InodeId parent, std::string name, InodeId inode, Acl acl) {
    return meta::DirEntry(parent, name, {inode, InodeType::Directory, acl});
  }
  static DirEntry root() {
    return meta::DirEntry(InodeId::root(), ".", {InodeId::root(), InodeType::Directory, Acl::root()});
  }

  /** Key format: prefix + parent-InodeId.key + name */
  std::string packKey() const;
  static std::string packKey(InodeId parent, std::string_view name);
  Result<Void> unpackKey(const std::string_view key);

  // load inode from dir entry
  CoTryTask<Inode> loadInode(IReadOnlyTransaction &txn) const;
  CoTryTask<Inode> snapshotLoadInode(IReadOnlyTransaction &txn) const;
  CoTryTask<void> addIntoReadConflict(IReadWriteTransaction &txn) const {
#ifndef NDEBUG
    snapshotLoaded_ = false;
#endif
    co_return co_await txn.addReadConflict(packKey());
  }
  CoTryTask<void> store(IReadWriteTransaction &txn) const;
  CoTryTask<void> remove(IReadWriteTransaction &txn, bool ignoreSnapshotCheck = false) const;

 private:
  friend struct DirEntryList;
  friend class MetaTestHelper;

  template <const bool SNAPSHOT>
  static CoTryTask<std::optional<DirEntry>> loadImpl(IReadOnlyTransaction &txn, InodeId parent, std::string_view name);

#ifndef NDEBUG
  mutable bool snapshotLoaded_ = false;
#endif
};

struct DirEntryList {
  std::vector<DirEntry> entries;
  std::vector<Inode> inodes;
  bool more;

  // (prev, end)
  static CoTryTask<DirEntryList> snapshotLoad(IReadOnlyTransaction &txn,
                                              InodeId parent,
                                              std::string_view prev,
                                              int32_t limit,
                                              bool loadInodes = false,
                                              size_t loadInodesConcurrent = 0);

  // (begin, end)
  static CoTryTask<DirEntryList> snapshotLoad(IReadOnlyTransaction &txn,
                                              InodeId parent,
                                              std::string_view begin,
                                              std::string_view end,
                                              int32_t limit,
                                              bool loadInodes = false,
                                              size_t loadInodesConcurrent = 0);

  static CoTryTask<DirEntryList> load(IReadWriteTransaction &txn, InodeId parent, std::string_view prev, int32_t limit);

  static CoTryTask<bool> checkEmpty(IReadWriteTransaction &txn, InodeId parent);

  // For recursive remove and move to the trash, permission checks are required.
  // However, because the directory may be very large, we may not able to check permissions for entire
  // directory tree. This method is best effort.
  static CoTryTask<Void> recursiveCheckRmPerm(IReadWriteTransaction &txn,
                                              InodeId parent,
                                              flat::UserInfo user,
                                              int32_t limit,
                                              size_t listBatchSize) {
    static monitor::CountRecorder failed("meta_server.recursive_check_rm_perm_failed");
    auto guard = folly::makeGuard([&]() {
      failed.addSample(1, {{"uid", folly::to<std::string>(user.uid.toUnderType())}});
    });

    auto queue = std::queue<InodeId>();
    queue.push(parent);
    while (!queue.empty()) {
      auto currDir = queue.front();
      queue.pop();

      auto prev = std::string();
      auto foundDir = false;
      auto numEntries = 0;
      while (true) {
        if (limit-- <= 0) {
          break;
        }
        auto list = co_await DirEntryList::snapshotLoad(txn, currDir, prev, std::max(listBatchSize, 32ul));
        CO_RETURN_ON_ERROR(list);
        numEntries += list->entries.size();
        for (auto &entry : list->entries) {
          prev = entry.name;
          if (!entry.isDirectory()) {
            continue;
          }
          foundDir = true;
          if ((int64_t)queue.size() < limit) {
            queue.push(entry.id);
          }
          auto &acl = *entry.dirAcl;
          if (auto res = acl.checkRecursiveRmPerm(user, false); res.hasError()) {
            auto msg = fmt::format("user {} recursive remove {}, found {} without permission, msg {}",
                                   user.uid,
                                   parent,
                                   entry,
                                   res.error().message());
            XLOG(ERR, msg);
            co_return makeError(MetaCode::kNoPermission, msg);
          }
        }
        if (!list->more || (numEntries > 1024 && !foundDir)) {
          break;
        }
      }
    }
    guard.dismiss();
    co_return Void{};
  }

  DirEntry &entry(size_t i) { return entries.at(i); }
  const DirEntry &entry(size_t i) const { return entries.at(i); }

  const Inode &inode(size_t i) const { return inodes.at(i); }
  Inode &inode(size_t i) { return inodes.at(i); }

  operator ListRsp() && {
    ListRsp rsp;
    rsp.more = more;
    rsp.entries.reserve(entries.size());
    rsp.inodes.reserve(inodes.size());
    for (auto &entry : entries) {
      rsp.entries.emplace_back(std::move(entry));
    }
    for (auto &inode : inodes) {
      rsp.inodes.emplace_back(std::move(inode));
    }
    return rsp;
  }

 private:
  template <const bool SNAPSHOT>
  static CoTryTask<DirEntryList> loadImpl(IReadOnlyTransaction &txn,
                                          InodeId parent,
                                          IReadOnlyTransaction::KeySelector begin,
                                          IReadOnlyTransaction::KeySelector end,
                                          int32_t limit,
                                          bool loadInodes,
                                          size_t loadInodesConcurrent);
};

}  // namespace hf3fs::meta::server

#pragma once

#include <algorithm>
#include <chrono>
#include <folly/Likely.h>
#include <folly/Random.h>
#include <folly/experimental/coro/Sleep.h>
#include <folly/experimental/symbolizer/Symbolizer.h>
#include <folly/logging/xlog.h>
#include <foundationdb/fdb_c_types.h>
#include <string_view>
#include <utility>

#include "common/kv/ITransaction.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Duration.h"
#include "common/utils/Result.h"
#include "common/utils/Status.h"
#include "fdb/FDBTransaction.h"

namespace hf3fs::kv {

class FDBRetryStrategy {
 public:
  static constexpr Duration kMinBackoff = 10_ms;  // same with FDB, 0.01s

  struct Config {
    Duration maxBackoff = 1_s;
    size_t maxRetryCount = 10;
    bool retryMaybeCommitted = true;
  };

  FDBRetryStrategy()
      : FDBRetryStrategy(Config()) {}
  FDBRetryStrategy(Config config)
      : config_(config),
        backoff_(kMinBackoff),
        retry_(0) {}

  template <typename Txn>
  Result<Void> init(Txn *txn) {
    retry_ = 0;
    backoff_ = kMinBackoff;

    auto *fdbTransaction = dynamic_cast<FDBTransaction *>(txn);
    if (fdbTransaction) {
      uint64_t value = std::chrono::duration_cast<std::chrono::milliseconds>(config_.maxBackoff).count();
      auto result = fdbTransaction->setOption(FDBTransactionOption::FDB_TR_OPTION_MAX_RETRY_DELAY,
                                              std::string_view((char *)&value, sizeof(value)));
      if (result.hasError()) {
        XLOGF(ERR, "Failed to set option on FDBTransaction, {}", result.error().describe());
        RETURN_ERROR(result);
      }
    }

    return Void{};
  }

  template <typename Txn>
  CoTryTask<void> onError(Txn *txn, Status error) {
    if (retry_ >= config_.maxRetryCount) {
      XLOGF(ERR, "Transaction failed after retry {} times, error {}", retry_, error.describe());
      co_return makeError(std::move(error));
    }
    if (!TransactionHelper::isTransactionError(error)) {
      XLOGF(DBG, "Not transaction error {}", error);
      co_return makeError(std::move(error));
    }
    XLOGF(DBG, "Transaction error {}", error);

    SCOPE_EXIT {
      // update retry and backoff before exit.
      backoff_ = std::min(config_.maxBackoff, Duration(backoff_ * 2));
      retry_++;
    };

    auto *fdbTransaction = dynamic_cast<FDBTransaction *>(txn);
    if (fdbTransaction) {
      co_return co_await fdbBackoff(fdbTransaction, std::move(error));
    } else {
      co_return co_await defaultBackoff(txn, std::move(error));
    }
  }

  CoTryTask<void> fdbBackoff(FDBTransaction *txn, Status error) {
    auto errcode = txn->errcode();
    if (UNLIKELY(!errcode)) {
      XLOGF_IF(CRITICAL,
               error.code() != TransactionCode::kTooOld,
               "Failed to get FDB errcode, error {}, stacktrace {}",
               error,
               folly::symbolizer::getStackTraceStr());
      co_return co_await defaultBackoff(txn, std::move(error));
    }

    FDBErrorPredicate predict =
        config_.retryMaybeCommitted ? FDB_ERROR_PREDICATE_RETRYABLE : FDB_ERROR_PREDICATE_RETRYABLE_NOT_COMMITTED;
    if (!fdb_error_predicate(predict, errcode)) {
      XLOGF(ERR, "Transaction error not retryable: {}, errcode {}", error, errcode);
      co_return makeError(std::move(error));
    }

    XLOGF(DBG, "FDBRetryStrategy backoff by FoundationDB");
    auto ok = co_await txn->onError(errcode);
    if (!ok) {
      co_return makeError(std::move(error));
    }
    co_return Void{};
  }

  template <typename Txn>
  CoTryTask<void> defaultBackoff(Txn *txn, Status error) {
    // fallback to our backoff implementation
    if (!TransactionHelper::isRetryable(error, config_.retryMaybeCommitted)) {
      XLOGF(ERR, "Transaction error not retryable: {}", error);
      co_return makeError(std::move(error));
    }
    XLOGF(WARN, "Transaction retryable error: {}", error);

    XLOGF(DBG, "FDBRetryStrategy backoff transaction {}ms", backoff_.count());
    txn->reset();

    auto duration = Duration(backoff_ / 100 * folly::Random::rand32(80, 120));
    co_await folly::coro::sleep(duration.asUs());
    co_return Void{};
  }

 private:
  Config config_;
  Duration backoff_;
  size_t retry_;
};

}  // namespace hf3fs::kv

#pragma once

#include <array>
#include <cassert>
#include <cstdint>
#include <folly/Conv.h>
#include <folly/logging/xlog.h>
#include <map>
#include <optional>
#include <string>
#include <string_view>
#include <type_traits>
#include <utility>
#include <variant>
#include <vector>

#include "common/app/NodeId.h"
#include "common/monitor/Recorder.h"
#include "common/monitor/Sample.h"
#include "common/serde/Serde.h"
#include "common/utils/Duration.h"
#include "common/utils/MurmurHash3.h"
#include "common/utils/Reflection.h"
#include "common/utils/Result.h"
#include "common/utils/SerDeser.h"
#include "common/utils/Status.h"
#include "common/utils/StatusCode.h"
#include "common/utils/TypeTraits.h"
#include "common/utils/Uuid.h"
#include "fbs/core/user/User.h"
#include "fbs/meta/Common.h"
#include "fbs/meta/Schema.h"
#include "fbs/meta/Service.h"
#include "fbs/mgmtd/ChainRef.h"
#include "fbs/mgmtd/RoutingInfo.h"
#include "fmt/core.h"

namespace hf3fs::meta {

struct ErrorHandling {
  static bool success(const auto &result) { return !result.hasError() || success(result.error()); }

  // The operation was performed successfully, or an expected error code was returned
  static bool success(const Status &status) {
    auto code = status.code();
    switch (StatusCode::typeOf(code)) {
      case StatusCodeType::Common:
        switch (code) {
          case StatusCode::kOK:
          case StatusCode::kInvalidArg:
          case StatusCode::kAuthenticationFail:
            return true;
          default:
            return false;
        }
      case StatusCodeType::Meta:
        switch (code) {
          case MetaCode::kNotFound:
          case MetaCode::kNotEmpty:
          case MetaCode::kNotDirectory:
          case MetaCode::kTooManySymlinks:
          case MetaCode::kIsDirectory:
          case MetaCode::kExists:
          case MetaCode::kNoPermission:
          case MetaCode::kNotFile:
          case MetaCode::kInvalidFileLayout:
          case MetaCode::kMoreChunksToRemove:
          case MetaCode::kNameTooLong:
            return true;
          default:
            return (code >= MetaCode::kExpected && code < MetaCode::kRetryable);
        }
      default:
        return false;
    }
  }

  static bool retryable(const Status &status) {
    auto code = status.code();
    switch (StatusCode::typeOf(code)) {
      case StatusCodeType::Common:
        return false;
      case StatusCodeType::Meta:
        switch (code) {
          case MetaCode::kNotFound:
          case MetaCode::kNotEmpty:
          case MetaCode::kNotDirectory:
          case MetaCode::kTooManySymlinks:
          case MetaCode::kIsDirectory:
          case MetaCode::kExists:
          case MetaCode::kNoPermission:
          case MetaCode::kInconsistent:
          case MetaCode::kNotFile:
          case MetaCode::kBadFileSystem:
          case MetaCode::kInvalidFileLayout:
          case MetaCode::kFileHasHole:
          case MetaCode::kNameTooLong:
          case MetaCode::kRequestCanceled:
          case MetaCode::kFoundBug:
            return false;
          case MetaCode::kOTruncFailed:
          case MetaCode::kMoreChunksToRemove:
          case MetaCode::kBusy:
          case MetaCode::kInodeIdAllocFailed:
            return true;
          default:
            return code < MetaCode::kNotRetryable;
        }
      case StatusCodeType::StorageClient:
        switch (code) {
          case StorageClientCode::kChunkNotFound:
          case StorageClientCode::kChecksumMismatch:
          case StorageClientCode::kReadOnlyServer:
          case StorageClientCode::kFoundBug:
            return false;
          default:
            return true;
        }
      default:
        return code != RPCCode::kInvalidMethodID;
    }
  }

  static bool serverError(const Status &status) {
    switch (StatusCode::typeOf(status.code())) {
      case StatusCodeType::Transaction:
        return status.code() == TransactionCode::kNetworkError;
      case StatusCodeType::RPC:
        return true;
      default:
        return false;
    }
  }

  static bool needPruneSession(const Status &error) {
    auto code = error.code();
    switch (StatusCode::typeOf(code)) {
      case StatusCodeType::Transaction:
        return code == TransactionCode::kMaybeCommitted;
      case StatusCodeType::Meta:
        return code == MetaCode::kOTruncFailed;
      case StatusCodeType::RPC:
        switch (code) {
          case RPCCode::kSendFailed:
          case RPCCode::kConnectFailed:
          case RPCCode::kIBInitFailed:
          case RPCCode::kInvalidMethodID:
            return false;
          default:
            return true;
        }
      default:
        return false;
    }
  }
};

class OperationRecorder {
 public:
  OperationRecorder(std::string_view prefix)
      : total_(fmt::format("{}_total", prefix)),
        failed_(fmt::format("{}_failed", prefix)),
        running_(fmt::format("{}_running", prefix), false),
        code_(fmt::format("{}_code", prefix)),
        latency_(fmt::format("{}_latency", prefix)),
        retry_(fmt::format("{}_retry", prefix)),
        idempotent_(fmt::format("{}_idempotent", prefix)),
        duplicate_(fmt::format("{}_duplicate", prefix)) {}

  static OperationRecorder &server() {
    static OperationRecorder recorder("meta_server.op");
    return recorder;
  }

  static OperationRecorder &client() {
    static OperationRecorder recorder("meta_client.op");
    return recorder;
  }

  class Guard {
   public:
    Guard(OperationRecorder &recorder, std::string_view op, flat::Uid user)
        : recorder_(recorder),
          op_("instance", std::string(op)),
          user_("uid", folly::to<std::string>((int32_t)user.toUnderType())),
          begin_(RelativeTime::now()) {
      begin_ = RelativeTime::now();
      recorder_.running_.addSample(1, {{op_}});
    }

    ~Guard() {
      auto op = monitor::TagSet{op_};
      auto opUser = monitor::TagSet{{op_, user_}};
      auto latency = RelativeTime::now() - begin_;
      recorder_.total_.addSample(1, opUser);
      if (!succ_) {
        recorder_.failed_.addSample(1, opUser);
        recorder_.code_.addSample(1, {{"tag", folly::to<std::string>(code_.value_or(0))}, user_});
      } else {
        if (code_) {
          recorder_.code_.addSample(1, {{"tag", folly::to<std::string>(*code_)}, user_});
        }
      }
      recorder_.running_.addSample(-1, op);
      recorder_.latency_.addSample(latency, op);
      recorder_.retry_.addSample(retry_, op);
      if (duplicate_) {
        recorder_.duplicate_.addSample(1, op);
      }
    }

    void finish(const auto &result, bool duplicate = false) {
      duplicate_ = duplicate;
      succ_ = ErrorHandling::success(result);
      code_ = result.hasError() ? result.error().code() : 0;
    }

    void finish(const Status &status, bool duplicate = false) {
      duplicate_ = duplicate;
      succ_ = ErrorHandling::success(status) || status.code() == MetaCode::kRequestCanceled;
      code_ = status.code();
    }

    int &retry() { return retry_; }

   private:
    using Tag = std::pair<std::string, std::string>;
    OperationRecorder &recorder_;
    Tag op_;
    Tag user_;
    RelativeTime begin_;
    std::optional<status_code_t> code_;
    bool succ_ = false;
    int retry_ = 1;
    bool duplicate_ = false;
  };

  void addIdempotentCount() { idempotent_.addSample(1); }

 private:
  monitor::CountRecorder total_;
  monitor::CountRecorder failed_;
  monitor::CountRecorder running_;
  monitor::CountRecorder code_;
  monitor::LatencyRecorder latency_;
  monitor::DistributionRecorder retry_;
  monitor::CountRecorder idempotent_;
  monitor::CountRecorder duplicate_;
};

struct Weight : std::array<uint8_t, 16> {
  static Weight calculate(flat::NodeId node, InodeId inodeId) {
    // Note: don't change this
    auto key = Serializer::serRawArgs((uint64_t)node.toUnderType(), inodeId.u64());
    return hash(key.data(), key.size());
  }

  static flat::NodeId select(const std::vector<flat::NodeId> &nodes, InodeId inodeId) {
    return selectImpl(nodes, inodeId);
  }

  static Weight calculate(flat::NodeId node, Uuid clientId) {
    // todo: maybe should change to client host name?
    auto key = Serializer::serRawArgs((uint64_t)node.toUnderType(), clientId.data);
    return hash(key.data(), key.size());
  }

  static flat::NodeId select(const std::vector<flat::NodeId> &nodes, Uuid clientId) {
    return selectImpl(nodes, clientId);
  }

 private:
  static Weight hash(void *key, size_t len) {
    // NOTE: don't change this
    Weight w;
    MurmurHash3_x64_128(key, len, 0, &w);
    return w;
  }

  static flat::NodeId selectImpl(const std::vector<flat::NodeId> &nodes, auto &key) {
    if (nodes.empty()) {
      return flat::NodeId(0);  // flat::NodeId(0) is invalid
    }

    auto node = nodes.at(0);
    auto weight = calculate(node, key);
    for (size_t i = 1; i < nodes.size(); i++) {
      auto n = nodes.at(i);
      auto w = calculate(n, key);
      if (w > weight) {
        node = n;
        weight = w;
      }
    }
    return node;
  }
};
static_assert(sizeof(Weight) == 16);

struct RoutingInfoChecker {
  template <typename T>
  static constexpr bool hasInode() {
    return false;
  }

  template <typename T>
  requires serde::SerdeType<T>
  static constexpr bool hasInode() {
    if constexpr (std::is_same_v<Inode, T>) {
      return true;
    }
    bool inode = false;
    refl::Helper::iterate<T>([&](auto info) {
      using Item = std::remove_reference_t<decltype(std::declval<T>().*info.getter)>;
      inode |= hasInode<Item>();
    });
    return inode;
  }

  template <typename T>
  requires is_variant_v<T>
  static constexpr bool hasInode() { return variantHasInode<T>(); }

  template <typename T, size_t I = 0>
  static constexpr bool variantHasInode() {
    if constexpr (I < std::variant_size_v<T>) {
      if constexpr (hasInode<std::variant_alternative<I, T>>()) {
        return true;
      }
      return variantHasInode<T, I + 1>();
    }
    return false;
  }

  template <typename T>
  requires is_vector_v<T> || is_set_v<T>
  static constexpr bool hasInode() { return hasInode<typename T::value_type>(); }

  template <typename T>
  requires is_map_v<T>
  static constexpr bool hasInode() { return hasInode<typename T::key_type>() || hasInode<typename T::mapped_type>(); }

  template <typename T>
  requires is_optional_v<T>
  static constexpr bool hasInode() { return hasInode<typename T::value_type>(); }

  static bool checkRoutingInfo(const Inode &inode, const flat::RoutingInfo &routing) {
    if (inode.isFile()) {
      auto table = inode.asFile().layout.tableId;
      auto tableVer = inode.asFile().layout.tableVersion;
      switch (inode.asFile().layout.type()) {
        case Layout::Type::ChainRange:
          XLOGF_IF(DFATAL, (!table || !tableVer), "File {}, invalid layout", inode);
          if (!routing.getChainTable(table, tableVer)) {
            XLOGF(WARN, "File {}, chain table {} version {}, not found in RoutingInfo", inode.id, table, tableVer);
            return false;
          }
          break;
        case Layout::Type::ChainList:
          if (table && tableVer && !routing.getChainTable(table, tableVer)) {
            XLOGF(WARN, "File {}, chain table {} version {}, not found in RoutingInfo", inode.id, table, tableVer);
            return false;
          }
          break;
        case Layout::Type::Empty:
          break;
      }
    }
    return true;
  }

  template <typename T>
  static bool checkRoutingInfo(const T &, const flat::RoutingInfo &) {
    return true;
  }

  template <typename T>
  requires serde::SerdeType<T>
  static bool checkRoutingInfo(const T &t, const flat::RoutingInfo &routing) {
    if constexpr (!hasInode<T>()) {
      return true;
    }

    bool succ = true;
    refl::Helper::iterate<T>([&](auto info) { succ &= checkRoutingInfo(t.*info.getter, routing); });
    return succ;
  }

  template <typename T>
  requires is_variant_v<T>
  static bool checkRoutingInfo(const T &t, const flat::RoutingInfo &routing) {
    if constexpr (!hasInode<T>()) {
      return true;
    }
    return std::visit([&](const auto &v) { return checkRoutingInfo(v, routing); }, t);
  }

  template <typename T>
  requires is_vector_v<T> || is_set_v<T>
  static bool checkRoutingInfo(const T &t, const flat::RoutingInfo &routing) {
    if constexpr (!hasInode<T>()) {
      return true;
    }
    for (auto &i : t) {
      if (!checkRoutingInfo(i, routing)) {
        return false;
      }
    }
    return true;
  }

  template <typename T>
  requires is_map_v<T>
  static bool checkRoutingInfo(const T &t, const flat::RoutingInfo &routing) {
    if constexpr (!hasInode<T>()) {
      return true;
    }
    for (auto &[k, v] : t) {
      if (!checkRoutingInfo(k, routing) || !checkRoutingInfo(v, routing)) {
        return false;
      }
    }
    return true;
  }

  template <typename T>
  requires is_optional_v<T>
  static bool checkRoutingInfo(const T &t, const flat::RoutingInfo &routing) {
    if constexpr (!hasInode<T>()) {
      return true;
    }
    return t ? checkRoutingInfo(*t, routing) : true;
  }
};

static_assert(RoutingInfoChecker::hasInode<Inode>());
static_assert(RoutingInfoChecker::hasInode<std::vector<Inode>>());
static_assert(RoutingInfoChecker::hasInode<std::set<Inode>>());
static_assert(RoutingInfoChecker::hasInode<std::optional<Inode>>());
static_assert(RoutingInfoChecker::hasInode<std::map<Inode, int>>());
static_assert(RoutingInfoChecker::hasInode<std::map<int, Inode>>());
static_assert(RoutingInfoChecker::hasInode<OpenRsp>());
static_assert(RoutingInfoChecker::hasInode<StatRsp>());
static_assert(RoutingInfoChecker::hasInode<SyncRsp>());
static_assert(RoutingInfoChecker::hasInode<CloseRsp>());
static_assert(RoutingInfoChecker::hasInode<BatchStatRsp>());
}  // namespace hf3fs::meta

#pragma once

#include <map>
#include <mutex>

#include "kv/KVStore.h"

namespace hf3fs::kv {

class MemDBStore : public KVStore {
 public:
  MemDBStore(const Config &config)
      : config_(config) {}

  // get value corresponding to key.
  Result<std::string> get(std::string_view key) final {
    auto lock = std::unique_lock(mutex_);
    auto it = map_.find(std::string{key});
    if (it == map_.end()) {
      return makeError(StatusCode::kKVStoreNotFound);
    }
    return it->second;
  }

  // get the first key which is greater than input key.
  Result<Void> iterateKeysWithPrefix(std::string_view prefix,
                                     uint32_t limit,
                                     IterateFunc func,
                                     std::optional<std::string> *nextValidKey = nullptr) final {
    auto lock = std::unique_lock(mutex_);
    auto it = map_.lower_bound(std::string{prefix});
    for (auto i = 0u; i < limit && it != map_.end(); ++i, ++it) {
      if (it->first.starts_with(prefix)) {
        RETURN_ON_ERROR(func(it->first, it->second));
      } else {
        break;
      }
    }
    if (nextValidKey && it != map_.end() && it->first.starts_with(prefix)) {
      *nextValidKey = it->first;
    }
    return Void{};
  }

  // put a key-value pair.
  Result<Void> put(std::string_view key, std::string_view value, bool) final {
    auto lock = std::unique_lock(mutex_);
    map_[std::string{key}] = value;
    return Void{};
  }

  // remove a key-value pair.
  Result<Void> remove(std::string_view key) final {
    auto lock = std::unique_lock(mutex_);
    map_.erase(std::string{key});
    return Void{};
  }

  // batch operations.
  class MemBatchOperations : public BatchOperations {
   public:
    MemBatchOperations(MemDBStore &db)
        : db_(db) {}
    // put a key-value pair.
    void put(std::string_view key, std::string_view value) override {
      writeBatch_.emplace_back(std::string{key}, std::string{value});
    }
    // remove a key.
    void remove(std::string_view key) override { writeBatch_.emplace_back(std::string{key}, std::nullopt); }
    // clear a batch operations.
    void clear() override { writeBatch_.clear(); }
    // commit a batch of operations.
    Result<Void> commit() override {
      auto lock = std::unique_lock(db_.mutex_);
      for (auto &pair : writeBatch_) {
        if (pair.second.has_value()) {
          db_.map_[std::string{pair.first}] = pair.second.value();
        } else {
          db_.map_.erase(std::string{pair.first});
        }
      }
      return Void{};
    }
    // destroy self.
    void destroy() override { delete this; }

   private:
    MemDBStore &db_;
    std::vector<std::pair<std::string, std::optional<std::string>>> writeBatch_;
  };
  BatchOptionsPtr createBatchOps() override {
    return BatchOptionsPtr{std::make_unique<MemBatchOperations>(*this).release()};
  }

  // iterator.
  class MemIterator : public Iterator {
   public:
    MemIterator(MemDBStore &db)
        : lock_(db.mutex_),
          map_(db.map_),
          iterator_(map_.end()) {}
    void seek(std::string_view key) override { iterator_ = map_.lower_bound(std::string{key}); }
    void seekToFirst() override { iterator_ = map_.begin(); }
    void seekToLast() override { iterator_ = --map_.end(); }
    void next() override { ++iterator_; }
    Result<Void> status() const override { return Void{}; }
    bool valid() const override { return iterator_ != map_.end(); }
    std::string_view key() const override { return iterator_->first; }
    std::string_view value() const override { return iterator_->second; }
    void destroy() override { delete this; }

   private:
    std::unique_lock<std::mutex> lock_;
    std::map<std::string, std::string> &map_;
    std::map<std::string, std::string>::iterator iterator_;
  };
  IteratorPtr createIterator() override { return IteratorPtr{std::make_unique<MemIterator>(*this).release()}; }

 private:
  [[maybe_unused]] const Config &config_;
  std::mutex mutex_;
  std::map<std::string, std::string> map_;
};

}  // namespace hf3fs::kv

#pragma once

#include <type_traits>

#include "common/serde/Serde.h"
#include "common/utils/Duration.h"
#include "common/utils/Result.h"

namespace hf3fs::serde {

enum EssentialFlags : uint16_t {
  IsReq = (1 << 0),
  UseCompress = (1 << 1),
  ControlRDMA = (1 << 2),
};

struct Version {
  SERDE_STRUCT_FIELD(major, uint8_t{});
  SERDE_STRUCT_FIELD(minor, uint8_t{});
  SERDE_STRUCT_FIELD(patch, uint8_t{});
  SERDE_STRUCT_FIELD(hash, uint32_t{});
};

template <class T>
struct PointerWrapper {
 public:
  PointerWrapper() = default;
  explicit PointerWrapper(const T &t)
      : t(&t) {}
  const T *t = nullptr;
};

template <class T>
using Payload = std::conditional_t<std::is_same_v<T, Void>, std::string_view, PointerWrapper<T>>;

struct Timestamp {
  auto serverLatency() const { return Duration(std::chrono::microseconds{serverSerialized - serverReceived}); }
  auto inflightLatency() const { return Duration(std::chrono::microseconds{clientReceived - clientSerialized}); }
  auto networkLatency() const { return inflightLatency() - serverLatency(); }
  auto queueLatency() const { return Duration(std::chrono::microseconds{serverWaked - serverReceived}); }
  auto totalLatency() const { return Duration(std::chrono::microseconds{clientWaked - clientCalled}); }

  SERDE_STRUCT_FIELD(clientCalled, int64_t{});
  SERDE_STRUCT_FIELD(clientSerialized, int64_t{});
  SERDE_STRUCT_FIELD(serverReceived, int64_t{});
  SERDE_STRUCT_FIELD(serverWaked, int64_t{});
  SERDE_STRUCT_FIELD(serverProcessed, int64_t{});
  SERDE_STRUCT_FIELD(serverSerialized, int64_t{});
  SERDE_STRUCT_FIELD(clientReceived, int64_t{});
  SERDE_STRUCT_FIELD(clientWaked, int64_t{});
};

template <class T = Void>
struct MessagePacket {
  MessagePacket() requires(std::is_same_v<T, Void>) = default;
  explicit MessagePacket(const T &req)
      : payload(req) {}

  bool isRequest() const { return flags & EssentialFlags::IsReq; }
  bool useCompress() const { return flags & EssentialFlags::UseCompress; }
  bool controlRDMA() const { return flags & EssentialFlags::ControlRDMA; }

  SERDE_STRUCT_FIELD(uuid, uint64_t{});
  SERDE_STRUCT_FIELD(serviceId, uint16_t{});
  SERDE_STRUCT_FIELD(methodId, uint16_t{});
  SERDE_STRUCT_FIELD(flags, uint16_t{});
  SERDE_STRUCT_FIELD(version, Version{});
  SERDE_STRUCT_FIELD(payload, Payload<T>{});
  SERDE_STRUCT_FIELD(timestamp, std::optional<Timestamp>{});
};

}  // namespace hf3fs::serde

template <class T>
requires(!std::is_same_v<T, hf3fs::Void>) struct hf3fs::serde::SerdeMethod<hf3fs::serde::PointerWrapper<T>> {
  static void serialize(const hf3fs::serde::PointerWrapper<T> &payload, auto &out) {
    auto size = out.tableBegin(false);
    serde::serialize(*payload.t, out);
    out.tableEnd(size);
  }

  static void serializeReadable(const hf3fs::serde::PointerWrapper<T> &payload, auto &out) {
    serde::serialize(*payload.t, out);
  }

  static Result<Void> deserialize(hf3fs::serde::PointerWrapper<T> &, auto &) {
    return makeError(StatusCode::kSerdeNotContainer);
  }
};

#pragma once

#include <cerrno>
#include <folly/FBString.h>
#include <folly/IPAddressV4.h>
#include <folly/Range.h>
#include <folly/experimental/Bits.h>
#include <folly/logging/xlog.h>
#include <folly/system/Shell.h>
#include <map>
#include <net/if.h>
#include <sys/ioctl.h>
#include <vector>

#include "common/utils/Result.h"
#include "ifaddrs.h"

namespace hf3fs::net {

class IfAddrs {
 public:
  struct Addr {
    folly::IPAddressV4 ip;
    size_t mask;
    bool up;

    Addr(folly::IPAddressV4 ip, size_t mask, bool up)
        : ip(ip),
          mask(mask),
          up(up) {}
    folly::CIDRNetworkV4 subnet() const { return {ip.mask(mask), mask}; }
  };
  using Map = std::multimap<String, Addr>;

  static Result<std::multimap<String, Addr>> load() {
    static Result<std::multimap<String, Addr>> result = loadOnce();
    return result;
  }

 private:
  static Result<Map> loadOnce() {
    Map net2addr;
    struct ifaddrs *ifaddrList;
    if (getifaddrs(&ifaddrList)) {
      XLOGF(ERR, "Failed to call getifaddrs, errno {}", errno);
      return makeError(StatusCode::kOSError, "Failed to call getifaddrs");
    }

    for (auto ifaddrIter = ifaddrList; ifaddrIter != nullptr; ifaddrIter = ifaddrIter->ifa_next) {
      auto name = ifaddrIter->ifa_name;
      if (ifaddrIter->ifa_addr == nullptr || ifaddrIter->ifa_netmask == nullptr) {
        XLOGF(DBG,
              "Skip ifaddr of {}, ifa_addr {} or ifa_netmask {} missing.",
              name,
              (void *)ifaddrIter->ifa_addr,
              (void *)ifaddrIter->ifa_netmask);
        continue;
      }
      if (ifaddrIter->ifa_addr->sa_family != AF_INET || ifaddrIter->ifa_netmask->sa_family != AF_INET) {
        XLOGF(DBG,
              "Skip ifaddr of {}, address family {} {} != AF_INET",
              name,
              ifaddrList->ifa_addr->sa_family,
              ifaddrIter->ifa_netmask->sa_family);
        continue;
      }

      auto ip = folly::IPAddressV4(((sockaddr_in *)ifaddrIter->ifa_addr)->sin_addr);
      auto mask = folly::IPAddressV4(((sockaddr_in *)ifaddrIter->ifa_netmask)->sin_addr).toByteArray();
      auto maskBits = folly::Bits<uint8_t>::count(mask.begin(), mask.end());
      if (folly::IPAddressV4::fetchMask(maskBits) != mask) {
        XLOGF(WARN, "IfAddr of {}, addr {}, mask {} is not valid!!!", name, ip.str(), folly::IPAddressV4(mask).str());
        continue;
      }

      bool up = false;
      int sock = socket(AF_INET, SOCK_DGRAM, 0);
      if (sock < 0) {
        return makeError(StatusCode::kOSError, "Failed to create socket {}", errno);
      } else {
        struct ifreq ifr;
        strncpy(ifr.ifr_name, ifaddrIter->ifa_name, IFNAMSIZ - 1);
        if (ioctl(sock, SIOCGIFFLAGS, &ifr) != 0) {
          XLOG(ERR, "Failed to query status of {}, errno {}", name, errno);
          up = false;
        } else {
          up = (ifr.ifr_flags & IFF_UP);
        }
        close(sock);
      }

      auto addr = Addr{ip, maskBits, up};
      net2addr.emplace(name, addr);

      auto fbs = folly::StringPiece(ifaddrIter->ifa_name);
      XLOGF_IF(INFO,
               fbs.startsWith("en") || fbs.startsWith("eth") || fbs.startsWith("ib") || fbs.startsWith("bond"),
               "Get ifaddr of {}, addr {}, subnet {}, up {}",
               ifaddrIter->ifa_name,
               addr,
               addr.subnet(),
               up);
    }

    freeifaddrs(ifaddrList);

    return net2addr;
  }
};

}  // namespace hf3fs::net

FMT_BEGIN_NAMESPACE

template <>
struct formatter<hf3fs::net::IfAddrs::Addr> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(const hf3fs::net::IfAddrs::Addr &addr, FormatContext &ctx) const {
    return fmt::format_to(ctx.out(), "{}/{}", addr.ip.str(), addr.mask);
  }
};

template <>
struct formatter<folly::CIDRNetworkV4> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(const folly::CIDRNetworkV4 &network, FormatContext &ctx) const {
    return fmt::format_to(ctx.out(), "{}/{}", network.first.str(), network.second);
  }
};

FMT_END_NAMESPACE

#pragma once

#include <cstdint>
#include <folly/Utility.h>
#include <folly/logging/xlog.h>
#include <optional>
#include <string_view>
#include <vector>

#include "common/utils/Coroutine.h"
#include "common/utils/Result.h"
#include "common/utils/Status.h"
#include "common/utils/String.h"
#include "common/utils/Transform.h"

namespace hf3fs::kv {

// - The metadata version key "\xff/metadataVersion" is a key intended to help layers deal with hot keys. The value of
// this key is sent to clients along with the read version from the proxy, so a client can read its value without
// communicating with a storage server. The value of this key can only be set with SetVersionStampValue operations.
// - fdbdr will always capture changes to this key, regardless of the specified range for the DR.
// - Restore will set this key right before a restore is complete.
// - Calling setVersion on a transaction does not include the value of the metadataVersionKey, so a cache of recent
// version to metadataVersion mappings are kept in the database context. This is useful when caching read versions from
// one transaction and calling setVersion on a different transaction to avoid waiting for read version there.
static constexpr std::string_view kMetadataVersionKey = "\xff/metadataVersion";

// A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed
// transaction. The first 8 bytes are the committed version of the database (serialized in big-endian order). The last
// 2 bytes are monotonic in the serialization order for transactions (serialized in big-endian order).
using Versionstamp = std::array<uint8_t, 10>;
static_assert(sizeof(Versionstamp) == 10);

class IReadOnlyTransaction {
 public:
  virtual ~IReadOnlyTransaction() = default;

  virtual void setReadVersion(int64_t version) = 0;

  virtual CoTryTask<std::optional<String>> snapshotGet(std::string_view key) = 0;

  virtual CoTryTask<std::optional<String>> get(std::string_view key) {
    // fallback to snapshotGet by default.
    co_return co_await snapshotGet(key);
  }

  struct KeyValue {
    String key;
    String value;

    KeyValue(String k, String v)
        : key(std::move(k)),
          value(std::move(v)) {}

    KeyValue(std::string_view k, std::string_view v)
        : key(k),
          value(v) {}

    std::tuple<const String &, const String &> pair() const { return {key, value}; }
  };

  struct KeySelector {
    std::string_view key;
    bool inclusive;

    KeySelector(std::string_view k, bool argInclusive)
        : key(k),
          inclusive(argInclusive) {}
  };

  struct GetRangeResult : public folly::MoveOnly {
    std::vector<KeyValue> kvs;
    bool hasMore;

    GetRangeResult(std::vector<KeyValue> argKvs, bool argHasMore)
        : kvs(std::move(argKvs)),
          hasMore(argHasMore) {}
  };
  virtual CoTryTask<GetRangeResult> snapshotGetRange(const KeySelector &begin,
                                                     const KeySelector &end,
                                                     int32_t limit) = 0;
  virtual CoTryTask<GetRangeResult> getRange(const KeySelector &begin, const KeySelector &end, int32_t limit) = 0;

  virtual CoTryTask<void> cancel() = 0;
  virtual void reset() = 0;
};

class IReadWriteTransaction : public IReadOnlyTransaction {
 public:
  ~IReadWriteTransaction() override = default;

  // The difference of `snapshotGet` and `get` is the former needs no conflict validation and hence won't cause a
  // read-write transaction fail.
  CoTryTask<std::optional<String>> get(std::string_view key) override = 0;
  CoTryTask<GetRangeResult> getRange(const KeySelector &begin, const KeySelector &end, int32_t limit) override = 0;

  // Add a read conflict key to transaction without performing associated read.
  virtual CoTryTask<void> addReadConflict(std::string_view key) = 0;
  virtual CoTryTask<void> addReadConflictRange(std::string_view begin, std::string_view end) = 0;

  virtual CoTryTask<void> set(std::string_view key, std::string_view value) = 0;
  virtual CoTryTask<void> clear(std::string_view key) = 0;

  // A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed
  // transaction. The first 8 bytes are the committed version of the database (serialized in big-endian order). The last
  // 2 bytes are monotonic in the serialization order for transactions (serialized in big-endian order).
  virtual CoTryTask<void> setVersionstampedKey(std::string_view key, uint32_t offset, std::string_view value) = 0;
  virtual CoTryTask<void> setVersionstampedValue(std::string_view key, std::string_view value, uint32_t offset) = 0;

  virtual CoTryTask<void> commit() = 0;
  virtual int64_t getCommittedVersion() = 0;
};

struct TransactionHelper {
  static bool isTransactionError(const Status &error);

  static bool isRetryable(const Status &error, bool allowMaybeCommitted);

  static String keyAfter(std::string_view key);

  static String prefixListEndKey(std::string_view prefix);

  struct ListByPrefixOptions {
    bool snapshot = true;
    bool inclusive = true;
    size_t limit = 0;

    ListByPrefixOptions() = default;
    ListByPrefixOptions &withSnapshot(bool v) {
      snapshot = v;
      return *this;
    }
    ListByPrefixOptions &withInclusive(bool v) {
      inclusive = v;
      return *this;
    }
    ListByPrefixOptions &withLimit(size_t v) {
      limit = v;
      return *this;
    }
  };

  using KeyValue = IReadOnlyTransaction::KeyValue;
  static CoTryTask<std::vector<KeyValue>> listByPrefix(IReadOnlyTransaction &txn,
                                                       std::string_view prefix,
                                                       ListByPrefixOptions options);

  template <typename T, typename Fn>
  static CoTryTask<std::vector<T>> listByPrefix(IReadOnlyTransaction &txn,
                                                std::string_view prefix,
                                                ListByPrefixOptions options,
                                                Fn &&unpack) {
    auto listRes = co_await listByPrefix(txn, prefix, options);
    CO_RETURN_ON_ERROR(listRes);
    std::vector<T> res;
    for (auto &[k, v] : *listRes) {
      auto entry = unpack(k, v);
      CO_RETURN_ON_ERROR(entry);
      res.push_back(std::move(*entry));
    }
    co_return res;
  }
};

}  // namespace hf3fs::kv

#pragma once

#include <folly/logging/xlog.h>
#include <functional>
#include <memory>
#include <type_traits>
#include <variant>

#include "client/mgmtd/ICommonMgmtdClient.h"
#include "client/mgmtd/MgmtdClientForServer.h"
#include "common/app/NodeId.h"
#include "common/net/Client.h"
#include "common/net/RequestOptions.h"
#include "common/serde/CallContext.h"
#include "common/serde/ClientContext.h"
#include "common/serde/ClientMockContext.h"
#include "common/serde/MessagePacket.h"
#include "common/utils/Address.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Result.h"
#include "common/utils/StatusCode.h"
#include "fbs/meta/Common.h"
#include "fbs/meta/Service.h"
#include "fbs/meta/Utils.h"
#include "fmt/core.h"
#include "meta/store/Inode.h"

namespace hf3fs::meta::server {

class Forward {
 public:
  struct Config : ConfigBase<Config> {
    CONFIG_HOT_UPDATED_ITEM(debug, true);
    CONFIG_HOT_UPDATED_ITEM(addr_type, net::Address::Type::RDMA);
    CONFIG_HOT_UPDATED_ITEM(timeout, 10_s);
  };

  using NetClient = std::reference_wrapper<net::Client>;
  using MockClient = std::reference_wrapper<std::map<flat::NodeId, serde::ClientMockContext>>;

  Forward(const Config &config,
          flat::NodeId nodeId,
          std::variant<NetClient, MockClient> client,
          std::shared_ptr<::hf3fs::client::ICommonMgmtdClient> mgmtdClient)
      : config_(config),
        nodeId_(nodeId),
        client_(client),
        mgmtdClient_(mgmtdClient) {
    XLOGF_IF(FATAL, !nodeId_, "invalid nodeId {}", nodeId_);
  }

  template <typename Req, typename Rsp>
  CoTryTask<Rsp> forward(flat::NodeId node, Req req) {
    OperationRecorder::Guard record(OperationRecorder::server(), "forward", req.user.uid);
    auto result = co_await forwardImpl<Req, Rsp>(node, std::move(req));
    record.finish(result);
    co_return result;
  }

 private:
  template <typename Req, typename Rsp, typename Context>
  struct ForwardMethod {};

  template <typename Context>
  struct ForwardMethod<SyncReq, SyncRsp, Context> {
    static constexpr auto rpcMethod = MetaSerde<>::sync<Context>;
  };

  template <typename Context>
  struct ForwardMethod<CloseReq, CloseRsp, Context> {
    static constexpr auto rpcMethod = MetaSerde<>::close<Context>;
  };

  template <typename Context>
  struct ForwardMethod<SetAttrReq, SetAttrRsp, Context> {
    static constexpr auto rpcMethod = MetaSerde<>::setAttr<Context>;
  };

  template <typename Context>
  struct ForwardMethod<CreateReq, CreateRsp, Context> {
    static constexpr auto rpcMethod = MetaSerde<>::create<Context>;
  };

  template <typename Req>
  Result<Void> check(flat::NodeId node, Req &req) {
    if (!node) {
      XLOGF(WARN, "request {}, unknown corresponding server, need retry", req);
      return makeError(MetaCode::kForwardFailed, "unknown corresponding server");
    }
    if (req.forward) {
      XLOGF_IF(INFO, config_.debug(), "request is forward from {}, can't forward again, req {}.", req.forward, req);
      return makeError(MetaCode::kForwardFailed, "double forward, retry");
    }
    req.forward = nodeId_;

    XLOGF_IF(INFO, config_.debug(), "forward req {} to {}", req, node);
    XLOGF_IF(DBG, !config_.debug(), "forward req {} to {}", req, node);
    XLOGF_IF(FATAL, nodeId_ == node, "forward to self, {} == {}", nodeId_, node);

    return Void{};
  }

  CoTryTask<net::Address> getAddress(flat::NodeId node) {
    auto routing = mgmtdClient_->getRoutingInfo();
    if (!routing) {
      co_return makeError(MetaCode::kForwardFailed, "routing info not ready, need retry");
    }

    auto *nodeInfo = routing->raw()->getNode(node);
    if (!nodeInfo) {
      auto msg = fmt::format("req forward: routing info doesn't contains node {}", node);
      XLOG(WARN, msg);
      co_return makeError(MetaCode::kForwardFailed, std::move(msg));
    }
    auto addrs = nodeInfo->extractAddresses("MetaSerde", config_.addr_type());
    if (addrs.empty()) {
      auto msg =
          fmt::format("req forward: node {} doesn't have {} addr.", node, magic_enum::enum_name(config_.addr_type()));
      XLOG(WARN, msg);
      co_return makeError(MetaCode::kForwardFailed, std::move(msg));
    }
    co_return addrs.front();
  }

  template <typename Req, typename Rsp>
  CoTryTask<Rsp> forwardImpl(flat::NodeId node, Req req) {
    CO_RETURN_ON_ERROR(check(node, req));

    auto opts = net::UserRequestOptions();
    opts.timeout = config_.timeout();
    opts.sendRetryTimes = 3;
    opts.compression = std::nullopt;

    Result<Rsp> result = makeError(MetaCode::kFoundBug);
    if (std::holds_alternative<NetClient>(client_)) {
      auto &client = std::get<NetClient>(client_);
      auto addr = co_await getAddress(node);
      CO_RETURN_ON_ERROR(addr);
      auto ctx = client.get().serdeCtx(*addr);
      result = co_await ForwardMethod<Req, Rsp, serde::ClientContext>::rpcMethod(ctx, req, &opts, nullptr);
    } else {
      auto &client = std::get<MockClient>(client_);
      if (!client.get().contains(node)) {
        co_return makeError(MetaCode::kForwardFailed, fmt::format("{} not found", node));
      }
      auto &ctx = client.get()[node];
      result = co_await ForwardMethod<Req, Rsp, serde::ClientMockContext>::rpcMethod(ctx, req, &opts, nullptr);
    }

    if (result.hasError() && StatusCode::typeOf(result.error().code()) == StatusCodeType::RPC) {
      XLOGF(ERR, "failed to forward req to {}, error {}", node, result.error());
      co_return makeError(MetaCode::kForwardTimeout,
                          fmt::format("failed to forward req to {}, error {}", node, result.error()));
    }

    if (result.hasError()) {
      XLOGF_IF(INFO, config_.debug(), "forward req {} to {}, rsp {}", req, node, result.error());
    } else {
      XLOGF_IF(INFO, config_.debug(), "forward req {} to {}, rsp {}", req, node, result.value());
    }

    co_return result;
  }

  const Config &config_;
  flat::NodeId nodeId_;
  std::variant<NetClient, MockClient> client_;
  std::shared_ptr<::hf3fs::client::ICommonMgmtdClient> mgmtdClient_;
};

}  // namespace hf3fs::meta::server

#pragma once

#include <folly/Synchronized.h>
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <folly/fibers/Semaphore.h>
#include <limits>

#include "common/net/ib/RDMABuf.h"
#include "common/utils/CPUExecutorGroup.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/ConstructLog.h"
#include "common/utils/Size.h"

namespace hf3fs::storage {

struct BufferIndex {
  uint32_t registerIndex;
  net::RDMABuf buffer;
};

class BufferPool {
 public:
  class Config : public ConfigBase<Config> {
    CONFIG_ITEM(rdmabuf_size, 4_MB);
    CONFIG_ITEM(rdmabuf_count, 1024u);
    CONFIG_ITEM(big_rdmabuf_size, 64_MB);
    CONFIG_ITEM(big_rdmabuf_count, 64u);
  };
  BufferPool(const Config &config)
      : config_(config),
        rdmabufSize_(config_.rdmabuf_size()),
        semaphore_(config_.rdmabuf_count()),
        bigRdmabufSize_(config_.big_rdmabuf_size()),
        bigSemaphore_(config_.big_rdmabuf_count()) {}

  Result<Void> init(CPUExecutorGroup &executor);

  auto &iovecs() const { return iovecs_; }

  class Buffer {
   public:
    explicit Buffer(BufferPool &pool)
        : pool_(&pool) {}
    Buffer(const Buffer &) = delete;
    Buffer(Buffer &&other) = default;
    Buffer &operator=(Buffer &&other) = default;
    ~Buffer();

    Result<net::RDMABuf> tryAllocate(uint32_t size);

    CoTryTask<net::RDMABuf> allocate(uint32_t size);

    auto index() const { return indices_.back().registerIndex; }

   private:
    BufferPool *pool_{};
    std::vector<BufferIndex> indices_;
    net::RDMABuf current_;
  };
  auto get() { return Buffer{*this}; }

  void clear(CPUExecutorGroup &executor);

 protected:
  static Result<std::vector<BufferIndex>> initBuffers(CPUExecutorGroup &executor,
                                                      Size rdmabufSize,
                                                      uint32_t rdmabufCount,
                                                      uint32_t limit,
                                                      std::vector<net::RDMABuf> &outBuffers);

  BufferIndex allocate();

  BufferIndex allocateBig();

  void deallocate(const BufferIndex &index);

 private:
  ConstructLog<"storage::BufferPool"> constructLog_;
  const Config &config_;
  Size rdmabufSize_;
  std::vector<net::RDMABuf> buffers_;
  std::vector<struct iovec> iovecs_;
  folly::fibers::Semaphore semaphore_;
  folly::Synchronized<std::vector<BufferIndex>, std::mutex> freeIndex_;

  Size bigRdmabufSize_;
  uint32_t bigBufferRegisterIndexStart_ = 0;
  folly::fibers::Semaphore bigSemaphore_;
  folly::Synchronized<std::vector<BufferIndex>, std::mutex> bigFreeIndex_;
};

}  // namespace hf3fs::storage

#pragma once

#include <folly/Conv.h>
#include <optional>

#include "client/cli/common/Utils.h"
#include "common/utils/ArgParse.h"
#include "fbs/meta/Schema.h"
#include "fbs/mgmtd/ChainRef.h"

namespace hf3fs::client::cli {

inline void addLayoutArguments(argparse::ArgumentParser &parser) {
  parser.add_argument("--chain-table-id").scan<'u', uint32_t>();
  parser.add_argument("--chain-table-ver").scan<'u', uint32_t>();
  parser.add_argument("--chain-list").help("Chain IDs separated by ',', eg: 1,2,3,4,5, can use with chain-table");
  parser.add_argument("--chunk-size");
  parser.add_argument("--stripe-size").scan<'u', uint32_t>();
}

inline std::optional<meta::Layout> parseLayout(const argparse::ArgumentParser &parser,
                                               std::optional<meta::Layout> base = {}) {
  auto chainTableId = parser.present<uint32_t>("--chain-table-id");
  auto chainTableVersion = parser.present<uint32_t>("--chain-table-ver");
  auto chainList = parser.present<std::string>("--chain-list");
  std::optional<uint32_t> chunkSize;
  if (auto l = parser.present<std::string>("--chunk-size")) {
    chunkSize = Size::from(*l).value();
  }
  auto stripeSize = parser.present<uint32_t>("--stripe-size");
  size_t args = chainTableId.has_value() + chainTableVersion.has_value() + chainList.has_value() +
                chunkSize.has_value() + stripeSize.has_value();
  if (args == 0) {
    return std::nullopt;
  }
  if (base.has_value() && !chainList.has_value() && !chainTableId.has_value() && !chainTableVersion.has_value()) {
    base->chunkSize = chunkSize.value_or(base->chunkSize);
    base->stripeSize = stripeSize.value_or(base->stripeSize);
    return *base;
  }
  ENSURE_USAGE(chunkSize.has_value() && (stripeSize.has_value() || chainList.has_value()) &&
               (chainList.has_value() || chainTableId.has_value()));
  if (chainList.has_value()) {
    std::vector<std::string> idList;
    folly::split(',', *chainList, idList, true);
    ENSURE_USAGE(!idList.empty(), "chain-list is empty");
    ENSURE_USAGE(!stripeSize.has_value() || *stripeSize == idList.size(),
                 fmt::format("chain-list and stripe-size not match, len({}) == {}, stripe {}",
                             *chainList,
                             idList.size(),
                             *stripeSize));
    std::vector<uint32_t> chains;
    for (const auto &id : idList) {
      auto chain = folly::tryTo<uint32_t>(id);
      ENSURE_USAGE(chain.hasValue(), fmt::format("Failed to parse chain-list {}, element {}", *chainList, id));
      chains.push_back(*chain);
    }
    return meta::Layout::newChainList(flat::ChainTableId(chainTableId.value_or(0)),
                                      flat::ChainTableVersion(chainTableVersion.value_or(0)),
                                      *chunkSize,
                                      std::move(chains));
  } else {
    return meta::Layout::newEmpty(flat::ChainTableId(*chainTableId), *chunkSize, *stripeSize);
  }
}

}  // namespace hf3fs::client::cli

#pragma once

#include <folly/experimental/coro/Baton.h>

#include "chunk_engine/src/cxx.rs.h"
#include "fbs/storage/Common.h"
#include "storage/store/ChunkMetadata.h"
#include "storage/store/ChunkStore.h"

namespace hf3fs::storage {

class StorageTarget;

class ChunkEngineUpdateJob {
 public:
  ChunkEngineUpdateJob() = default;
  ChunkEngineUpdateJob(const ChunkEngineUpdateJob &) = delete;
  ChunkEngineUpdateJob(ChunkEngineUpdateJob &&other)
      : engine_(std::exchange(other.engine_, nullptr)),
        chunk_(std::exchange(other.chunk_, nullptr)) {}

  void set(chunk_engine::Engine &engine, chunk_engine::WritingChunk *chunk) {
    reset();
    engine_ = &engine;
    chunk_ = chunk;
  }

  auto release() { return std::exchange(engine_, nullptr); }
  auto chunk() const { return chunk_; }

  void reset() {
    if (engine_ && chunk_) {
      release()->release_writing_chunk(chunk_);
    }
  }

  ~ChunkEngineUpdateJob() { reset(); }

 private:
  chunk_engine::Engine *engine_{};
  chunk_engine::WritingChunk *chunk_{};
};

class UpdateJob {
 public:
  UpdateJob(ServiceRequestContext &requestCtx,
            const UpdateIO &updateIO,
            const UpdateOptions &options,
            ChunkEngineUpdateJob &chunkEngineJob,
            std::shared_ptr<StorageTarget> target,
            bool allowToAllocate = true)
      : requestCtx_(requestCtx),
        type_(updateIO.updateType),
        chunkId_(updateIO.key.chunkId),
        target_(std::move(target)),
        updateIO_(updateIO),
        chunkEngineJob_(chunkEngineJob),
        options_(options),
        allowToAllocate_(allowToAllocate) {}

  UpdateJob(ServiceRequestContext &requestCtx,
            const CommitIO &commitIO,
            const UpdateOptions &options,
            ChunkEngineUpdateJob &chunkEngineJob,
            std::shared_ptr<StorageTarget> target)
      : requestCtx_(requestCtx),
        type_(UpdateType::COMMIT),
        chunkId_(commitIO.key.chunkId),
        target_(std::move(target)),
        commitIO_(commitIO),
        chunkEngineJob_(chunkEngineJob),
        options_(options) {}

  auto &requestCtx() { return requestCtx_; }
  auto type() const { return type_; }
  const auto &chunkId() const { return chunkId_; }
  auto &target() const { return target_; }
  auto &updateIO() { return updateIO_; }
  auto &commitIO() { return commitIO_; }
  auto &chunkEngineJob() { return chunkEngineJob_; }
  auto &options() { return options_; }
  auto &result() { return result_; }
  auto &state() { return state_; }
  auto allowToAllocate() const { return allowToAllocate_; }
  ChainVer commitChainVer() const {
    if (options_.isSyncing) {
      return options_.commitChainVer;
    } else if (type() == UpdateType::COMMIT) {
      return commitIO_.commitChainVer;
    } else {
      return updateIO_.key.vChainId.chainVer;
    }
  }

  CoTask<void> complete() const { co_await baton_; }
  void setResult(Result<uint32_t> result) {
    result_.lengthInfo = std::move(result);
    baton_.post();
  }

 protected:
  ServiceRequestContext &requestCtx_;
  UpdateType type_;
  ChunkId chunkId_;
  std::shared_ptr<StorageTarget> target_;
  UpdateIO updateIO_;
  CommitIO commitIO_;
  ChunkEngineUpdateJob &chunkEngineJob_;
  UpdateOptions options_;
  IOResult result_;
  folly::coro::Baton baton_;
  struct State {
    const uint8_t *data = nullptr;
  } state_;
  bool allowToAllocate_ = true;
};

}  // namespace hf3fs::storage

#pragma once

#include <folly/Synchronized.h>
#include <unordered_map>

#include "chunk_engine/src/cxx.rs.h"
#include "common/monitor/Recorder.h"
#include "common/utils/CoLockManager.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/LockManager.h"
#include "common/utils/Path.h"
#include "storage/aio/BatchReadJob.h"
#include "storage/store/ChunkEngine.h"
#include "storage/store/ChunkStore.h"
#include "storage/store/PhysicalConfig.h"
#include "storage/update/UpdateJob.h"

namespace hf3fs::storage {

class StorageTarget : public enable_shared_from_this<StorageTarget> {
 protected:
  StorageTarget(const ChunkStore::Config &config,
                GlobalFileStore &globalFileStore,
                uint32_t diskIndex,
                chunk_engine::Engine *engine);

 public:
  using Config = ChunkStore::Config;

  ~StorageTarget();

  // create storage target.
  Result<Void> create(const PhysicalConfig &config);

  // load storage target.
  Result<Void> load(const Path &path);

  // add new chunk size.
  Result<Void> addChunkSize(const std::vector<Size> &sizeList);

  // get target id. [guaranteed loaded]
  TargetId targetId() const { return TargetId{targetConfig_.target_id}; }

  // get chain id. [guaranteed loaded]
  ChainId chainId() const { return ChainId{targetConfig_.chain_id}; }

  // set chain id.
  Result<Void> setChainId(ChainId chainId);

  // get disk index.
  uint32_t diskIndex() const { return diskIndex_; }

  // get target path. [guaranteed loaded]
  Path path() const { return targetConfig_.path; }

  // get all chunk metadata
  Result<Void> getAllMetadata(ChunkMetaVector &metadataVec);
  Result<Void> getAllMetadataMap(std::unordered_map<ChunkId, ChunkMetadata> &metas);

  // lock chunk.
  auto lockChunk(folly::coro::Baton &baton, const ChunkId &chunk, const std::string &tag) {
    return chunkLocks_.lock(baton, chunk.data(), tag);
  }

  // try lock channel.
  auto tryLockChannel(folly::coro::Baton &baton, const std::string &key) { return channelLocks_.tryLock(baton, key); }

  // prepare aio read.
  Result<Void> aioPrepareRead(AioReadJob &job);

  // finish aio read.
  Result<Void> aioFinishRead(AioReadJob &job);

  // update chunk (write/remove/truncate).
  void updateChunk(UpdateJob &job, folly::CPUThreadPoolExecutor &executor);

  // query chunks: the chunk ids in result are in reverse lexicographical order
  Result<std::vector<std::pair<ChunkId, ChunkMetadata>>> queryChunks(const ChunkIdRange &chunkIdRange);

  // query chunk.
  Result<ChunkMetadata> queryChunk(const ChunkId &chunkId);

  // recycle a batch of chunks. return true if all holes are punched.
  Result<bool> punchHole() {
    if (useChunkEngine()) {
      return true;
    } else {
      return chunkStore_.punchHole();
    }
  }

  // sync meta kv.
  Result<Void> sync() {
    if (useChunkEngine()) {
      return Void{};
    } else {
      return chunkStore_.sync();
    }
  }

  // report unrecycled size.
  Result<Void> reportUnrecycledSize();

  // get used size.
  uint64_t usedSize() const {
    if (useChunkEngine()) {
      return ChunkEngine::chainUsedSize(*engine_, ChainId{targetConfig_.chain_id});
    } else {
      return chunkStore_.usedSize();
    }
  }

  // get unused size.
  uint64_t unusedSize() const { return unusedSize_; }

  // get all uncommitted chunk ids.
  Result<std::vector<ChunkId>> uncommitted() {
    if (useChunkEngine()) {
      return ChunkEngine::queryUncommittedChunks(*engine_, chainId());
    } else {
      return chunkStore_.uncommitted();
    }
  }

  // reset uncommitted chunk to committed state.
  Result<Void> resetUncommitted(ChainVer chainVer) {
    if (useChunkEngine()) {
      return ChunkEngine::resetUncommittedChunks(*engine_, chainId(), chainVer);
    } else {
      return chunkStore_.resetUncommitted(chainVer);
    }
  }

  // enable or disable emergency recycling.
  void setEmergencyRecycling(bool enable) {
    if (useChunkEngine()) {
      return;
    } else {
      return chunkStore_.setEmergencyRecycling(enable);
    }
  }

  // record real read.
  void recordRealRead(uint32_t bytes, Duration latency) const;

  // disk monitor tag.
  auto &tag() const { return diskTag_; }

  // check alive or not.
  std::weak_ptr<bool> aliveWeakPtr() const { return alive_; }

  // global serial number.
  auto generationId() const { return generationId_; }

  // release self.
  Result<Void> release() {
    released_ = true;
    return sync();
  }

  // check if chunk engine is used.
  inline bool useChunkEngine() const { return targetConfig_.only_chunk_engine; }

 private:
  const Config &config_;
  std::shared_ptr<bool> alive_ = std::make_shared<bool>();
  uint32_t diskIndex_;
  uint32_t generationId_;
  chunk_engine::Engine *engine_{};
  std::atomic<uint64_t> unusedSize_{};
  monitor::TagSet diskTag_;
  monitor::TagSet targetTag_;
  monitor::Recorder::TagRef<monitor::CountRecorder> readCountPerDisk_;
  monitor::Recorder::TagRef<monitor::CountRecorder> readBytesPerDisk_;
  monitor::Recorder::TagRef<monitor::CountRecorder> readSuccBytesPerDisk_;
  monitor::Recorder::TagRef<monitor::LatencyRecorder> readSuccLatencyPerDisk_;
  monitor::Recorder::TagRef<monitor::ValueRecorder> targetUsedSize_;
  monitor::Recorder::TagRef<monitor::ValueRecorder> targetReservedSize_;
  monitor::Recorder::TagRef<monitor::ValueRecorder> targetUnrecycledSize_;
  PhysicalConfig targetConfig_;
  ChunkStore chunkStore_;
  CoLockManager<> chunkLocks_;
  CoLockManager<> channelLocks_;
  folly::Synchronized<std::set<Size>, std::mutex> chunkSizeList_;
  bool released_ = false;
};

}  // namespace hf3fs::storage

#pragma once

#include <folly/concurrency/ConcurrentHashMap.h>
#include <folly/executors/CPUThreadPoolExecutor.h>

#include "analytics/StructuredTraceLog.h"
#include "client/mgmtd/IMgmtdClientForServer.h"
#include "client/mgmtd/RoutingInfo.h"
#include "client/storage/StorageMessenger.h"
#include "common/net/Server.h"
#include "common/net/Transport.h"
#include "common/net/ib/IBSocket.h"
#include "common/net/ib/RDMABuf.h"
#include "common/utils/Address.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/Coroutine.h"
#include "common/utils/LockManager.h"
#include "common/utils/Semaphore.h"
#include "storage/aio/AioReadWorker.h"
#include "storage/service/BufferPool.h"
#include "storage/service/ReliableForwarding.h"
#include "storage/service/ReliableUpdate.h"
#include "storage/store/StorageTargets.h"
#include "storage/update/UpdateWorker.h"

namespace hf3fs::storage {

struct Components;

class StorageOperator {
 public:
  class Config : public ConfigBase<Config> {
    CONFIG_OBJ(write_worker, UpdateWorker::Config);
    CONFIG_OBJ(event_trace_log, analytics::StructuredTraceLog<StorageEventTrace>::Config);
    CONFIG_HOT_UPDATED_ITEM(max_num_results_per_query, uint32_t{100});
    CONFIG_HOT_UPDATED_ITEM(batch_read_job_split_size, uint32_t{1024});
    CONFIG_HOT_UPDATED_ITEM(post_buffer_per_bytes, 64_KB);
    CONFIG_HOT_UPDATED_ITEM(batch_read_ignore_chain_version, false);
    CONFIG_HOT_UPDATED_ITEM(max_concurrent_rdma_writes, 256U);
    CONFIG_HOT_UPDATED_ITEM(max_concurrent_rdma_reads, 256U);
    CONFIG_HOT_UPDATED_ITEM(read_only, false);
    CONFIG_HOT_UPDATED_ITEM(rdma_transmission_req_timeout, 0_ms);
    CONFIG_HOT_UPDATED_ITEM(apply_transmission_before_getting_semaphore, true);
  };

  StorageOperator(const Config &config, Components &components)
      : config_(config),
        components_(components),
        updateWorker_(config_.write_worker()),
        storageEventTrace_(config.event_trace_log()) {
    for (const auto &ibdev : net::IBDevice::all()) {
      concurrentRdmaWriteSemaphore_.emplace(ibdev->id(), config.max_concurrent_rdma_writes());
      concurrentRdmaReadSemaphore_.emplace(ibdev->id(), config.max_concurrent_rdma_reads());
    }

    onConfigUpdated_ = config_.addCallbackGuard([this]() {
      for (auto &[_, semaphore] : concurrentRdmaWriteSemaphore_) {
        semaphore.changeUsableTokens(config_.max_concurrent_rdma_writes());
      }
      for (auto &[_, semaphore] : concurrentRdmaReadSemaphore_) {
        semaphore.changeUsableTokens(config_.max_concurrent_rdma_reads());
      }
    });
  }

  Result<Void> init(uint32_t numberOfDisks);

  Result<Void> stopAndJoin();

  CoTryTask<BatchReadRsp> batchRead(ServiceRequestContext &requestCtx,
                                    const BatchReadReq &req,
                                    serde::CallContext &ctx);

  CoTryTask<WriteRsp> write(ServiceRequestContext &requestCtx, const WriteReq &req, net::IBSocket *ibSocket);

  CoTryTask<UpdateRsp> update(ServiceRequestContext &requestCtx, const UpdateReq &req, net::IBSocket *ibSocket);

  CoTryTask<QueryLastChunkRsp> queryLastChunk(ServiceRequestContext &requestCtx, const QueryLastChunkReq &req);

  CoTryTask<TruncateChunksRsp> truncateChunks(ServiceRequestContext &requestCtx, const TruncateChunksReq &req);

  CoTryTask<RemoveChunksRsp> removeChunks(ServiceRequestContext &requestCtx, const RemoveChunksReq &req);

  CoTryTask<TargetSyncInfo> syncStart(const SyncStartReq &req);

  CoTryTask<SyncDoneRsp> syncDone(const SyncDoneReq &req);

  CoTryTask<SpaceInfoRsp> spaceInfo(const SpaceInfoReq &req);

  CoTryTask<CreateTargetRsp> createTarget(const CreateTargetReq &req);

  CoTryTask<OfflineTargetRsp> offlineTarget(const OfflineTargetReq &req);

  CoTryTask<RemoveTargetRsp> removeTarget(const RemoveTargetReq &req);

  CoTryTask<QueryChunkRsp> queryChunk(const QueryChunkReq &req);

  CoTryTask<GetAllChunkMetadataRsp> getAllChunkMetadata(const GetAllChunkMetadataReq &req);

 protected:
  using ChunkMetadataProcessor = std::function<CoTryTask<void>(const ChunkId &, const ChunkMetadata &)>;

  CoTask<IOResult> handleUpdate(ServiceRequestContext &requestCtx,
                                UpdateReq &req,
                                net::IBSocket *ibSocket,
                                TargetPtr &target);

  CoTask<IOResult> doUpdate(ServiceRequestContext &requestCtx,
                            const UpdateIO &updateIO,
                            const UpdateOptions &updateOptions,
                            uint32_t featureFlags,
                            const std::shared_ptr<StorageTarget> &target,
                            net::IBSocket *ibSocket,
                            BufferPool::Buffer &buffer,
                            net::RDMARemoteBuf &remoteBuf,
                            ChunkEngineUpdateJob &chunkEngineJob,
                            bool allowToAllocate);

  CoTask<IOResult> doCommit(ServiceRequestContext &requestCtx,
                            const CommitIO &commitIO,
                            const UpdateOptions &updateOptions,
                            ChunkEngineUpdateJob &chunkEngineJob,
                            uint32_t featureFlags,
                            const std::shared_ptr<StorageTarget> &target);

  Result<std::vector<std::pair<ChunkId, ChunkMetadata>>> doQuery(ServiceRequestContext &requestCtx,
                                                                 const VersionedChainId &vChainId,
                                                                 const ChunkIdRange &chunkIdRange);

  CoTryTask<uint32_t> processQueryResults(ServiceRequestContext &requestCtx,
                                          const VersionedChainId &vChainId,
                                          const ChunkIdRange &chunkIdRanges,
                                          ChunkMetadataProcessor processor,
                                          bool &moreChunksInRange);

  CoTask<IOResult> doTruncate(ServiceRequestContext &requestCtx,
                              const TruncateChunkOp &op,
                              flat::UserInfo userInfo,
                              uint32_t featureFlags);

  CoTask<IOResult> doRemove(ServiceRequestContext &requestCtx,
                            const RemoveChunksOp &op,
                            flat::UserInfo userInfo,
                            uint32_t featureFlags);

 private:
  friend class ReliableUpdate;

  ConstructLog<"storage::StorageOperator"> constructLog_;
  const Config &config_;
  Components &components_;
  UpdateWorker updateWorker_;
  analytics::StructuredTraceLog<StorageEventTrace> storageEventTrace_;
  std::unique_ptr<ConfigCallbackGuard> onConfigUpdated_;
  std::map<uint8_t, hf3fs::Semaphore> concurrentRdmaWriteSemaphore_;
  std::map<uint8_t, hf3fs::Semaphore> concurrentRdmaReadSemaphore_;
  std::atomic<uint64_t> totalReadBytes_{};
  std::atomic<uint64_t> totalReadIOs_{};
};

}  // namespace hf3fs::storage

#pragma once

#include <folly/logging/xlog.h>
#include <optional>
#include <string>
#include <string_view>
#include <utility>

#include "common/kv/ITransaction.h"
#include "common/kv/KeyPrefix.h"
#include "common/serde/MessagePacket.h"
#include "common/serde/Serde.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Duration.h"
#include "common/utils/Nameof.hpp"
#include "common/utils/Result.h"
#include "common/utils/SerDeser.h"
#include "common/utils/UtcTime.h"
#include "common/utils/Uuid.h"
#include "fbs/meta/Service.h"

namespace hf3fs::meta::server {
/** Store transaction result to ensure idempotency during retries. Currently used for remove operations.
 */
struct Idempotent {
  static constexpr auto keyPrefix = kv::KeyPrefix::MetaIdempotent;

  template <typename T>
  struct Record {
    Record() requires(std::is_same_v<T, Void>) = default;
    explicit Record(const T &result)
        : result(result) {}

    SERDE_STRUCT_FIELD(clientId, Uuid::zero());
    SERDE_STRUCT_FIELD(requestId, Uuid::zero());
    SERDE_STRUCT_FIELD(timestamp, UtcTime());
    SERDE_STRUCT_FIELD(result, serde::Payload<T>());

   public:
    std::string packKey() const {
      // requestId + clientId to avoid hotspot
      XLOGF_IF(FATAL, clientId == Uuid::zero() || requestId == Uuid::zero(), "invalid uuid");
      return Serializer::serRawArgs(keyPrefix, requestId, clientId);
    }
  };

  template <class T, class ReqInfo>
  static CoTryTask<std::optional<Result<T>>> load(kv::IReadWriteTransaction &txn,
                                                  const Uuid clientId,
                                                  const Uuid requestId,
                                                  const ReqInfo &req) {
    if (clientId == Uuid::zero() || requestId == Uuid::zero()) {
      XLOGF(CRITICAL, "Request invalid uuid {} {}", clientId, requestId);
      co_return makeError(StatusCode::kInvalidArg, "Invalid uuid");
    }

    Record<Void> record;
    record.clientId = clientId;
    record.requestId = requestId;
    auto res = co_await txn.get(record.packKey());
    CO_RETURN_ON_ERROR(res);
    if (!res->has_value()) {
      co_return std::nullopt;
    }
    auto desRes = serde::deserialize(record, res->value());
    if (!desRes) {
      XLOGF(DFATAL, "IdempotentRecord deserialize failed, request {}, error {}", req, desRes.error());
      co_return makeError(StatusCode::kDataCorruption, "IdempotentRecord des failed");
    }
    if (record.clientId != clientId || record.requestId != requestId) {
      XLOGF(DFATAL, "IdempotentRecord mismatch, request {}, record {}", req, record);
      co_return makeError(MetaCode::kInconsistent, "IdempotentRecord uuid mismatch");
    }

    Result<T> result = makeError(StatusCode::kUnknown);
    auto desResult = serde::deserialize(result, record.result);
    if (!desResult) {
      XLOGF(DFATAL, "IdempotentRecord deserialize result failed, request {}, error {}", req, desResult.error());
      co_return makeError(StatusCode::kDataCorruption, "IdempotentRecord deserialize result failed");
    }
    XLOGF(CRITICAL, "Duplicated request {}, result {}, prev {}, now {}", req, result, record.timestamp, UtcTime::now());
    co_return std::optional(result);
  }

  template <class T>
  static CoTryTask<Void> store(kv::IReadWriteTransaction &txn,
                               const Uuid clientId,
                               const Uuid requestId,
                               const Result<T> &result) {
    Record<Result<T>> record(result);
    record.clientId = clientId;
    record.requestId = requestId;
    record.timestamp = UtcClock::now();

    auto key = record.packKey();
    auto value = serde::serialize(record);
    co_return co_await txn.set(key, value);
  }

  static CoTryTask<std::pair<std::string, bool>> clean(kv::IReadWriteTransaction &txn,
                                                       std::optional<std::string> prev,
                                                       Duration expire,
                                                       size_t limit,
                                                       size_t &total,
                                                       size_t &cleaned) {
    auto now = UtcClock::now();
    auto prefix = Serializer::serRawArgs(keyPrefix);
    auto begin = prev.value_or(prefix);
    XLOGF_IF(FATAL, begin < prefix, "{} < {}", begin, prefix);
    auto end = kv::TransactionHelper::prefixListEndKey(prefix);
    kv::IReadOnlyTransaction::KeySelector selBegin{begin, false};
    kv::IReadOnlyTransaction::KeySelector selEnd{end, false};
    auto res = co_await txn.getRange(selBegin, selEnd, limit);
    CO_RETURN_ON_ERROR(res);

    total = res->kvs.size();
    cleaned = 0;
    for (const auto &kv : res->kvs) {
      Record<Void> record;
      auto des = serde::deserialize(record, kv.value);
      if (!des) {
        XLOGF(CRITICAL, "IdempotentRecord deserialize failed {}", des.error());
        continue;
      }
      if (record.timestamp + expire < now) {
        cleaned++;
        CO_RETURN_ON_ERROR(co_await txn.clear(kv.key));
      }
    }

    auto nextPrev = res->kvs.empty() ? begin : res->kvs.back().key;
    co_return std::pair<std::string, bool>{nextPrev, res->hasMore};
  }
};

}  // namespace hf3fs::meta::server

// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
//
// Endian-neutral encoding:
// * Fixed-length numbers are encoded with least-significant byte first
// * In addition we support variable length "varint" encoding
// * Strings are encoded prefixed by their length in varint format

#pragma once

#include <cstdint>
#include <cstring>
#include <string>
#include <string_view>

namespace hf3fs {

// Standard Put... routines append to a string
void PutFixed32(std::string *dst, uint32_t value);
void PutFixed64(std::string *dst, uint64_t value);
void PutVarint32(std::string *dst, uint32_t value);
void PutVarint64(std::string *dst, uint64_t value);
void PutLengthPrefixedSlice(std::string *dst, std::string_view value);

// Standard Get... routines parse a value from the beginning of a Slice
// and advance the slice past the parsed value.
bool GetVarint32(std::string_view *input, uint32_t *value);
bool GetVarint64(std::string_view *input, uint64_t *value);
bool GetLengthPrefixedSlice(std::string_view *input, std::string_view *result);

// Pointer-based variants of GetVarint...  These either store a value
// in *v and return a pointer just past the parsed value, or return
// nullptr on error.  These routines only look at bytes in the range
// [p..limit-1]
const char *GetVarint32Ptr(const char *p, const char *limit, uint32_t *v);
const char *GetVarint64Ptr(const char *p, const char *limit, uint64_t *v);

// Returns the length of the varint32 or varint64 encoding of "v"
int VarintLength(uint64_t v);

// Lower-level versions of Put... that write directly into a character buffer
// and return a pointer just past the last byte written.
// REQUIRES: dst has enough space for the value being written
char *EncodeVarint32(char *dst, uint32_t value);
char *EncodeVarint64(char *dst, uint64_t value);

// Lower-level versions of Put... that write directly into a character buffer
// REQUIRES: dst has enough space for the value being written

inline void EncodeFixed32(char *dst, uint32_t value) {
  uint8_t *const buffer = reinterpret_cast<uint8_t *>(dst);

  // Recent clang and gcc optimize this to a single mov / str instruction.
  buffer[0] = static_cast<uint8_t>(value);
  buffer[1] = static_cast<uint8_t>(value >> 8);
  buffer[2] = static_cast<uint8_t>(value >> 16);
  buffer[3] = static_cast<uint8_t>(value >> 24);
}

inline void EncodeFixed64(char *dst, uint64_t value) {
  uint8_t *const buffer = reinterpret_cast<uint8_t *>(dst);

  // Recent clang and gcc optimize this to a single mov / str instruction.
  buffer[0] = static_cast<uint8_t>(value);
  buffer[1] = static_cast<uint8_t>(value >> 8);
  buffer[2] = static_cast<uint8_t>(value >> 16);
  buffer[3] = static_cast<uint8_t>(value >> 24);
  buffer[4] = static_cast<uint8_t>(value >> 32);
  buffer[5] = static_cast<uint8_t>(value >> 40);
  buffer[6] = static_cast<uint8_t>(value >> 48);
  buffer[7] = static_cast<uint8_t>(value >> 56);
}

// Lower-level versions of Get... that read directly from a character buffer
// without any bounds checking.

inline uint32_t DecodeFixed32(const char *ptr) {
  const uint8_t *const buffer = reinterpret_cast<const uint8_t *>(ptr);

  // Recent clang and gcc optimize this to a single mov / ldr instruction.
  return (static_cast<uint32_t>(buffer[0])) | (static_cast<uint32_t>(buffer[1]) << 8) |
         (static_cast<uint32_t>(buffer[2]) << 16) | (static_cast<uint32_t>(buffer[3]) << 24);
}

inline uint64_t DecodeFixed64(const char *ptr) {
  const uint8_t *const buffer = reinterpret_cast<const uint8_t *>(ptr);

  // Recent clang and gcc optimize this to a single mov / ldr instruction.
  return (static_cast<uint64_t>(buffer[0])) | (static_cast<uint64_t>(buffer[1]) << 8) |
         (static_cast<uint64_t>(buffer[2]) << 16) | (static_cast<uint64_t>(buffer[3]) << 24) |
         (static_cast<uint64_t>(buffer[4]) << 32) | (static_cast<uint64_t>(buffer[5]) << 40) |
         (static_cast<uint64_t>(buffer[6]) << 48) | (static_cast<uint64_t>(buffer[7]) << 56);
}

// Internal routine for use by fallback path of GetVarint32Ptr
const char *GetVarint32PtrFallback(const char *p, const char *limit, uint32_t *value);
inline const char *GetVarint32Ptr(const char *p, const char *limit, uint32_t *value) {
  if (p < limit) {
    uint32_t result = *(reinterpret_cast<const uint8_t *>(p));
    if ((result & 128) == 0) {
      *value = result;
      return p + 1;
    }
  }
  return GetVarint32PtrFallback(p, limit, value);
}

}  // namespace hf3fs

#pragma once

#include <algorithm>
#include <folly/Random.h>
#include <folly/experimental/coro/Invoke.h>
#include <folly/logging/xlog.h>
#include <memory>
#include <vector>

#include "common/app/AppInfo.h"
#include "common/net/ServiceGroup.h"
#include "common/net/ThreadPoolGroup.h"
#include "common/serde/ClientContext.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/Result.h"

namespace hf3fs::net {

class Server {
 public:
  static constexpr auto kName = "Server";

  struct Config : public ConfigBase<Config> {
    CONFIG_OBJ(thread_pool, ThreadPoolGroup::Config);
    CONFIG_OBJ(independent_thread_pool, ThreadPoolGroup::Config);
    CONFIG_OBJ_ARRAY(groups, ServiceGroup::Config, 4);
  };

  Server(const Config &config);
  virtual ~Server() { stopAndJoin(); }

  // get service groups.
  auto &groups() { return groups_; }
  auto &groups() const { return groups_; }

  auto serdeCtxCreator() {
    return [this](Address addr) { return serde::ClientContext(groups_.front()->ioWorker(), addr, options_); };
  }

  // add service into group.
  template <class Service>
  Result<Void> addSerdeService(std::unique_ptr<Service> &&obj, bool strict = false) {
    for (auto &group : groups_) {
      if (group->serviceNameList().contains(std::string{Service::kServiceName})) {
        return group->addSerdeService(std::move(obj));
      }
    }
    if (strict) {
      return makeError(RPCCode::kInvalidServiceName);
    }
    return groups_.front()->addSerdeService(std::move(obj));
  }

  // setup the server.
  Result<Void> setup();

  // start the server.
  Result<Void> start(const flat::AppInfo &info = {});

  // stop the server.
  void stopAndJoin();

  // set processor frozen.
  void setFrozen(bool frozen = true);

  // get the primary thread pool group.
  auto &tpg() { return tpg_; }

  // get the app info.
  const auto &appInfo() const { return appInfo_; }

  // describe the server.
  std::string describe() const;

  std::vector<flat::ServiceGroupInfo> getServiceGroupInfos() const;

 protected:
  // call it before start.
  virtual Result<Void> beforeStart() { return Void{}; }

  // call it after start.
  virtual Result<Void> afterStart() { return Void{}; }

  // call it before stop.
  virtual Result<Void> beforeStop() { return Void{}; }

  // call it after stop.
  virtual Result<Void> afterStop() { return Void{}; }

 private:
  ConstructLog<"net::Server"> constructLog_;
  const Config &config_;
  ThreadPoolGroup tpg_;
  ThreadPoolGroup independentTpg_;
  std::vector<std::unique_ptr<ServiceGroup>> groups_;
  flat::AppInfo appInfo_;
  std::atomic_flag stopped_;
  folly::atomic_shared_ptr<const CoreRequestOptions> options_{std::make_shared<CoreRequestOptions>()};
};

}  // namespace hf3fs::net

#pragma once

#include <algorithm>
#include <cassert>
#include <fmt/core.h>
#include <folly/Random.h>
#include <folly/Utility.h>
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <folly/logging/xlog.h>
#include <map>
#include <memory>
#include <unistd.h>

#include "client/mgmtd/ICommonMgmtdClient.h"
#include "client/storage/StorageClient.h"
#include "common/app/NodeId.h"
#include "common/kv/IKVEngine.h"
#include "common/kv/mem/MemKVEngine.h"
#include "common/serde/ClientMockContext.h"
#include "common/utils/CPUExecutorGroup.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/Coroutine.h"
#include "meta/components/ChainAllocator.h"
#include "meta/components/FileHelper.h"
#include "meta/components/GcManager.h"
#include "meta/service/MetaOperator.h"
#include "meta/service/MetaSerdeService.h"
#include "meta/store/MetaStore.h"

namespace hf3fs::meta::server {

class MockMeta : folly::NonCopyableNonMovable {
 public:
  static CoTryTask<std::unique_ptr<MockMeta>> create(const Config &cfg,
                                                     std::shared_ptr<kv::IKVEngine> kv,
                                                     std::shared_ptr<client::ICommonMgmtdClient> mgmtdClient) {
    auto meta = std::unique_ptr<MockMeta>(new MockMeta(cfg, kv, mgmtdClient));
    for (auto &moperator : meta->operators_) {
      CO_RETURN_ON_ERROR(co_await moperator->init(Layout::newEmpty(ChainTableId(1), 512 << 10, 128)));
    }
    co_return meta;
  }

  ~MockMeta() { stop(); }

  void start(CPUExecutorGroup &exec) {
    for (auto &moperator : operators_) {
      moperator->start(exec);
    }
  }

  void stop() {
    for (auto &moperator : operators_) {
      moperator->beforeStop();
      moperator->afterStop();
    }
  }

  std::unique_ptr<MetaSerdeService> getService() { return std::make_unique<MetaSerdeService>(*operators_.at(0)); }

  MetaOperator &getOperator() { return *operators_.at(0); }

  MetaStore &getStore() { return dynamic_cast<MetaStore &>(*getOperator().metaStore_); }

  storage::client::StorageClient &getStorageClient() { return *storageClient_; }

  FileHelper &getFileHelper() { return *getOperator().fileHelper_; }

  GcManager &getGcManager() { return *getOperator().gcManager_; }

  SessionManager &getSessionManager() { return *getOperator().sessionManager_; }

 private:
  MockMeta(const Config &cfg,
           std::shared_ptr<kv::IKVEngine> kv,
           std::shared_ptr<client::ICommonMgmtdClient> mgmtdClient)
      : cfg_(cfg),
        mgmtdClient_(mgmtdClient) {
    storageClientCfg_.set_implementation_type(storage::client::StorageClient::ImplementationType::InMem);
    storageClient_ = storage::client::StorageClient::create(ClientId::random(), storageClientCfg_, *mgmtdClient_);

    auto routing = mgmtdClient->getRoutingInfo();
    XLOGF_IF(FATAL, !routing, "routing info not available");
    auto nodes = routing->getNodeBy(flat::selectNodeByType(flat::NodeType::META) && flat::selectActiveNode());
    XLOGF_IF(FATAL, nodes.empty(), "no active metas");
    for (auto &node : nodes) {
      XLOGF_IF(FATAL, contexts_.contains(node.app.nodeId), "duplicated {}", node.app.nodeId);
      auto moperator = std::make_unique<MetaOperator>(
          cfg,
          node.app.nodeId,
          kv,
          mgmtdClient_,
          storageClient_,
          std::make_unique<Forward>(cfg.forward(), node.app.nodeId, contexts_, mgmtdClient_));
      contexts_[node.app.nodeId] = serde::ClientMockContext::create(std::make_unique<MetaSerdeService>(*moperator));
      operators_.push_back(std::move(moperator));
    }
  }

  [[maybe_unused]] const Config &cfg_;
  storage::client::StorageClient::Config storageClientCfg_;
  std::vector<std::unique_ptr<MetaOperator>> operators_;
  std::map<flat::NodeId, serde::ClientMockContext> contexts_;
  std::shared_ptr<client::ICommonMgmtdClient> mgmtdClient_;
  std::shared_ptr<storage::client::StorageClient> storageClient_;
};

}  // namespace hf3fs::meta::server

#include <algorithm>
#include <cmath>
#include <compare>
#include <folly/ScopeGuard.h>
#include <folly/lang/Ordering.h>
#include <folly/logging/xlog.h>
#include <linux/fs.h>
#include <memory>
#include <optional>
#include <type_traits>

#include "common/kv/ITransaction.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Result.h"
#include "common/utils/UtcTime.h"
#include "fbs/meta/Schema.h"
#include "fbs/meta/Service.h"
#include "meta/base/Config.h"
#include "meta/service/MetaOperator.h"
#include "meta/store/Inode.h"
#include "meta/store/Utils.h"

namespace hf3fs::meta::server {

class SetAttr {
 public:
  static Result<Void> check(const Inode &inode, const SetAttrReq &req, const Config &config) {
    RETURN_ON_ERROR(req.valid());

    // permission check for setPermission
    if (inode.id.isTreeRoot() && (req.perm || req.uid || req.gid)) {
      XLOGF(WARN, "Don't allow change permission of tree root {}!", inode.id);
      return makeError(MetaCode::kNoPermission, fmt::format("Don't allow change permission of {}", inode.id));
    }
    if (req.iflags.has_value() && *req.iflags != inode.acl.iflags) {
      auto setChainAllocation = !(inode.acl.iflags & FS_CHAIN_ALLOCATION_FL) && (*req.iflags & FS_CHAIN_ALLOCATION_FL);
      if (setChainAllocation && !config.iflags_chain_allocation()) {
        return makeError(MetaCode::kNoPermission, "FS_CHAIN_ALLOCATION_FL disabled");
      }
      auto setNewChunkEngine = !(inode.acl.iflags & FS_NEW_CHUNK_ENGINE) && (*req.iflags & FS_NEW_CHUNK_ENGINE);
      if (setNewChunkEngine && !config.iflags_chunk_engine()) {
        return makeError(MetaCode::kNoPermission, "FS_NEW_CHUNK_ENGINE disabled");
      }
      auto changed = *req.iflags ^ inode.acl.iflags;
      auto ownerChangeable = config.allow_owner_change_immutable() ? (uint32_t)(FS_HUGE_FILE_FL | FS_IMMUTABLE_FL)
                                                                   : (uint32_t)(FS_HUGE_FILE_FL);
      auto permCheck = req.user.isRoot() || (req.user.uid == inode.acl.uid && changed == (changed & ownerChangeable));
      if (!permCheck) {
        // NOTE: only allow root user set inode flags, file owner can use chattr +/- i, or set FS_HUGE_FILE_FL
        return makeError(MetaCode::kNoPermission, "only root can set iflags");
      }
    }
    if (req.perm.has_value() && *req.perm != inode.acl.perm && !req.user.isRoot() && req.user.uid != inode.acl.uid) {
      // man 2 chmod: The effective UID of the calling process must match the owner of the file, or the process must be
      // privileged (Linux: it must have the CAP_FOWNER capability).
      return makeError(MetaCode::kNoPermission, "no perm to set perm");
    }
    if (req.uid.has_value() && *req.uid != inode.acl.uid && !req.user.isRoot()) {
      // Only a privileged process (Linux: one with the CAP_CHOWN capability) may change the owner of a file.
      return makeError(MetaCode::kNoPermission, "no perm to set uid");
    }
    if (req.gid.has_value() && *req.gid != inode.acl.gid && !req.user.isRoot() &&
        (req.user.uid != inode.acl.uid || !req.user.inGroup(req.gid.value()))) {
      // The owner of a file may change the group of the file to any group of which that owner is a member.  A
      // privileged process (Linux: with CAP_CHOWN) may change the group arbitrarily.
      return makeError(MetaCode::kNoPermission, "no perm to set gid");
    }

    // permission check for utimes
    // To set both file timestamps to the current time (i.e., times is NULL, or both tv_nsec fields specify UTIME_NOW),
    // either:
    //   1. the caller must have write access to the file;
    //   2. the caller's effective user ID must match the owner of the file; or
    //   3. the caller must have appropriate privileges.
    // NOTE: we use UtcTime(0) as UTIME_NOW
    auto cond1 = inode.acl.checkPermission(req.user, AccessType::WRITE).hasValue();
    auto cond2 = req.user.uid == inode.acl.uid;
    auto cond3 = req.user.isRoot();
    if (req.atime || req.mtime) {
      if ((req.atime && req.atime != SETATTR_TIME_NOW) || (req.mtime && req.mtime != SETATTR_TIME_NOW)) {
        // To make any change other than setting both timestamps to the current time (i.e., times is not NULL, and
        // neither tv_nsec field is UTIME_NOW and neither tv_nsec field is UTIME_OMIT), either condition 2 or 3 above
        // must apply.
        if (!cond2 && !cond3) {
          return makeError(MetaCode::kNoPermission);
        }
      } else {
        if (!cond1 && !cond2 && !cond3) {
          return makeError(MetaCode::kNoPermission);
        }
      }
    }

    // permission check for setLayout
    if (req.layout) {
      if (!inode.isDirectory()) {
        return makeError(MetaCode::kNotDirectory, "setLayout but not directory");
      }
      RETURN_ON_ERROR(inode.acl.checkPermission(req.user, AccessType::WRITE));
    }

    if (!inode.isFile() && req.dynStripe) {
      return makeError(MetaCode::kNotFile, "extend dynStripe but not file");
    }

    return Void{};
  }

  static bool apply(Inode &inode, const SetAttrReq &req, Duration resolution, uint32_t stripeGrowth) {
    // now we can do update.
    bool dirty = false;

    // setPermission
    dirty |= update(inode.acl.iflags, req.iflags);
    dirty |= update(inode.acl.uid, req.uid);
    dirty |= update(inode.acl.gid, req.gid);
    dirty |= update(inode.acl.perm, req.perm);
    // setLayout
    if (req.layout.has_value()) {
      dirty |= update(inode.asDirectory().layout, req.layout);
    }
    if (dirty) {
      update(inode.ctime, SETATTR_TIME_NOW, resolution, true /* cmp */);
    }
    // utimes
    dirty |= update(inode.atime, req.atime, resolution, false /* cmp */);
    dirty |= update(inode.mtime, req.mtime, resolution, false /* cmp */);

    // extend
    if (req.dynStripe && inode.asFile().dynStripe && inode.asFile().dynStripe < req.dynStripe) {
      XLOGF_IF(FATAL, !inode.isFile(), "inode {} is not file", inode);

      auto growth = std::max(2u, stripeGrowth);
      auto dynStripe = inode.asFile().dynStripe;
      while (dynStripe < std::min(req.dynStripe, inode.asFile().layout.stripeSize)) {
        dynStripe = std::min(dynStripe * growth, inode.asFile().layout.stripeSize);
      }
      dirty |= update(inode.asFile().dynStripe, dynStripe);
    }

    return dirty;
  }

  static bool update(UtcTime &v, std::optional<UtcTime> nv, Duration resolution, bool cmp) {
    if (!nv) {
      return false;
    }
    if (*nv == SETATTR_TIME_NOW) {
      nv = UtcClock::now();
    }
    nv = nv->castGranularity(resolution);
    if (*nv != v && (!cmp || (*nv > v))) {
      v = *nv;
      return true;
    } else {
      return false;
    }
  }

  template <typename T>
  static bool update(T &v, std::optional<T> nv) {
    static_assert(!std::is_same_v<T, UtcTime>);
    if (nv.has_value() && nv != v) {
      v = *nv;
      return true;
    } else {
      return false;
    }
  }

  template <typename T>
  static bool update(T &v, T nv) {
    if (nv != v) {
      v = nv;
      return true;
    } else {
      return false;
    }
  }
};

}  // namespace hf3fs::meta::server

// This version targets C++11 and later.
//
// Copyright (C) 2016-2020 Martin Moene.
//
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
// expected lite is based on:
//   A proposal to add a utility class to represent expected monad
//   by Vicente J. Botet Escriba and Pierre Talbot. http:://wg21.link/p0323

#ifndef NONSTD_EXPECTED_LITE_HPP
#define NONSTD_EXPECTED_LITE_HPP

#define expected_lite_MAJOR 0
#define expected_lite_MINOR 6
#define expected_lite_PATCH 3

#define expected_lite_VERSION                                                                                \
  expected_STRINGIFY(expected_lite_MAJOR) "." expected_STRINGIFY(expected_lite_MINOR) "." expected_STRINGIFY( \
      expected_lite_PATCH)

#define expected_STRINGIFY(x) expected_STRINGIFY_(x)
#define expected_STRINGIFY_(x) #x

// expected-lite configuration:

#define nsel_EXPECTED_DEFAULT 0
#define nsel_EXPECTED_NONSTD 1
#define nsel_EXPECTED_STD 2

// tweak header support:

#ifdef __has_include
#if __has_include(<nonstd/expected.tweak.hpp>)
#include <nonstd/expected.tweak.hpp>
#endif
#define expected_HAVE_TWEAK_HEADER 1
#else
#define expected_HAVE_TWEAK_HEADER 0
//# pragma message("expected.hpp: Note: Tweak header not supported.")
#endif

// expected selection and configuration:

#if !defined(nsel_CONFIG_SELECT_EXPECTED)
#define nsel_CONFIG_SELECT_EXPECTED (nsel_HAVE_STD_EXPECTED ? nsel_EXPECTED_STD : nsel_EXPECTED_NONSTD)
#endif

// Proposal revisions:
//
// DXXXXR0: --
// N4015  : -2 (2014-05-26)
// N4109  : -1 (2014-06-29)
// P0323R0:  0 (2016-05-28)
// P0323R1:  1 (2016-10-12)
// -------:
// P0323R2:  2 (2017-06-15)
// P0323R3:  3 (2017-10-15)
// P0323R4:  4 (2017-11-26)
// P0323R5:  5 (2018-02-08)
// P0323R6:  6 (2018-04-02)
// P0323R7:  7 (2018-06-22) *
//
// expected-lite uses 2 and higher

#ifndef nsel_P0323R
#define nsel_P0323R 7
#endif

// Control presence of C++ exception handling (try and auto discover):

#ifndef nsel_CONFIG_NO_EXCEPTIONS
#if defined(_MSC_VER)
#include <cstddef>  // for _HAS_EXCEPTIONS
#endif
#if defined(__cpp_exceptions) || defined(__EXCEPTIONS) || (_HAS_EXCEPTIONS)
#define nsel_CONFIG_NO_EXCEPTIONS 0
#else
#define nsel_CONFIG_NO_EXCEPTIONS 1
#endif
#endif

// at default use SEH with MSVC for no C++ exceptions

#ifndef nsel_CONFIG_NO_EXCEPTIONS_SEH
#define nsel_CONFIG_NO_EXCEPTIONS_SEH (nsel_CONFIG_NO_EXCEPTIONS && _MSC_VER)
#endif

// C++ language version detection (C++23 is speculative):
// Note: VC14.0/1900 (VS2015) lacks too much from C++14.

#ifndef nsel_CPLUSPLUS
#if defined(_MSVC_LANG) && !defined(__clang__)
#define nsel_CPLUSPLUS (_MSC_VER == 1900 ? 201103L : _MSVC_LANG)
#else
#define nsel_CPLUSPLUS __cplusplus
#endif
#endif

#define nsel_CPP98_OR_GREATER (nsel_CPLUSPLUS >= 199711L)
#define nsel_CPP11_OR_GREATER (nsel_CPLUSPLUS >= 201103L)
#define nsel_CPP14_OR_GREATER (nsel_CPLUSPLUS >= 201402L)
#define nsel_CPP17_OR_GREATER (nsel_CPLUSPLUS >= 201703L)
#define nsel_CPP20_OR_GREATER (nsel_CPLUSPLUS >= 202002L)
#define nsel_CPP23_OR_GREATER (nsel_CPLUSPLUS >= 202300L)

// Use C++23 std::expected if available and requested:

#if nsel_CPP23_OR_GREATER && defined(__has_include)
#if __has_include(<expected> )
#define nsel_HAVE_STD_EXPECTED 1
#else
#define nsel_HAVE_STD_EXPECTED 0
#endif
#else
#define nsel_HAVE_STD_EXPECTED 0
#endif

#define nsel_USES_STD_EXPECTED                           \
  ((nsel_CONFIG_SELECT_EXPECTED == nsel_EXPECTED_STD) || \
   ((nsel_CONFIG_SELECT_EXPECTED == nsel_EXPECTED_DEFAULT) && nsel_HAVE_STD_EXPECTED))

//
// in_place: code duplicated in any-lite, expected-lite, expected-lite, value-ptr-lite, variant-lite:
//

#ifndef nonstd_lite_HAVE_IN_PLACE_TYPES
#define nonstd_lite_HAVE_IN_PLACE_TYPES 1

// C++17 std::in_place in <utility>:

#if nsel_CPP17_OR_GREATER

#include <utility>

namespace nonstd {

using std::in_place;
using std::in_place_index;
using std::in_place_index_t;
using std::in_place_t;
using std::in_place_type;
using std::in_place_type_t;

#define nonstd_lite_in_place_t(T) std::in_place_t
#define nonstd_lite_in_place_type_t(T) std::in_place_type_t<T>
#define nonstd_lite_in_place_index_t(K) std::in_place_index_t<K>

#define nonstd_lite_in_place(T) \
  std::in_place_t {}
#define nonstd_lite_in_place_type(T) \
  std::in_place_type_t<T> {}
#define nonstd_lite_in_place_index(K) \
  std::in_place_index_t<K> {}

}  // namespace nonstd

#else  // nsel_CPP17_OR_GREATER

#include <cstddef>

namespace nonstd {
namespace detail {

template <class T>
struct in_place_type_tag {};

template <std::size_t K>
struct in_place_index_tag {};

}  // namespace detail

struct in_place_t {};

template <class T>
inline in_place_t in_place(detail::in_place_type_tag<T> = detail::in_place_type_tag<T>()) {
  return in_place_t();
}

template <std::size_t K>
inline in_place_t in_place(detail::in_place_index_tag<K> = detail::in_place_index_tag<K>()) {
  return in_place_t();
}

template <class T>
inline in_place_t in_place_type(detail::in_place_type_tag<T> = detail::in_place_type_tag<T>()) {
  return in_place_t();
}

template <std::size_t K>
inline in_place_t in_place_index(detail::in_place_index_tag<K> = detail::in_place_index_tag<K>()) {
  return in_place_t();
}

// mimic templated typedef:

#define nonstd_lite_in_place_t(T) nonstd::in_place_t (&)(nonstd::detail::in_place_type_tag<T>)
#define nonstd_lite_in_place_type_t(T) nonstd::in_place_t (&)(nonstd::detail::in_place_type_tag<T>)
#define nonstd_lite_in_place_index_t(K) nonstd::in_place_t (&)(nonstd::detail::in_place_index_tag<K>)

#define nonstd_lite_in_place(T) nonstd::in_place_type<T>
#define nonstd_lite_in_place_type(T) nonstd::in_place_type<T>
#define nonstd_lite_in_place_index(K) nonstd::in_place_index<K>

}  // namespace nonstd

#endif  // nsel_CPP17_OR_GREATER
#endif  // nonstd_lite_HAVE_IN_PLACE_TYPES

//
// Using std::expected:
//

#if nsel_USES_STD_EXPECTED

#include <expected>

namespace nonstd {

using std::expected;
//  ...
}  // namespace nonstd

#else  // nsel_USES_STD_EXPECTED

#include <cassert>
#include <exception>
#include <functional>
#include <initializer_list>
#include <memory>
#include <new>
#include <system_error>
#include <type_traits>
#include <utility>

// additional includes:

#if nsel_CONFIG_NO_EXCEPTIONS
#if nsel_CONFIG_NO_EXCEPTIONS_SEH
#include <windows.h>  // for ExceptionCodes
#else
// already included: <cassert>
#endif
#else
#include <stdexcept>
#endif

// C++ feature usage:

#if nsel_CPP11_OR_GREATER
#define nsel_constexpr constexpr
#else
#define nsel_constexpr /*constexpr*/
#endif

#if nsel_CPP14_OR_GREATER
#define nsel_constexpr14 constexpr
#else
#define nsel_constexpr14 /*constexpr*/
#endif

#if nsel_CPP17_OR_GREATER
#define nsel_inline17 inline
#else
#define nsel_inline17 /*inline*/
#endif

// Compiler versions:
//
// MSVC++  6.0  _MSC_VER == 1200  nsel_COMPILER_MSVC_VERSION ==  60  (Visual Studio 6.0)
// MSVC++  7.0  _MSC_VER == 1300  nsel_COMPILER_MSVC_VERSION ==  70  (Visual Studio .NET 2002)
// MSVC++  7.1  _MSC_VER == 1310  nsel_COMPILER_MSVC_VERSION ==  71  (Visual Studio .NET 2003)
// MSVC++  8.0  _MSC_VER == 1400  nsel_COMPILER_MSVC_VERSION ==  80  (Visual Studio 2005)
// MSVC++  9.0  _MSC_VER == 1500  nsel_COMPILER_MSVC_VERSION ==  90  (Visual Studio 2008)
// MSVC++ 10.0  _MSC_VER == 1600  nsel_COMPILER_MSVC_VERSION == 100  (Visual Studio 2010)
// MSVC++ 11.0  _MSC_VER == 1700  nsel_COMPILER_MSVC_VERSION == 110  (Visual Studio 2012)
// MSVC++ 12.0  _MSC_VER == 1800  nsel_COMPILER_MSVC_VERSION == 120  (Visual Studio 2013)
// MSVC++ 14.0  _MSC_VER == 1900  nsel_COMPILER_MSVC_VERSION == 140  (Visual Studio 2015)
// MSVC++ 14.1  _MSC_VER >= 1910  nsel_COMPILER_MSVC_VERSION == 141  (Visual Studio 2017)
// MSVC++ 14.2  _MSC_VER >= 1920  nsel_COMPILER_MSVC_VERSION == 142  (Visual Studio 2019)

#if defined(_MSC_VER) && !defined(__clang__)
#define nsel_COMPILER_MSVC_VER (_MSC_VER)
#define nsel_COMPILER_MSVC_VERSION (_MSC_VER / 10 - 10 * (5 + (_MSC_VER < 1900)))
#else
#define nsel_COMPILER_MSVC_VER 0
#define nsel_COMPILER_MSVC_VERSION 0
#endif

#define nsel_COMPILER_VERSION(major, minor, patch) (10 * (10 * (major) + (minor)) + (patch))

#if defined(__clang__)
#define nsel_COMPILER_CLANG_VERSION nsel_COMPILER_VERSION(__clang_major__, __clang_minor__, __clang_patchlevel__)
#else
#define nsel_COMPILER_CLANG_VERSION 0
#endif

#if defined(__GNUC__) && !defined(__clang__)
#define nsel_COMPILER_GNUC_VERSION nsel_COMPILER_VERSION(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__)
#else
#define nsel_COMPILER_GNUC_VERSION 0
#endif

// half-open range [lo..hi):
//#define nsel_BETWEEN( v, lo, hi ) ( (lo) <= (v) && (v) < (hi) )

// Method enabling

#define nsel_REQUIRES_0(...) template <bool B = (__VA_ARGS__), typename std::enable_if<B, int>::type = 0>

#define nsel_REQUIRES_T(...) , typename std::enable_if<(__VA_ARGS__), int>::type = 0

#define nsel_REQUIRES_R(R, ...) typename std::enable_if<(__VA_ARGS__), R>::type

#define nsel_REQUIRES_A(...) , typename std::enable_if<(__VA_ARGS__), void *>::type = nullptr

// Presence of language and library features:

#ifdef _HAS_CPP0X
#define nsel_HAS_CPP0X _HAS_CPP0X
#else
#define nsel_HAS_CPP0X 0
#endif

//#define nsel_CPP11_140  (nsel_CPP11_OR_GREATER || nsel_COMPILER_MSVC_VER >= 1900)

// Clang, GNUC, MSVC warning suppression macros:

#ifdef __clang__
#pragma clang diagnostic push
#elif defined __GNUC__
#pragma GCC diagnostic push
#endif  // __clang__

#if nsel_COMPILER_MSVC_VERSION >= 140
#pragma warning(push)
#define nsel_DISABLE_MSVC_WARNINGS(codes) __pragma(warning(disable : codes))
#else
#define nsel_DISABLE_MSVC_WARNINGS(codes)
#endif

#ifdef __clang__
#define nsel_RESTORE_WARNINGS() _Pragma("clang diagnostic pop")
#elif defined __GNUC__
#define nsel_RESTORE_WARNINGS() _Pragma("GCC diagnostic pop")
#elif nsel_COMPILER_MSVC_VERSION >= 140
#define nsel_RESTORE_WARNINGS() __pragma(warning(pop))
#else
#define nsel_RESTORE_WARNINGS()
#endif

// Suppress the following MSVC (GSL) warnings:
// - C26409: Avoid calling new and delete explicitly, use std::make_unique<T> instead (r.11)

nsel_DISABLE_MSVC_WARNINGS(26409)

    //
    // expected:
    //

    namespace nonstd {
  namespace expected_lite {

  // type traits C++17:

  namespace std17 {

#if nsel_CPP17_OR_GREATER

  using std::conjunction;
  using std::is_nothrow_swappable;
  using std::is_swappable;

#else  // nsel_CPP17_OR_GREATER

  namespace detail {

  using std::swap;

  struct is_swappable {
    template <typename T, typename = decltype(swap(std::declval<T &>(), std::declval<T &>()))>
    static std::true_type test(int /* unused */);

    template <typename>
    static std::false_type test(...);
  };

  struct is_nothrow_swappable {
    // wrap noexcept(expr) in separate function as work-around for VC140 (VS2015):

    template <typename T>
    static constexpr bool satisfies() {
      return noexcept(swap(std::declval<T &>(), std::declval<T &>()));
    }

    template <typename T>
    static auto test(int) -> std::integral_constant<bool, satisfies<T>()> {}

    template <typename>
    static auto test(...) -> std::false_type;
  };
  }  // namespace detail

  // is [nothrow] swappable:

  template <typename T>
  struct is_swappable : decltype(detail::is_swappable::test<T>(0)) {};

  template <typename T>
  struct is_nothrow_swappable : decltype(detail::is_nothrow_swappable::test<T>(0)) {};

  // conjunction:

  template <typename...>
  struct conjunction : std::true_type {};
  template <typename B1>
  struct conjunction<B1> : B1 {};

  template <typename B1, typename... Bn>
  struct conjunction<B1, Bn...> : std::conditional<bool(B1::value), conjunction<Bn...>, B1>::type {};

#endif  // nsel_CPP17_OR_GREATER

  }  // namespace std17

  // type traits C++20:

  namespace std20 {

#if defined(__cpp_lib_remove_cvref)

  using std::remove_cvref;

#else

  template <typename T>
  struct remove_cvref {
    typedef typename std::remove_cv<typename std::remove_reference<T>::type>::type type;
  };

#endif

  }  // namespace std20

  // forward declaration:

  template <typename T, typename E>
  class expected;

  namespace detail {

  /// discriminated union to hold value or 'error'.

  template <typename T, typename E>
  class storage_t_impl {
    template <typename, typename>
    friend class nonstd::expected_lite::expected;

   public:
    using value_type = T;
    using error_type = E;

    // no-op construction
    storage_t_impl() {}
    ~storage_t_impl() {}

    explicit storage_t_impl(bool has_value)
        : m_has_value(has_value) {}

    void construct_value(value_type const &e) { new (&m_value) value_type(e); }

    void construct_value(value_type &&e) { new (&m_value) value_type(std::move(e)); }

    template <class... Args>
    void emplace_value(Args &&...args) {
      new (&m_value) value_type(std::forward<Args>(args)...);
    }

    template <class U, class... Args>
    void emplace_value(std::initializer_list<U> il, Args &&...args) {
      new (&m_value) value_type(il, std::forward<Args>(args)...);
    }

    void destruct_value() { m_value.~value_type(); }

    void construct_error(error_type const &e) { new (&m_error) error_type(e); }

    void construct_error(error_type &&e) { new (&m_error) error_type(std::move(e)); }

    template <class... Args>
    void emplace_error(Args &&...args) {
      new (&m_error) error_type(std::forward<Args>(args)...);
    }

    template <class U, class... Args>
    void emplace_error(std::initializer_list<U> il, Args &&...args) {
      new (&m_error) error_type(il, std::forward<Args>(args)...);
    }

    void destruct_error() { m_error.~error_type(); }

    constexpr value_type const &value() const & { return m_value; }

    value_type &value() & { return m_value; }

    constexpr value_type const &&value() const && { return std::move(m_value); }

    nsel_constexpr14 value_type &&value() && { return std::move(m_value); }

    value_type const *value_ptr() const { return &m_value; }

    value_type *value_ptr() { return &m_value; }

    error_type const &error() const & { return m_error; }

    error_type &error() & { return m_error; }

    constexpr error_type const &&error() const && { return std::move(m_error); }

    nsel_constexpr14 error_type &&error() && { return std::move(m_error); }

    bool has_value() const { return m_has_value; }

    void set_has_value(bool v) { m_has_value = v; }

   private:
    union {
      value_type m_value;
      error_type m_error;
    };

    bool m_has_value = false;
  };

  /// discriminated union to hold only 'error'.

  template <typename E>
  struct storage_t_impl<void, E> {
    template <typename, typename>
    friend class nonstd::expected_lite::expected;

   public:
    using value_type = void;
    using error_type = E;

    // no-op construction
    storage_t_impl() {}
    ~storage_t_impl() {}

    explicit storage_t_impl(bool has_value)
        : m_has_value(has_value) {}

    void construct_error(error_type const &e) { new (&m_error) error_type(e); }

    void construct_error(error_type &&e) { new (&m_error) error_type(std::move(e)); }

    template <class... Args>
    void emplace_error(Args &&...args) {
      new (&m_error) error_type(std::forward<Args>(args)...);
    }

    template <class U, class... Args>
    void emplace_error(std::initializer_list<U> il, Args &&...args) {
      new (&m_error) error_type(il, std::forward<Args>(args)...);
    }

    void destruct_error() { m_error.~error_type(); }

    error_type const &error() const & { return m_error; }

    error_type &error() & { return m_error; }

    constexpr error_type const &&error() const && { return std::move(m_error); }

    nsel_constexpr14 error_type &&error() && { return std::move(m_error); }

    bool has_value() const { return m_has_value; }

    void set_has_value(bool v) { m_has_value = v; }

   private:
    union {
      char m_dummy;
      error_type m_error;
    };

    bool m_has_value = false;
  };

  template <typename T, typename E, bool isConstructable, bool isMoveable>
  class storage_t {
   public:
    storage_t() = default;
    ~storage_t() = default;

    explicit storage_t(bool has_value)
        : storage_t_impl<T, E>(has_value) {}

    storage_t(storage_t const &other) = delete;
    storage_t(storage_t &&other) = delete;
  };

  template <typename T, typename E>
  class storage_t<T, E, true, true> : public storage_t_impl<T, E> {
   public:
    storage_t() = default;
    ~storage_t() = default;

    explicit storage_t(bool has_value)
        : storage_t_impl<T, E>(has_value) {}

    storage_t(storage_t const &other)
        : storage_t_impl<T, E>(other.has_value()) {
      if (this->has_value())
        this->construct_value(other.value());
      else
        this->construct_error(other.error());
    }

    storage_t(storage_t &&other)
        : storage_t_impl<T, E>(other.has_value()) {
      if (this->has_value())
        this->construct_value(std::move(other.value()));
      else
        this->construct_error(std::move(other.error()));
    }
  };

  template <typename E>
  class storage_t<void, E, true, true> : public storage_t_impl<void, E> {
   public:
    storage_t() = default;
    ~storage_t() = default;

    explicit storage_t(bool has_value)
        : storage_t_impl<void, E>(has_value) {}

    storage_t(storage_t const &other)
        : storage_t_impl<void, E>(other.has_value()) {
      if (this->has_value())
        ;
      else
        this->construct_error(other.error());
    }

    storage_t(storage_t &&other)
        : storage_t_impl<void, E>(other.has_value()) {
      if (this->has_value())
        ;
      else
        this->construct_error(std::move(other.error()));
    }
  };

  template <typename T, typename E>
  class storage_t<T, E, true, false> : public storage_t_impl<T, E> {
   public:
    storage_t() = default;
    ~storage_t() = default;

    explicit storage_t(bool has_value)
        : storage_t_impl<T, E>(has_value) {}

    storage_t(storage_t const &other)
        : storage_t_impl<T, E>(other.has_value()) {
      if (this->has_value())
        this->construct_value(other.value());
      else
        this->construct_error(other.error());
    }

    storage_t(storage_t &&other) = delete;
  };

  template <typename E>
  class storage_t<void, E, true, false> : public storage_t_impl<void, E> {
   public:
    storage_t() = default;
    ~storage_t() = default;

    explicit storage_t(bool has_value)
        : storage_t_impl<void, E>(has_value) {}

    storage_t(storage_t const &other)
        : storage_t_impl<void, E>(other.has_value()) {
      if (this->has_value())
        ;
      else
        this->construct_error(other.error());
    }

    storage_t(storage_t &&other) = delete;
  };

  template <typename T, typename E>
  class storage_t<T, E, false, true> : public storage_t_impl<T, E> {
   public:
    storage_t() = default;
    ~storage_t() = default;

    explicit storage_t(bool has_value)
        : storage_t_impl<T, E>(has_value) {}

    storage_t(storage_t const &other) = delete;

    storage_t(storage_t &&other)
        : storage_t_impl<T, E>(other.has_value()) {
      if (this->has_value())
        this->construct_value(std::move(other.value()));
      else
        this->construct_error(std::move(other.error()));
    }
  };

  template <typename E>
  class storage_t<void, E, false, true> : public storage_t_impl<void, E> {
   public:
    storage_t() = default;
    ~storage_t() = default;

    explicit storage_t(bool has_value)
        : storage_t_impl<void, E>(has_value) {}

    storage_t(storage_t const &other) = delete;

    storage_t(storage_t &&other)
        : storage_t_impl<void, E>(other.has_value()) {
      if (this->has_value())
        ;
      else
        this->construct_error(std::move(other.error()));
    }
  };

  }  // namespace detail

  /// x.x.5 Unexpected object type; unexpected_type; C++17 and later can also use aliased type unexpected.

#if nsel_P0323R <= 2
  template <typename E = std::exception_ptr>
  class unexpected_type
#else
  template <typename E>
  class unexpected_type
#endif  // nsel_P0323R
  {
   public:
    using error_type = E;

    // x.x.5.2.1 Constructors

    //  unexpected_type() = delete;

    constexpr unexpected_type(unexpected_type const &) = default;
    constexpr unexpected_type(unexpected_type &&) = default;

    template <typename... Args nsel_REQUIRES_T(std::is_constructible<E, Args &&...>::value)>
    constexpr explicit unexpected_type(nonstd_lite_in_place_t(E), Args &&...args)
        : m_error(std::forward<Args>(args)...) {}

    template <typename U,
              typename... Args nsel_REQUIRES_T(std::is_constructible<E, std::initializer_list<U>, Args &&...>::value)>
    constexpr explicit unexpected_type(nonstd_lite_in_place_t(E), std::initializer_list<U> il, Args &&...args)
        : m_error(il, std::forward<Args>(args)...) {}

    template <typename E2 nsel_REQUIRES_T(
        std::is_constructible<E, E2>::value &&
        !std::is_same<typename std20::remove_cvref<E2>::type, nonstd_lite_in_place_t(E2)>::value &&
        !std::is_same<typename std20::remove_cvref<E2>::type, unexpected_type>::value)>
    constexpr explicit unexpected_type(E2 &&error)
        : m_error(std::forward<E2>(error)) {}

    template <typename E2 nsel_REQUIRES_T(
        std::is_constructible<E, E2>::value && !std::is_constructible<E, unexpected_type<E2> &>::value &&
        !std::is_constructible<E, unexpected_type<E2>>::value &&
        !std::is_constructible<E, unexpected_type<E2> const &>::value &&
        !std::is_constructible<E, unexpected_type<E2> const>::value &&
        !std::is_convertible<unexpected_type<E2> &, E>::value && !std::is_convertible<unexpected_type<E2>, E>::value &&
        !std::is_convertible<unexpected_type<E2> const &, E>::value &&
        !std::is_convertible<unexpected_type<E2> const, E>::value &&
        !std::is_convertible<E2 const &, E>::value /*=> explicit */
        )>
    constexpr explicit unexpected_type(unexpected_type<E2> const &error)
        : m_error(E{error.value()}) {}

    template <typename E2 nsel_REQUIRES_T(
        std::is_constructible<E, E2>::value && !std::is_constructible<E, unexpected_type<E2> &>::value &&
        !std::is_constructible<E, unexpected_type<E2>>::value &&
        !std::is_constructible<E, unexpected_type<E2> const &>::value &&
        !std::is_constructible<E, unexpected_type<E2> const>::value &&
        !std::is_convertible<unexpected_type<E2> &, E>::value && !std::is_convertible<unexpected_type<E2>, E>::value &&
        !std::is_convertible<unexpected_type<E2> const &, E>::value &&
        !std::is_convertible<unexpected_type<E2> const, E>::value &&
        std::is_convertible<E2 const &, E>::value /*=> explicit */
        )>
    constexpr /*non-explicit*/ unexpected_type(unexpected_type<E2> const &error)
        : m_error(error.value()) {}

    template <typename E2 nsel_REQUIRES_T(
        std::is_constructible<E, E2>::value && !std::is_constructible<E, unexpected_type<E2> &>::value &&
        !std::is_constructible<E, unexpected_type<E2>>::value &&
        !std::is_constructible<E, unexpected_type<E2> const &>::value &&
        !std::is_constructible<E, unexpected_type<E2> const>::value &&
        !std::is_convertible<unexpected_type<E2> &, E>::value && !std::is_convertible<unexpected_type<E2>, E>::value &&
        !std::is_convertible<unexpected_type<E2> const &, E>::value &&
        !std::is_convertible<unexpected_type<E2> const, E>::value &&
        !std::is_convertible<E2 const &, E>::value /*=> explicit */
        )>
    constexpr explicit unexpected_type(unexpected_type<E2> &&error)
        : m_error(E{std::move(error.value())}) {}

    template <typename E2 nsel_REQUIRES_T(
        std::is_constructible<E, E2>::value && !std::is_constructible<E, unexpected_type<E2> &>::value &&
        !std::is_constructible<E, unexpected_type<E2>>::value &&
        !std::is_constructible<E, unexpected_type<E2> const &>::value &&
        !std::is_constructible<E, unexpected_type<E2> const>::value &&
        !std::is_convertible<unexpected_type<E2> &, E>::value && !std::is_convertible<unexpected_type<E2>, E>::value &&
        !std::is_convertible<unexpected_type<E2> const &, E>::value &&
        !std::is_convertible<unexpected_type<E2> const, E>::value &&
        std::is_convertible<E2 const &, E>::value /*=> non-explicit */
        )>
    constexpr /*non-explicit*/ unexpected_type(unexpected_type<E2> &&error)
        : m_error(std::move(error.value())) {}

    // x.x.5.2.2 Assignment

    nsel_constexpr14 unexpected_type &operator=(unexpected_type const &) = default;
    nsel_constexpr14 unexpected_type &operator=(unexpected_type &&) = default;

    template <typename E2 = E>
    nsel_constexpr14 unexpected_type &operator=(unexpected_type<E2> const &other) {
      unexpected_type{other.value()}.swap(*this);
      return *this;
    }

    template <typename E2 = E>
    nsel_constexpr14 unexpected_type &operator=(unexpected_type<E2> &&other) {
      unexpected_type{std::move(other.value())}.swap(*this);
      return *this;
    }

    // x.x.5.2.3 Observers

    nsel_constexpr14 E &value() &noexcept { return m_error; }

    constexpr E const &value() const &noexcept { return m_error; }

#if !nsel_COMPILER_GNUC_VERSION || nsel_COMPILER_GNUC_VERSION >= 490

    nsel_constexpr14 E &&value() &&noexcept { return std::move(m_error); }

    constexpr E const &&value() const &&noexcept { return std::move(m_error); }

#endif

    // x.x.5.2.4 Swap

    template <typename U = E>
    nsel_REQUIRES_R(void, std17::is_swappable<U>::value)
        swap(unexpected_type &other) noexcept(std17::is_nothrow_swappable<U>::value) {
      using std::swap;
      swap(m_error, other.m_error);
    }

    // TODO: ??? unexpected_type: in-class friend operator==, !=

   private:
    error_type m_error;
  };

#if nsel_CPP17_OR_GREATER

  /// template deduction guide:

  template <typename E>
  unexpected_type(E) -> unexpected_type<E>;

#endif

  /// class unexpected_type, std::exception_ptr specialization (P0323R2)

#if !nsel_CONFIG_NO_EXCEPTIONS
#if nsel_P0323R <= 2

  // TODO: Should expected be specialized for particular E types such as exception_ptr and how?
  //       See p0323r7 2.1. Ergonomics, http://wg21.link/p0323
  template <>
  class unexpected_type<std::exception_ptr> {
   public:
    using error_type = std::exception_ptr;

    unexpected_type() = delete;

    ~unexpected_type() {}

    explicit unexpected_type(std::exception_ptr const &error)
        : m_error(error) {}

    explicit unexpected_type(std::exception_ptr &&error)
        : m_error(std::move(error)) {}

    template <typename E>
    explicit unexpected_type(E error)
        : m_error(std::make_exception_ptr(error)) {}

    std::exception_ptr const &value() const { return m_error; }

    std::exception_ptr &value() { return m_error; }

   private:
    std::exception_ptr m_error;
  };

#endif  // nsel_P0323R
#endif  // !nsel_CONFIG_NO_EXCEPTIONS

  /// x.x.4, Unexpected equality operators

  template <typename E1, typename E2>
  constexpr bool operator==(unexpected_type<E1> const &x, unexpected_type<E2> const &y) {
    return x.value() == y.value();
  }

  template <typename E1, typename E2>
  constexpr bool operator!=(unexpected_type<E1> const &x, unexpected_type<E2> const &y) {
    return !(x == y);
  }

#if nsel_P0323R <= 2

  template <typename E>
  constexpr bool operator<(unexpected_type<E> const &x, unexpected_type<E> const &y) {
    return x.value() < y.value();
  }

  template <typename E>
  constexpr bool operator>(unexpected_type<E> const &x, unexpected_type<E> const &y) {
    return (y < x);
  }

  template <typename E>
  constexpr bool operator<=(unexpected_type<E> const &x, unexpected_type<E> const &y) {
    return !(y < x);
  }

  template <typename E>
  constexpr bool operator>=(unexpected_type<E> const &x, unexpected_type<E> const &y) {
    return !(x < y);
  }

#endif  // nsel_P0323R

  /// x.x.5 Specialized algorithms

  template <typename E nsel_REQUIRES_T(std17::is_swappable<E>::value)>
  void swap(unexpected_type<E> &x, unexpected_type<E> &y) noexcept(noexcept(x.swap(y))) {
    x.swap(y);
  }

#if nsel_P0323R <= 2

  // unexpected: relational operators for std::exception_ptr:

  inline constexpr bool operator<(unexpected_type<std::exception_ptr> const & /*x*/,
                                  unexpected_type<std::exception_ptr> const & /*y*/) {
    return false;
  }

  inline constexpr bool operator>(unexpected_type<std::exception_ptr> const & /*x*/,
                                  unexpected_type<std::exception_ptr> const & /*y*/) {
    return false;
  }

  inline constexpr bool operator<=(unexpected_type<std::exception_ptr> const &x,
                                   unexpected_type<std::exception_ptr> const &y) {
    return (x == y);
  }

  inline constexpr bool operator>=(unexpected_type<std::exception_ptr> const &x,
                                   unexpected_type<std::exception_ptr> const &y) {
    return (x == y);
  }

#endif  // nsel_P0323R

  // unexpected: traits

#if nsel_P0323R <= 3

  template <typename E>
  struct is_unexpected : std::false_type {};

  template <typename E>
  struct is_unexpected<unexpected_type<E>> : std::true_type {};

#endif  // nsel_P0323R

  // unexpected: factory

  // keep make_unexpected() removed in p0323r2 for pre-C++17:

  template <typename E>
  nsel_constexpr14 auto make_unexpected(E &&value) -> unexpected_type<typename std::decay<E>::type> {
    return unexpected_type<typename std::decay<E>::type>(std::forward<E>(value));
  }

#if nsel_P0323R <= 3

  /*nsel_constexpr14*/ auto inline make_unexpected_from_current_exception() -> unexpected_type<std::exception_ptr> {
    return unexpected_type<std::exception_ptr>(std::current_exception());
  }

#endif  // nsel_P0323R

  /// x.x.6, x.x.7 expected access error

  template <typename E>
  class bad_expected_access;

  /// x.x.7 bad_expected_access<void>: expected access error

  template <>
  class bad_expected_access<void> : public std::exception {
   public:
    explicit bad_expected_access()
        : std::exception() {}
  };

  /// x.x.6 bad_expected_access: expected access error

#if !nsel_CONFIG_NO_EXCEPTIONS

  template <typename E>
  class bad_expected_access : public bad_expected_access<void> {
   public:
    using error_type = E;

    explicit bad_expected_access(error_type error)
        : m_error(error) {}

    virtual char const *what() const noexcept override { return "bad_expected_access"; }

    nsel_constexpr14 error_type &error() & { return m_error; }

    constexpr error_type const &error() const & { return m_error; }

#if !nsel_COMPILER_GNUC_VERSION || nsel_COMPILER_GNUC_VERSION >= 490

    nsel_constexpr14 error_type &&error() && { return std::move(m_error); }

    constexpr error_type const &&error() const && { return std::move(m_error); }

#endif

   private:
    error_type m_error;
  };

#endif  // nsel_CONFIG_NO_EXCEPTIONS

  /// x.x.8 unexpect tag, in_place_unexpected tag: construct an error

  struct unexpect_t {};
  using in_place_unexpected_t = unexpect_t;

  nsel_inline17 constexpr unexpect_t unexpect{};
  nsel_inline17 constexpr unexpect_t in_place_unexpected{};

  /// class error_traits

#if nsel_CONFIG_NO_EXCEPTIONS

  namespace detail {
  inline bool text(char const * /*text*/) { return true; }
  }  // namespace detail

  template <typename Error>
  struct error_traits {
    static void rethrow(Error const & /*e*/) {
#if nsel_CONFIG_NO_EXCEPTIONS_SEH
      RaiseException(EXCEPTION_ACCESS_VIOLATION, EXCEPTION_NONCONTINUABLE, 0, NULL);
#else
      assert(false && detail::text("throw bad_expected_access<Error>{ e };"));
#endif
    }
  };

  template <>
  struct error_traits<std::exception_ptr> {
    static void rethrow(std::exception_ptr const & /*e*/) {
#if nsel_CONFIG_NO_EXCEPTIONS_SEH
      RaiseException(EXCEPTION_ACCESS_VIOLATION, EXCEPTION_NONCONTINUABLE, 0, NULL);
#else
      assert(false && detail::text("throw bad_expected_access<std::exception_ptr>{ e };"));
#endif
    }
  };

  template <>
  struct error_traits<std::error_code> {
    static void rethrow(std::error_code const & /*e*/) {
#if nsel_CONFIG_NO_EXCEPTIONS_SEH
      RaiseException(EXCEPTION_ACCESS_VIOLATION, EXCEPTION_NONCONTINUABLE, 0, NULL);
#else
      assert(false && detail::text("throw std::system_error( e );"));
#endif
    }
  };

#else  // nsel_CONFIG_NO_EXCEPTIONS

  template <typename Error>
  struct error_traits {
    static void rethrow(Error const &e) { throw bad_expected_access<Error>{e}; }
  };

  template <>
  struct error_traits<std::exception_ptr> {
    static void rethrow(std::exception_ptr const &e) { std::rethrow_exception(e); }
  };

  template <>
  struct error_traits<std::error_code> {
    static void rethrow(std::error_code const &e) { throw std::system_error(e); }
  };

#endif  // nsel_CONFIG_NO_EXCEPTIONS

  }  // namespace expected_lite

  // provide nonstd::unexpected_type:

  using expected_lite::unexpected_type;

  namespace expected_lite {

  /// class expected

#if nsel_P0323R <= 2
  template <typename T, typename E = std::exception_ptr>
  class expected
#else
  template <typename T, typename E>
  class expected
#endif  // nsel_P0323R
  {
   private:
    template <typename, typename>
    friend class expected;

   public:
    using value_type = T;
    using error_type = E;
    using unexpected_type = nonstd::unexpected_type<E>;

    template <typename U>
    struct rebind {
      using type = expected<U, error_type>;
    };

    // x.x.4.1 constructors

    nsel_REQUIRES_0(std::is_default_constructible<T>::value) nsel_constexpr14 expected()
        : contained(true) {
      contained.construct_value(value_type());
    }

    nsel_constexpr14 expected(expected const &) = default;
    nsel_constexpr14 expected(expected &&) = default;

    template <
        typename U,
        typename G nsel_REQUIRES_T(
            std::is_constructible<T, U const &>::value &&std::is_constructible<E, G const &>::value &&
            !std::is_constructible<T, expected<U, G> &>::value && !std::is_constructible<T, expected<U, G> &&>::value &&
            !std::is_constructible<T, expected<U, G> const &>::value &&
            !std::is_constructible<T, expected<U, G> const &&>::value &&
            !std::is_convertible<expected<U, G> &, T>::value && !std::is_convertible<expected<U, G> &&, T>::value &&
            !std::is_convertible<expected<U, G> const &, T>::value &&
            !std::is_convertible<expected<U, G> const &&, T>::value &&
            (!std::is_convertible<U const &, T>::value || !std::is_convertible<G const &, E>::value) /*=> explicit */
            )>
    nsel_constexpr14 explicit expected(expected<U, G> const &other)
        : contained(other.has_value()) {
      if (has_value())
        contained.construct_value(T{other.contained.value()});
      else
        contained.construct_error(E{other.contained.error()});
    }

    template <
        typename U,
        typename G nsel_REQUIRES_T(
            std::is_constructible<T, U const &>::value &&std::is_constructible<E, G const &>::value &&
            !std::is_constructible<T, expected<U, G> &>::value && !std::is_constructible<T, expected<U, G> &&>::value &&
            !std::is_constructible<T, expected<U, G> const &>::value &&
            !std::is_constructible<T, expected<U, G> const &&>::value &&
            !std::is_convertible<expected<U, G> &, T>::value && !std::is_convertible<expected<U, G> &&, T>::value &&
            !std::is_convertible<expected<U, G> const &, T>::value &&
            !std::is_convertible<expected<U, G> const &&, T>::value &&
            !(!std::is_convertible<U const &, T>::value || !std::is_convertible<G const &, E>::value) /*=> non-explicit
                                                                                */
            )>
    nsel_constexpr14 /*non-explicit*/ expected(expected<U, G> const &other)
        : contained(other.has_value()) {
      if (has_value())
        contained.construct_value(other.contained.value());
      else
        contained.construct_error(other.contained.error());
    }

    template <
        typename U,
        typename G nsel_REQUIRES_T(
            std::is_constructible<T, U>::value &&std::is_constructible<E, G>::value &&
            !std::is_constructible<T, expected<U, G> &>::value && !std::is_constructible<T, expected<U, G> &&>::value &&
            !std::is_constructible<T, expected<U, G> const &>::value &&
            !std::is_constructible<T, expected<U, G> const &&>::value &&
            !std::is_convertible<expected<U, G> &, T>::value && !std::is_convertible<expected<U, G> &&, T>::value &&
            !std::is_convertible<expected<U, G> const &, T>::value &&
            !std::is_convertible<expected<U, G> const &&, T>::value &&
            (!std::is_convertible<U, T>::value || !std::is_convertible<G, E>::value) /*=> explicit */
            )>
    nsel_constexpr14 explicit expected(expected<U, G> &&other)
        : contained(other.has_value()) {
      if (has_value())
        contained.construct_value(T{std::move(other.contained.value())});
      else
        contained.construct_error(E{std::move(other.contained.error())});
    }

    template <
        typename U,
        typename G nsel_REQUIRES_T(
            std::is_constructible<T, U>::value &&std::is_constructible<E, G>::value &&
            !std::is_constructible<T, expected<U, G> &>::value && !std::is_constructible<T, expected<U, G> &&>::value &&
            !std::is_constructible<T, expected<U, G> const &>::value &&
            !std::is_constructible<T, expected<U, G> const &&>::value &&
            !std::is_convertible<expected<U, G> &, T>::value && !std::is_convertible<expected<U, G> &&, T>::value &&
            !std::is_convertible<expected<U, G> const &, T>::value &&
            !std::is_convertible<expected<U, G> const &&, T>::value &&
            !(!std::is_convertible<U, T>::value || !std::is_convertible<G, E>::value) /*=> non-explicit */
            )>
    nsel_constexpr14 /*non-explicit*/ expected(expected<U, G> &&other)
        : contained(other.has_value()) {
      if (has_value())
        contained.construct_value(std::move(other.contained.value()));
      else
        contained.construct_error(std::move(other.contained.error()));
    }

    template <typename U = T nsel_REQUIRES_T(std::is_copy_constructible<U>::value)>
    nsel_constexpr14 expected(value_type const &value)
        : contained(true) {
      contained.construct_value(value);
    }

    template <typename U = T nsel_REQUIRES_T(
                  std::is_constructible<T, U &&>::value &&
                  !std::is_same<typename std20::remove_cvref<U>::type, nonstd_lite_in_place_t(U)>::value &&
                  !std::is_same<expected<T, E>, typename std20::remove_cvref<U>::type>::value &&
                  !std::is_same<nonstd::unexpected_type<E>, typename std20::remove_cvref<U>::type>::value &&
                  !std::is_convertible<U &&, T>::value /*=> explicit */
                  )>
    nsel_constexpr14 explicit expected(U &&value) noexcept(
        std::is_nothrow_move_constructible<U>::value &&std::is_nothrow_move_constructible<E>::value)
        : contained(true) {
      contained.construct_value(T{std::forward<U>(value)});
    }

    template <typename U = T nsel_REQUIRES_T(
                  std::is_constructible<T, U &&>::value &&
                  !std::is_same<typename std20::remove_cvref<U>::type, nonstd_lite_in_place_t(U)>::value &&
                  !std::is_same<expected<T, E>, typename std20::remove_cvref<U>::type>::value &&
                  !std::is_same<nonstd::unexpected_type<E>, typename std20::remove_cvref<U>::type>::value &&
                  std::is_convertible<U &&, T>::value /*=> non-explicit */
                  )>
    nsel_constexpr14 /*non-explicit*/ expected(U &&value) noexcept(
        std::is_nothrow_move_constructible<U>::value &&std::is_nothrow_move_constructible<E>::value)
        : contained(true) {
      contained.construct_value(std::forward<U>(value));
    }

    // construct error:

    template <typename G = E nsel_REQUIRES_T(std::is_constructible<E, G const &>::value &&
                                             !std::is_convertible<G const &, E>::value /*=> explicit */
                                             )>
    nsel_constexpr14 explicit expected(nonstd::unexpected_type<G> const &error)
        : contained(false) {
      contained.construct_error(E{error.value()});
    }

    template <typename G = E nsel_REQUIRES_T(std::is_constructible<E, G const &>::value
                                                 &&std::is_convertible<G const &, E>::value /*=> non-explicit */
                                             )>
    nsel_constexpr14 /*non-explicit*/ expected(nonstd::unexpected_type<G> const &error)
        : contained(false) {
      contained.construct_error(error.value());
    }

    template <typename G = E nsel_REQUIRES_T(std::is_constructible<E, G &&>::value &&
                                             !std::is_convertible<G &&, E>::value /*=> explicit */
                                             )>
    nsel_constexpr14 explicit expected(nonstd::unexpected_type<G> &&error)
        : contained(false) {
      contained.construct_error(E{std::move(error.value())});
    }

    template <typename G = E nsel_REQUIRES_T(
                  std::is_constructible<E, G &&>::value &&std::is_convertible<G &&, E>::value /*=> non-explicit */
                  )>
    nsel_constexpr14 /*non-explicit*/ expected(nonstd::unexpected_type<G> &&error)
        : contained(false) {
      contained.construct_error(std::move(error.value()));
    }

    // in-place construction, value

    template <typename... Args nsel_REQUIRES_T(std::is_constructible<T, Args &&...>::value)>
    nsel_constexpr14 explicit expected(nonstd_lite_in_place_t(T), Args &&...args)
        : contained(true) {
      contained.emplace_value(std::forward<Args>(args)...);
    }

    template <typename U,
              typename... Args nsel_REQUIRES_T(std::is_constructible<T, std::initializer_list<U>, Args &&...>::value)>
    nsel_constexpr14 explicit expected(nonstd_lite_in_place_t(T), std::initializer_list<U> il, Args &&...args)
        : contained(true) {
      contained.emplace_value(il, std::forward<Args>(args)...);
    }

    // in-place construction, error

    template <typename... Args nsel_REQUIRES_T(std::is_constructible<E, Args &&...>::value)>
    nsel_constexpr14 explicit expected(unexpect_t, Args &&...args)
        : contained(false) {
      contained.emplace_error(std::forward<Args>(args)...);
    }

    template <typename U,
              typename... Args nsel_REQUIRES_T(std::is_constructible<E, std::initializer_list<U>, Args &&...>::value)>
    nsel_constexpr14 explicit expected(unexpect_t, std::initializer_list<U> il, Args &&...args)
        : contained(false) {
      contained.emplace_error(il, std::forward<Args>(args)...);
    }

    // x.x.4.2 destructor

    // TODO: ~expected: triviality
    // Effects: If T is not cv void and is_trivially_destructible_v<T> is false and bool(*this), calls val.~T(). If
    // is_trivially_destructible_v<E> is false and !bool(*this), calls unexpect.~unexpected<E>(). Remarks: If either T
    // is cv void or is_trivially_destructible_v<T> is true, and is_trivially_destructible_v<E> is true, then this
    // destructor shall be a trivial destructor.

    ~expected() {
      if (has_value())
        contained.destruct_value();
      else
        contained.destruct_error();
    }

    // x.x.4.3 assignment

    expected &operator=(expected const &other) {
      expected(other).swap(*this);
      return *this;
    }

    expected &operator=(expected &&other) noexcept(
        std::is_nothrow_move_constructible<T>::value &&std::is_nothrow_move_assignable<T>::value
            &&std::is_nothrow_move_constructible<E>::value    // added for missing
                &&std::is_nothrow_move_assignable<E>::value)  //   nothrow above
    {
      expected(std::move(other)).swap(*this);
      return *this;
    }

    template <typename U nsel_REQUIRES_T(!std::is_same<expected<T, E>, typename std20::remove_cvref<U>::type>::value &&
                                         std17::conjunction<std::is_scalar<T>, std::is_same<T, std::decay<U>>>::value &&
                                         std::is_constructible<T, U>::value && std::is_assignable<T &, U>::value &&
                                         std::is_nothrow_move_constructible<E>::value)>
    expected &operator=(U &&value) {
      expected(std::forward<U>(value)).swap(*this);
      return *this;
    }

    template <typename G = E nsel_REQUIRES_T(std::is_constructible<E, G const &>::value &&std::is_copy_constructible<
                                             G>::value  // TODO: std::is_nothrow_copy_constructible<G>
                                                 &&std::is_copy_assignable<G>::value)>
    expected &operator=(nonstd::unexpected_type<G> const &error) {
      expected(unexpect, error.value()).swap(*this);
      return *this;
    }

    template <typename G = E nsel_REQUIRES_T(std::is_constructible<E, G &&>::value &&std::is_move_constructible<
                                             G>::value  // TODO: std::is_nothrow_move_constructible<G>
                                                 &&std::is_move_assignable<G>::value)>
    expected &operator=(nonstd::unexpected_type<G> &&error) {
      expected(unexpect, std::move(error.value())).swap(*this);
      return *this;
    }

    template <typename... Args nsel_REQUIRES_T(std::is_nothrow_constructible<T, Args &&...>::value)>
    value_type &emplace(Args &&...args) {
      expected(nonstd_lite_in_place(T), std::forward<Args>(args)...).swap(*this);
      return value();
    }

    template <typename U,
              typename... Args nsel_REQUIRES_T(
                  std::is_nothrow_constructible<T, std::initializer_list<U> &, Args &&...>::value)>
    value_type &emplace(std::initializer_list<U> il, Args &&...args) {
      expected(nonstd_lite_in_place(T), il, std::forward<Args>(args)...).swap(*this);
      return value();
    }

    // x.x.4.4 swap

    template <typename U = T, typename G = E>
    nsel_REQUIRES_R(void,
                    std17::is_swappable<U>::value &&std17::is_swappable<G>::value &&
                        (std::is_move_constructible<U>::value || std::is_move_constructible<G>::value))
        swap(expected &other) noexcept(
            std::is_nothrow_move_constructible<T>::value &&std17::is_nothrow_swappable<T &>::value
                &&std::is_nothrow_move_constructible<E>::value &&std17::is_nothrow_swappable<E &>::value) {
      using std::swap;

      if (bool(*this) && bool(other)) {
        swap(contained.value(), other.contained.value());
      } else if (!bool(*this) && !bool(other)) {
        swap(contained.error(), other.contained.error());
      } else if (bool(*this) && !bool(other)) {
        error_type t(std::move(other.error()));
        other.contained.destruct_error();
        other.contained.construct_value(std::move(contained.value()));
        contained.destruct_value();
        contained.construct_error(std::move(t));
        bool has_value = contained.has_value();
        bool other_has_value = other.has_value();
        other.contained.set_has_value(has_value);
        contained.set_has_value(other_has_value);
      } else if (!bool(*this) && bool(other)) {
        other.swap(*this);
      }
    }

    // x.x.4.5 observers

    constexpr value_type const *operator->() const { return assert(has_value()), contained.value_ptr(); }

    value_type *operator->() { return assert(has_value()), contained.value_ptr(); }

    constexpr value_type const &operator*() const & { return assert(has_value()), contained.value(); }

    value_type &operator*() & { return assert(has_value()), contained.value(); }

#if !nsel_COMPILER_GNUC_VERSION || nsel_COMPILER_GNUC_VERSION >= 490

    constexpr value_type const &&operator*() const && { return std::move((assert(has_value()), contained.value())); }

    nsel_constexpr14 value_type &&operator*() && { return std::move((assert(has_value()), contained.value())); }

#endif

    constexpr explicit operator bool() const noexcept { return has_value(); }

    constexpr bool has_value() const noexcept { return contained.has_value(); }

    constexpr value_type const &value() const & {
      return has_value() ? (contained.value())
                         : (error_traits<error_type>::rethrow(contained.error()), contained.value());
    }

    value_type &value() & {
      return has_value() ? (contained.value())
                         : (error_traits<error_type>::rethrow(contained.error()), contained.value());
    }

#if !nsel_COMPILER_GNUC_VERSION || nsel_COMPILER_GNUC_VERSION >= 490

    constexpr value_type const &&value() const && {
      return std::move(has_value() ? (contained.value())
                                   : (error_traits<error_type>::rethrow(contained.error()), contained.value()));
    }

    nsel_constexpr14 value_type &&value() && {
      return std::move(has_value() ? (contained.value())
                                   : (error_traits<error_type>::rethrow(contained.error()), contained.value()));
    }

#endif

    constexpr error_type const &error() const & { return assert(!has_value()), contained.error(); }

    error_type &error() & { return assert(!has_value()), contained.error(); }

#if !nsel_COMPILER_GNUC_VERSION || nsel_COMPILER_GNUC_VERSION >= 490

    constexpr error_type const &&error() const && { return std::move((assert(!has_value()), contained.error())); }

    error_type &&error() && { return std::move((assert(!has_value()), contained.error())); }

#endif

    constexpr unexpected_type get_unexpected() const { return make_unexpected(contained.error()); }

    template <typename Ex>
    bool has_exception() const {
      using ContainedEx = typename std::remove_reference<decltype(get_unexpected().value())>::type;
      return !has_value() && std::is_base_of<Ex, ContainedEx>::value;
    }

    template <typename U nsel_REQUIRES_T(std::is_copy_constructible<T>::value &&std::is_convertible<U &&, T>::value)>
    value_type value_or(U &&v) const & {
      return has_value() ? contained.value() : static_cast<T>(std::forward<U>(v));
    }

    template <typename U nsel_REQUIRES_T(std::is_move_constructible<T>::value &&std::is_convertible<U &&, T>::value)>
    value_type value_or(U &&v) && {
      return has_value() ? std::move(contained.value()) : static_cast<T>(std::forward<U>(v));
    }

    // unwrap()

    //  template <class U, class E>
    //  constexpr expected<U,E> expected<expected<U,E>,E>::unwrap() const&;

    //  template <class T, class E>
    //  constexpr expected<T,E> expected<T,E>::unwrap() const&;

    //  template <class U, class E>
    //  expected<U,E> expected<expected<U,E>, E>::unwrap() &&;

    //  template <class T, class E>
    //  template expected<T,E> expected<T,E>::unwrap() &&;

    // factories

    //  template< typename Ex, typename F>
    //  expected<T,E> catch_exception(F&& f);

    //  template< typename F>
    //  expected<decltype(func(declval<T>())),E> map(F&& func) ;

    //  template< typename F>
    //  'see below' bind(F&& func);

    //  template< typename F>
    //  expected<T,E> catch_error(F&& f);

    //  template< typename F>
    //  'see below' then(F&& func);

   private:
    detail::storage_t<T,
                      E,
                      std::is_copy_constructible<T>::value && std::is_copy_constructible<E>::value,
                      std::is_move_constructible<T>::value && std::is_move_constructible<E>::value>
        contained;
  };

  /// class expected, void specialization

  template <typename E>
  class expected<void, E> {
   private:
    template <typename, typename>
    friend class expected;

   public:
    using value_type = void;
    using error_type = E;
    using unexpected_type = nonstd::unexpected_type<E>;

    // x.x.4.1 constructors

    constexpr expected() noexcept
        : contained(true) {}

    nsel_constexpr14 expected(expected const &other) = default;
    nsel_constexpr14 expected(expected &&other) = default;

    constexpr explicit expected(nonstd_lite_in_place_t(void))
        : contained(true) {}

    template <typename G = E nsel_REQUIRES_T(!std::is_convertible<G const &, E>::value /*=> explicit */
                                             )>
    nsel_constexpr14 explicit expected(nonstd::unexpected_type<G> const &error)
        : contained(false) {
      contained.construct_error(E{error.value()});
    }

    template <typename G = E nsel_REQUIRES_T(std::is_convertible<G const &, E>::value /*=> non-explicit */
                                             )>
    nsel_constexpr14 /*non-explicit*/ expected(nonstd::unexpected_type<G> const &error)
        : contained(false) {
      contained.construct_error(error.value());
    }

    template <typename G = E nsel_REQUIRES_T(!std::is_convertible<G &&, E>::value /*=> explicit */
                                             )>
    nsel_constexpr14 explicit expected(nonstd::unexpected_type<G> &&error)
        : contained(false) {
      contained.construct_error(E{std::move(error.value())});
    }

    template <typename G = E nsel_REQUIRES_T(std::is_convertible<G &&, E>::value /*=> non-explicit */
                                             )>
    nsel_constexpr14 /*non-explicit*/ expected(nonstd::unexpected_type<G> &&error)
        : contained(false) {
      contained.construct_error(std::move(error.value()));
    }

    template <typename... Args nsel_REQUIRES_T(std::is_constructible<E, Args &&...>::value)>
    nsel_constexpr14 explicit expected(unexpect_t, Args &&...args)
        : contained(false) {
      contained.emplace_error(std::forward<Args>(args)...);
    }

    template <typename U,
              typename... Args nsel_REQUIRES_T(std::is_constructible<E, std::initializer_list<U>, Args &&...>::value)>
    nsel_constexpr14 explicit expected(unexpect_t, std::initializer_list<U> il, Args &&...args)
        : contained(false) {
      contained.emplace_error(il, std::forward<Args>(args)...);
    }

    // destructor

    ~expected() {
      if (!has_value()) {
        contained.destruct_error();
      }
    }

    // x.x.4.3 assignment

    expected &operator=(expected const &other) {
      expected(other).swap(*this);
      return *this;
    }

    expected &operator=(expected &&other) noexcept(
        std::is_nothrow_move_assignable<E>::value &&std::is_nothrow_move_constructible<E>::value) {
      expected(std::move(other)).swap(*this);
      return *this;
    }

    void emplace() { expected().swap(*this); }

    // x.x.4.4 swap

    template <typename G = E>
    nsel_REQUIRES_R(void, std17::is_swappable<G>::value &&std::is_move_constructible<G>::value)
        swap(expected &other) noexcept(
            std::is_nothrow_move_constructible<E>::value &&std17::is_nothrow_swappable<E &>::value) {
      using std::swap;

      if (!bool(*this) && !bool(other)) {
        swap(contained.error(), other.contained.error());
      } else if (bool(*this) && !bool(other)) {
        contained.construct_error(std::move(other.error()));
        bool has_value = contained.has_value();
        bool other_has_value = other.has_value();
        other.contained.set_has_value(has_value);
        contained.set_has_value(other_has_value);
      } else if (!bool(*this) && bool(other)) {
        other.swap(*this);
      }
    }

    // x.x.4.5 observers

    constexpr explicit operator bool() const noexcept { return has_value(); }

    constexpr bool has_value() const noexcept { return contained.has_value(); }

    void value() const {
      if (!has_value()) {
        error_traits<error_type>::rethrow(contained.error());
      }
    }

    constexpr error_type const &error() const & { return assert(!has_value()), contained.error(); }

    error_type &error() & { return assert(!has_value()), contained.error(); }

#if !nsel_COMPILER_GNUC_VERSION || nsel_COMPILER_GNUC_VERSION >= 490

    constexpr error_type const &&error() const && { return std::move((assert(!has_value()), contained.error())); }

    error_type &&error() && { return std::move((assert(!has_value()), contained.error())); }

#endif

    constexpr unexpected_type get_unexpected() const { return make_unexpected(contained.error()); }

    template <typename Ex>
    bool has_exception() const {
      using ContainedEx = typename std::remove_reference<decltype(get_unexpected().value())>::type;
      return !has_value() && std::is_base_of<Ex, ContainedEx>::value;
    }

    //  template constexpr 'see below' unwrap() const&;
    //
    //  template 'see below' unwrap() &&;

    // factories

    //  template< typename Ex, typename F>
    //  expected<void,E> catch_exception(F&& f);
    //
    //  template< typename F>
    //  expected<decltype(func()), E> map(F&& func) ;
    //
    //  template< typename F>
    //  'see below' bind(F&& func) ;
    //
    //  template< typename F>
    //  expected<void,E> catch_error(F&& f);
    //
    //  template< typename F>
    //  'see below' then(F&& func);

   private:
    detail::storage_t<void, E, std::is_copy_constructible<E>::value, std::is_move_constructible<E>::value> contained;
  };

  // x.x.4.6 expected<>: comparison operators

  template <typename T1,
            typename E1,
            typename T2,
            typename E2 nsel_REQUIRES_T(!std::is_void<T1>::value && !std::is_void<T2>::value)>
  constexpr bool operator==(expected<T1, E1> const &x, expected<T2, E2> const &y) {
    return bool(x) != bool(y) ? false : bool(x) ? *x == *y : x.error() == y.error();
  }

  template <typename T1,
            typename E1,
            typename T2,
            typename E2 nsel_REQUIRES_T(std::is_void<T1>::value &&std::is_void<T2>::value)>
  constexpr bool operator==(expected<T1, E1> const &x, expected<T2, E2> const &y) {
    return bool(x) != bool(y) ? false : bool(x) || static_cast<bool>(x.error() == y.error());
  }

  template <typename T1, typename E1, typename T2, typename E2>
  constexpr bool operator!=(expected<T1, E1> const &x, expected<T2, E2> const &y) {
    return !(x == y);
  }

#if nsel_P0323R <= 2

  template <typename T, typename E>
  constexpr bool operator<(expected<T, E> const &x, expected<T, E> const &y) {
    return (!y) ? false : (!x) ? true : *x < *y;
  }

  template <typename T, typename E>
  constexpr bool operator>(expected<T, E> const &x, expected<T, E> const &y) {
    return (y < x);
  }

  template <typename T, typename E>
  constexpr bool operator<=(expected<T, E> const &x, expected<T, E> const &y) {
    return !(y < x);
  }

  template <typename T, typename E>
  constexpr bool operator>=(expected<T, E> const &x, expected<T, E> const &y) {
    return !(x < y);
  }

#endif

  // x.x.4.7 expected: comparison with T

  template <typename T1, typename E1, typename T2 nsel_REQUIRES_T(!std::is_void<T1>::value)>
  constexpr bool operator==(expected<T1, E1> const &x, T2 const &v) {
    return bool(x) ? *x == v : false;
  }

  template <typename T1, typename E1, typename T2 nsel_REQUIRES_T(!std::is_void<T1>::value)>
  constexpr bool operator==(T2 const &v, expected<T1, E1> const &x) {
    return bool(x) ? v == *x : false;
  }

  template <typename T1, typename E1, typename T2>
  constexpr bool operator!=(expected<T1, E1> const &x, T2 const &v) {
    return bool(x) ? *x != v : true;
  }

  template <typename T1, typename E1, typename T2>
  constexpr bool operator!=(T2 const &v, expected<T1, E1> const &x) {
    return bool(x) ? v != *x : true;
  }

#if nsel_P0323R <= 2

  template <typename T, typename E>
  constexpr bool operator<(expected<T, E> const &x, T const &v) {
    return bool(x) ? *x < v : true;
  }

  template <typename T, typename E>
  constexpr bool operator<(T const &v, expected<T, E> const &x) {
    return bool(x) ? v < *x : false;
  }

  template <typename T, typename E>
  constexpr bool operator>(T const &v, expected<T, E> const &x) {
    return bool(x) ? *x < v : false;
  }

  template <typename T, typename E>
  constexpr bool operator>(expected<T, E> const &x, T const &v) {
    return bool(x) ? v < *x : false;
  }

  template <typename T, typename E>
  constexpr bool operator<=(T const &v, expected<T, E> const &x) {
    return bool(x) ? !(*x < v) : false;
  }

  template <typename T, typename E>
  constexpr bool operator<=(expected<T, E> const &x, T const &v) {
    return bool(x) ? !(v < *x) : true;
  }

  template <typename T, typename E>
  constexpr bool operator>=(expected<T, E> const &x, T const &v) {
    return bool(x) ? !(*x < v) : false;
  }

  template <typename T, typename E>
  constexpr bool operator>=(T const &v, expected<T, E> const &x) {
    return bool(x) ? !(v < *x) : true;
  }

#endif  // nsel_P0323R

  // x.x.4.8 expected: comparison with unexpected_type

  template <typename T1, typename E1, typename E2>
  constexpr bool operator==(expected<T1, E1> const &x, unexpected_type<E2> const &u) {
    return (!x) ? x.get_unexpected() == u : false;
  }

  template <typename T1, typename E1, typename E2>
  constexpr bool operator==(unexpected_type<E2> const &u, expected<T1, E1> const &x) {
    return (x == u);
  }

  template <typename T1, typename E1, typename E2>
  constexpr bool operator!=(expected<T1, E1> const &x, unexpected_type<E2> const &u) {
    return !(x == u);
  }

  template <typename T1, typename E1, typename E2>
  constexpr bool operator!=(unexpected_type<E2> const &u, expected<T1, E1> const &x) {
    return !(x == u);
  }

#if nsel_P0323R <= 2

  template <typename T, typename E>
  constexpr bool operator<(expected<T, E> const &x, unexpected_type<E> const &u) {
    return (!x) ? (x.get_unexpected() < u) : false;
  }

  template <typename T, typename E>
  constexpr bool operator<(unexpected_type<E> const &u, expected<T, E> const &x) {
    return (!x) ? (u < x.get_unexpected()) : true;
  }

  template <typename T, typename E>
  constexpr bool operator>(expected<T, E> const &x, unexpected_type<E> const &u) {
    return (u < x);
  }

  template <typename T, typename E>
  constexpr bool operator>(unexpected_type<E> const &u, expected<T, E> const &x) {
    return (x < u);
  }

  template <typename T, typename E>
  constexpr bool operator<=(expected<T, E> const &x, unexpected_type<E> const &u) {
    return !(u < x);
  }

  template <typename T, typename E>
  constexpr bool operator<=(unexpected_type<E> const &u, expected<T, E> const &x) {
    return !(x < u);
  }

  template <typename T, typename E>
  constexpr bool operator>=(expected<T, E> const &x, unexpected_type<E> const &u) {
    return !(u > x);
  }

  template <typename T, typename E>
  constexpr bool operator>=(unexpected_type<E> const &u, expected<T, E> const &x) {
    return !(x > u);
  }

#endif  // nsel_P0323R

  /// x.x.x Specialized algorithms

  template <typename T,
            typename E nsel_REQUIRES_T((std::is_void<T>::value || std::is_move_constructible<T>::value) &&
                                       std::is_move_constructible<E>::value && std17::is_swappable<T>::value &&
                                       std17::is_swappable<E>::value)>
  void swap(expected<T, E> &x, expected<T, E> &y) noexcept(noexcept(x.swap(y))) {
    x.swap(y);
  }

#if nsel_P0323R <= 3

  template <typename T>
  constexpr auto make_expected(T &&v) -> expected<typename std::decay<T>::type> {
    return expected<typename std::decay<T>::type>(std::forward<T>(v));
  }

  // expected<void> specialization:

  auto inline make_expected() -> expected<void> { return expected<void>(in_place); }

  template <typename T>
  constexpr auto make_expected_from_current_exception() -> expected<T> {
    return expected<T>(make_unexpected_from_current_exception());
  }

  template <typename T>
  auto make_expected_from_exception(std::exception_ptr v) -> expected<T> {
    return expected<T>(unexpected_type<std::exception_ptr>(std::forward<std::exception_ptr>(v)));
  }

  template <typename T, typename E>
  constexpr auto make_expected_from_error(E e) -> expected<T, typename std::decay<E>::type> {
    return expected<T, typename std::decay<E>::type>(make_unexpected(e));
  }

  template <typename F nsel_REQUIRES_T(!std::is_same<typename std::result_of<F()>::type, void>::value)>
  /*nsel_constexpr14*/
  auto make_expected_from_call(F f) -> expected<typename std::result_of<F()>::type> {
    try {
      return make_expected(f());
    } catch (...) {
      return make_unexpected_from_current_exception();
    }
  }

  template <typename F nsel_REQUIRES_T(std::is_same<typename std::result_of<F()>::type, void>::value)>
  /*nsel_constexpr14*/
  auto make_expected_from_call(F f) -> expected<void> {
    try {
      f();
      return make_expected();
    } catch (...) {
      return make_unexpected_from_current_exception();
    }
  }

#endif  // nsel_P0323R

  }  // namespace expected_lite

  using namespace expected_lite;

  // using expected_lite::expected;
  // using ...

}  // namespace nonstd

namespace std {

// expected: hash support

template <typename T, typename E>
struct hash<nonstd::expected<T, E>> {
  using result_type = std::size_t;
  using argument_type = nonstd::expected<T, E>;

  constexpr result_type operator()(argument_type const &arg) const {
    return arg ? std::hash<T>{}(*arg) : result_type{};
  }
};

// TBD - ?? remove? see spec.
template <typename T, typename E>
struct hash<nonstd::expected<T &, E>> {
  using result_type = std::size_t;
  using argument_type = nonstd::expected<T &, E>;

  constexpr result_type operator()(argument_type const &arg) const {
    return arg ? std::hash<T>{}(*arg) : result_type{};
  }
};

// TBD - implement
// bool(e), hash<expected<void,E>>()(e) shall evaluate to the hashing true;
// otherwise it evaluates to an unspecified value if E is exception_ptr or
// a combination of hashing false and hash<E>()(e.error()).

template <typename E>
struct hash<nonstd::expected<void, E>> {};

}  // namespace std

namespace nonstd {

// void unexpected() is deprecated && removed in C++17

#if nsel_CPP17_OR_GREATER || nsel_COMPILER_MSVC_VERSION > 141
template <typename E>
using unexpected = unexpected_type<E>;
#endif

}  // namespace nonstd

#undef nsel_REQUIRES
#undef nsel_REQUIRES_0
#undef nsel_REQUIRES_T

nsel_RESTORE_WARNINGS()

#endif  // nsel_USES_STD_EXPECTED

#endif  // NONSTD_EXPECTED_LITE_HPP

#pragma once

#include <array>
#include <folly/Random.h>
#include <folly/Synchronized.h>
#include <folly/container/EvictingCacheMap.h>
#include <optional>

#include "common/monitor/Recorder.h"
#include "common/utils/Duration.h"
#include "common/utils/UtcTime.h"
#include "fbs/meta/Common.h"
#include "fbs/meta/Schema.h"
namespace hf3fs::meta::server {

class AclCache {
 public:
  AclCache(size_t cacheSize) {
    for (size_t i = 0; i < kNumShards; i++) {
      shardedMaps_.emplace_back(
          folly::EvictingCacheMap<InodeId, CacheEntry>(std::max(cacheSize / kNumShards, 1ul << 10), 128));
    }
  }

  std::optional<Acl> get(InodeId inode, Duration ttl) {
    static monitor::CountRecorder hit("meta_server.aclcache_hit");
    static monitor::CountRecorder miss("meta_server.aclcache_miss");

    if (ttl.count() == 0) {
      return std::nullopt;
    }
    std::optional<CacheEntry> cached;
    {
      auto &shard = getShard(inode);
      auto guard = shard.lock();
      auto iter = guard->find(inode);
      if (iter != guard->end()) {
        cached = iter->second;
      }
    }

    if (!cached.has_value()) {
      miss.addSample(1);
      return std::nullopt;
    }
    auto deadline = cached->timestamp + ttl * folly::Random::randDouble(0.8, 1.0);
    if (deadline < SteadyClock::now()) {
      miss.addSample(1);
      return std::nullopt;
    }
    hit.addSample(1);
    return cached->acl;
  }

  void set(InodeId inode, Acl acl) {
    auto &shard = getShard(inode);
    shard.lock()->set(inode, {SteadyClock::now(), acl});
  }

  void invalid(InodeId inode) { getShard(inode).lock()->erase(inode); }

 private:
  static constexpr auto kNumShards = 32u;

  struct CacheEntry {
    SteadyTime timestamp;
    Acl acl;
  };
  using CacheMap = folly::Synchronized<folly::EvictingCacheMap<InodeId, CacheEntry>, std::mutex>;

  CacheMap &getShard(InodeId inode) {
    auto shardId = inode.u64() % kNumShards;
    return shardedMaps_[shardId];
  }

  std::vector<CacheMap> shardedMaps_;
};

}  // namespace hf3fs::meta::server

#pragma once

#include <arrow/util/macros.h>
#include <atomic>
#include <folly/Likely.h>
#include <folly/Utility.h>
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <folly/experimental/coro/Baton.h>
#include <folly/functional/Invoke.h>
#include <folly/logging/xlog.h>
#include <functional>
#include <memory>
#include <optional>
#include <type_traits>
#include <utility>

#include "client/mgmtd/ICommonMgmtdClient.h"
#include "client/mgmtd/IMgmtdClientForServer.h"
#include "client/storage/StorageClient.h"
#include "common/kv/IKVEngine.h"
#include "common/kv/ITransaction.h"
#include "common/kv/WithTransaction.h"
#include "common/utils/BackgroundRunner.h"
#include "common/utils/CPUExecutorGroup.h"
#include "common/utils/Coroutine.h"
#include "common/utils/CoroutinesPool.h"
#include "common/utils/Result.h"
#include "core/user/UserStoreEx.h"
#include "fbs/meta/Common.h"
#include "fbs/meta/Service.h"
#include "fdb/FDBRetryStrategy.h"
#include "meta/base/Config.h"
#include "meta/components/ChainAllocator.h"
#include "meta/components/Distributor.h"
#include "meta/components/FileHelper.h"
#include "meta/components/Forward.h"
#include "meta/components/GcManager.h"
#include "meta/components/SessionManager.h"
#include "meta/store/Inode.h"
#include "meta/store/MetaStore.h"
#include "meta/store/ops/BatchOperation.h"

namespace hf3fs::meta::server {

class BatchedOp;

class MetaOperator : public folly::NonCopyableNonMovable {
 public:
  MetaOperator(const Config &cfg,
               flat::NodeId nodeId,
               std::shared_ptr<kv::IKVEngine> kvEngine,
               std::shared_ptr<client::ICommonMgmtdClient> mgmtdClient,
               std::shared_ptr<storage::client::StorageClient> storageClient,
               std::unique_ptr<Forward> forward);

  CoTryTask<void> init(std::optional<Layout> rootLayout);

  void start(CPUExecutorGroup &exec);
  void beforeStop();
  void afterStop();

  CoTryTask<AuthRsp> authenticate(AuthReq req);

  CoTryTask<StatFsRsp> statFs(StatFsReq req);

  CoTryTask<StatRsp> stat(StatReq req);

  CoTryTask<GetRealPathRsp> getRealPath(GetRealPathReq req);

  CoTryTask<OpenRsp> open(OpenReq req);

  CoTryTask<CloseRsp> close(CloseReq req);

  CoTryTask<CreateRsp> create(CreateReq req);

  CoTryTask<MkdirsRsp> mkdirs(MkdirsReq req);

  CoTryTask<SymlinkRsp> symlink(SymlinkReq req);

  CoTryTask<RemoveRsp> remove(RemoveReq req);

  CoTryTask<RenameRsp> rename(RenameReq req);

  CoTryTask<ListRsp> list(ListReq req);

  CoTryTask<TruncateRsp> truncate(TruncateReq req);

  CoTryTask<SyncRsp> sync(SyncReq req);

  CoTryTask<HardLinkRsp> hardLink(HardLinkReq req);

  CoTryTask<SetAttrRsp> setAttr(SetAttrReq req);

  CoTryTask<PruneSessionRsp> pruneSession(PruneSessionReq req);

  CoTryTask<DropUserCacheRsp> dropUserCache(DropUserCacheReq req);

  CoTryTask<LockDirectoryRsp> lockDirectory(LockDirectoryReq req);

  CoTryTask<TestRpcRsp> testRpc(TestRpcReq req);

  CoTryTask<BatchStatRsp> batchStat(BatchStatReq req);

  CoTryTask<BatchStatByPathRsp> batchStatByPath(BatchStatByPathReq req);

 private:
  friend class MockMeta;

  template <typename>
  FRIEND_TEST(TestBatchOp, batch);

  template <typename>
  FRIEND_TEST(TestCreate, batch);

  class Batch {
   public:
    void setNext(BatchedOp *op, folly::coro::Baton *baton) {
      next = op;
      nextBaton = baton;
    }

    bool wakeupNext() {
      if (!next) {
        return false;
      }
      nextBaton->post();
      next = nullptr;
      nextBaton = nullptr;
      return true;
    }

    BatchedOp *getNext() const { return next; }

   private:
    BatchedOp *next = nullptr;
    folly::coro::Baton *nextBaton = nullptr;
  };

  kv::FDBRetryStrategy::Config createRetryConfig() const;
  kv::FDBRetryStrategy createRetryStrategy() const { return kv::FDBRetryStrategy(createRetryConfig()); }

  template <typename Func, typename Arg>
  auto runOp(Func &&func, Arg &&arg)
      -> CoTryTask<typename std::invoke_result_t<Func, MetaStore, Arg &&>::element_type::RspT>;

  template <typename Req, typename Rsp>
  std::unique_ptr<BatchedOp> addBatchReq(InodeId inodeId, BatchedOp::Waiter<Req, Rsp> &waiter) {
    auto func = [&](auto &map) {
      auto [iter, inserted] = map.try_emplace(inodeId);
      auto &batch = iter->second;
      if (inserted) {
        assert(!batch.getNext());
        auto op = std::make_unique<BatchedOp>(*metaStore_, inodeId);
        op->add(waiter);
        waiter.baton.post();
        return op;
      } else if (!batch.getNext()) {
        auto op = std::make_unique<BatchedOp>(*metaStore_, inodeId);
        op->add(waiter);
        batch.setNext(op.get(), &waiter.baton);
        return op;
      } else {
        auto next = batch.getNext();
        auto num_reqs = next->numReqs();
        if (UNLIKELY(config_.max_batch_operations() != 0 && num_reqs >= config_.max_batch_operations())) {
          auto msg = fmt::format("too many batch operations on {}", inodeId);
          XLOG(WARN, msg);
          waiter.result = makeError(MetaCode::kBusy, std::move(msg));
          waiter.baton.post();
        } else {
          if (num_reqs && num_reqs % 1024 == 0) {
            XLOGF(WARN, "{} batch operations on {}", num_reqs, inodeId);
          }
          next->add(waiter);
        }
        return std::unique_ptr<BatchedOp>();
      }
    };
    return batches_.withLock(func, inodeId);
  }

  CoTryTask<Inode> runBatch(InodeId inodeId,
                            std::unique_ptr<BatchedOp> op,
                            std::optional<SteadyTime> deadline = std::nullopt);

  template <typename Req, typename Rsp>
  CoTryTask<Rsp> runInBatch(InodeId inodeId, Req req);

  CoTryTask<void> authenticate(UserInfo &userInfo);

  const Config &config_;
  flat::NodeId nodeId_;
  analytics::StructuredTraceLog<MetaEventTrace> metaEventTraceLog_;
  std::shared_ptr<kv::IKVEngine> kvEngine_;
  std::shared_ptr<client::ICommonMgmtdClient> mgmtd_;
  std::shared_ptr<Distributor> distributor_;
  std::shared_ptr<core::UserStoreEx> userStore_;
  std::shared_ptr<InodeIdAllocator> inodeIdAlloc_;
  std::shared_ptr<ChainAllocator> chainAlloc_;
  std::shared_ptr<FileHelper> fileHelper_;
  std::shared_ptr<SessionManager> sessionManager_;
  std::shared_ptr<GcManager> gcManager_;
  std::unique_ptr<Forward> forward_;

  std::unique_ptr<MetaStore> metaStore_;

  Shards<std::map<InodeId, Batch>, 63> batches_;

  std::atomic_bool stop_{false};
  std::unique_ptr<BackgroundRunner> bgRunner_;
};
}  // namespace hf3fs::meta::server

// take from
// https://github.com/ClickHouse/ClickHouse/blob/766a3a0e111fa075879d7a8e15e749b42f7c748a/base/base/strong_typedef.h
#pragma once

#include <fmt/core.h>
#include <fmt/format.h>
#include <folly/dynamic.h>
#include <functional>
#include <optional>
#include <type_traits>
#include <utility>

#include "common/utils/Result.h"

namespace hf3fs {
template <typename T, typename Tag>
struct StrongTypedef {
 private:
  using Self = StrongTypedef;
  T t;

 public:
  using UnderlyingType = T;
  static constexpr auto kTypeName = Tag::kTypeName;
  using is_serde_copyable = void;

  template <class Enable = typename std::is_copy_constructible<T>::type>
  constexpr explicit StrongTypedef(const T &t_)
      : t(t_) {}
  template <class Enable = typename std::is_move_constructible<T>::type>
  constexpr explicit StrongTypedef(T &&t_)
      : t(std::move(t_)) {}

  template <class Enable = typename std::is_default_constructible<T>::type>
  constexpr StrongTypedef()
      : t() {}

  template <typename U>
  requires(std::same_as<T, U>) static Self from(U &&u) { return Self(std::forward<U>(u)); }

  constexpr StrongTypedef(const Self &) = default;
  constexpr StrongTypedef(Self &&) noexcept(std::is_nothrow_move_constructible_v<T>) = default;

  Self &operator=(const Self &) = default;
  Self &operator=(Self &&) noexcept(std::is_nothrow_move_assignable_v<T>) = default;

  constexpr operator const T &() const { return t; }
  constexpr operator T &() { return t; }
  operator folly::dynamic() const { return t; }

  auto operator<=>(const Self &) const = default;

  // Non-default `operator<=>` could not be used for generating `operator==` implicitly.
  //
  // Why we do not instantiate `operator==` and `operator<=>` between two arithmetic types:
  // 1. for a < 5 where a is a `StrongTypedef<size_t>`, the `t <=> u` is a comparison of different signed integers.
  // 2. without `operator<=>` the comparison above will invoke `operator const T &` and the raw `operator==` (not a
  // function).
  // 3. then the compiler could infer `5` as an unsigned int and avoid `-Wsign-compare`.
  template <typename U>
  requires(!std::is_arithmetic_v<T> && !std::is_arithmetic_v<U>) bool operator==(const U &u) const { return t == u; }
  template <typename U>
  requires(!std::is_arithmetic_v<T> && !std::is_arithmetic_v<U>) auto operator<=>(const U &u) const { return t <=> u; }

  Self &operator++() noexcept requires(std::is_integral_v<T>) {
    ++t;
    return *this;
  }

  Self operator++(int) noexcept requires(std::is_integral_v<T>) { return Self(t++); }

  Self &operator--() noexcept requires(std::is_integral_v<T>) {
    --t;
    return *this;
  }

  Self operator--(int) noexcept requires(std::is_integral_v<T>) { return Self(t--); }

  constexpr T &toUnderType() { return t; }
  constexpr const T &toUnderType() const { return t; }

  constexpr T serdeToReadable() const { return t; }

  static Result<Self> serdeFromReadable(T t) { return Self(std::move(t)); }
};

template <typename T>
concept StrongTyped = requires(const T &t) {
  typename T::UnderlyingType;
  t.toUnderType();
  T::kTypeName;
};

template <typename T>
struct IsStrongTyped : std::false_type {};

template <StrongTyped T>
struct IsStrongTyped<T> : std::true_type {};
}  // namespace hf3fs

template <hf3fs::StrongTyped T>
struct std::hash<T> {
  size_t operator()(const T &x) const { return std::hash<typename T::UnderlyingType>()(x.toUnderType()); }
};

FMT_BEGIN_NAMESPACE

template <hf3fs::StrongTyped T>
struct formatter<T> : formatter<typename T::UnderlyingType> {
  template <typename FormatContext>
  auto format(const T &strongTyped, FormatContext &ctx) const {
    fmt::format_to(ctx.out(), "{}(", T::kTypeName);
    formatter<typename T::UnderlyingType>::format(strongTyped.toUnderType(), ctx);
    return fmt::format_to(ctx.out(), ")");
  }
};

template <hf3fs::StrongTyped T>
struct formatter<std::optional<T>> : formatter<typename T::UnderlyingType> {
  template <typename FormatContext>
  auto format(const std::optional<T> &strongTyped, FormatContext &ctx) const {
    if (strongTyped.has_value()) {
      fmt::format_to(ctx.out(), "{}(", T::kTypeName);
      formatter<typename T::UnderlyingType>::format(strongTyped->toUnderType(), ctx);
      return fmt::format_to(ctx.out(), ")");
    } else {
      return fmt::format_to(ctx.out(), "{}(std::nullopt)", T::kTypeName);
    }
  }
};

FMT_END_NAMESPACE

#define STRONG_TYPEDEF(T, D)                     \
  struct D##Tag {                                \
    static constexpr const char *kTypeName = #D; \
  };                                             \
  using D = hf3fs::StrongTypedef<T, D##Tag>

STRONG_TYPEDEF(char, TypedChar);
static_assert(hf3fs::IsStrongTyped<TypedChar>::value);

#pragma once

#include <string_view>
#include <type_traits>
#include <utility>
#include <variant>

#include "Common.h"
#include "common/serde/Serde.h"
#include "common/utils/Nameof.hpp"
#include "common/utils/StrongType.h"
#include "common/utils/TypeTraits.h"

namespace hf3fs::analytics {

class ObjectVisitor {
 public:
  // default
  template <typename T>
  void visit(std::string_view, T &) = delete;

  template <typename T>
  requires std::is_arithmetic_v<T>
  void visit(std::string_view, T &) = delete;

  template <typename T>
  requires std::is_enum_v<T>
  void visit(std::string_view, T &) = delete;

  template <StrongTyped T>
  void visit(std::string_view, T &) = delete;

  template <serde::ConvertibleToString T>
  void visit(std::string_view, T &) = delete;

  template <serde::SerdeTypeWithoutSpecializedSerdeMethod T>
  void visit(std::string_view, T &) = delete;

  template <typename T>
  requires is_variant_v<T>
  void visit(std::string_view, T &) = delete;

  template <typename T>
  requires is_vector_v<T> || is_set_v<T>
  void visit(std::string_view, T &) = delete;

  template <typename T>
  requires is_optional_v<T>
  void visit(std::string_view, T &) = delete;
};

template <size_t I = 0>
inline void visitVariant(auto &&t, auto &&func) {
  using T = std::decay_t<decltype(t)>;
  using S = std::variant_alternative_t<I, T>;
  if (t.index() == I) {
    func(nameof::nameof_short_type<S>(), std::get<I>(t));
  } else {
    func(nameof::nameof_short_type<S>(), std::variant_alternative_t<I, T>{});
  }
  if constexpr (I + 1 < std::variant_size_v<T>) {
    visitVariant<I + 1>(t, func);
  }
}

template <typename Derived>
class BaseObjectVisitor : public ObjectVisitor {
 public:
  template <typename T>
  void visit(std::string_view k, T &) = delete;

  template <typename T>
  requires std::is_arithmetic_v<T>
  void visit(std::string_view k, T &v) {
    XLOGF(DBG3, "arithmetic visit({})", k);
    static_cast<Derived *>(this)->template visit<T>(k, v);
  }

  template <typename T>
  requires std::is_enum_v<T>
  void visit(std::string_view k, T &) = delete;

  template <serde::ConvertibleToString T>
  void visit(std::string_view k, T &) = delete;

  template <StrongTyped T>
  void visit(std::string_view k, T &v) {
    XLOGF(DBG3, "strongtyped visit({})", k);
    static_cast<Derived *>(this)->template visit<typename T::UnderlyingType>(k, v);
  }

  template <serde::SerdeTypeWithoutSpecializedSerdeMethod T>
  void visit(std::string_view k, T &val) {
    XLOGF(DBG3, "serdetype visit({})", k);
    refl::Helper::iterate<T>(
        [&](auto type) { static_cast<Derived *>(this)->template visit(type.name, val.*type.getter); });
  }

  template <typename T>
  requires is_variant_v<T>
  void visit(std::string_view k, T &val) {
    XLOGF(DBG3, "variant visit({})", k);
    visitVariant(val, [&](std::string_view typeName, auto &&v) {
      std::string altTypeName = std::string{k} + std::string{typeName};
      static_cast<Derived *>(this)->template visit(altTypeName, v);
    });
  }

  template <typename T>
  requires is_vector_v<T> || is_set_v<T>
  void visit(std::string_view k, T &val) {
    XLOGF(DBG3, "container visit({})", k);
    for (auto item : val) {
      static_cast<Derived *>(this)->template visit(k, item);
    }
  }

  template <typename T>
  requires is_optional_v<T>
  void visit(std::string_view k, T &val) {
    XLOGF(DBG3, "optional visit({})", k);
    using ValueType = typename T::value_type;
    if (val.has_value()) {
      static_cast<Derived *>(this)->template visit<ValueType>(k, *val);
    }
  }
};

}  // namespace hf3fs::analytics

#pragma once

#include <folly/AtomicUnorderedMap.h>
#include <memory>
#include <queue>

#include "common/utils/ConfigBase.h"
#include "common/utils/Path.h"
#include "common/utils/UtcTime.h"
#include "kv/KVStore.h"
#include "storage/store/ChunkFileStore.h"
#include "storage/store/ChunkMetadata.h"
#include "storage/store/PhysicalConfig.h"

namespace hf3fs::storage {

class ChunkMetaStore {
 public:
  class Config : public ConfigBase<Config> {
    CONFIG_HOT_UPDATED_ITEM(allocate_size, 256_MB, [](Size s) { return s && s % kMaxChunkSize == 0; });
    CONFIG_HOT_UPDATED_ITEM(recycle_batch_size, 256u, ConfigCheckers::checkPositive);
    CONFIG_HOT_UPDATED_ITEM(punch_hole_batch_size, 16u, ConfigCheckers::checkPositive);
    CONFIG_HOT_UPDATED_ITEM(removed_chunk_expiration_time, 3_d);
    CONFIG_HOT_UPDATED_ITEM(removed_chunk_force_recycled_time, 1_h);
  };

  ChunkMetaStore(const Config &config, ChunkFileStore &fileStore)
      : config_(config),
        fileStore_(fileStore),
        allocateState_(16) {}

  ~ChunkMetaStore();

  // create chunk meta store.
  Result<Void> create(const kv::KVStore::Config &config, const PhysicalConfig &targetConfig);

  // load chunk meta store.
  Result<Void> load(const kv::KVStore::Config &config,
                    const PhysicalConfig &targetConfig,
                    bool createIfMissing = false);

  // add new chunk size.
  Result<Void> addChunkSize(const std::vector<Size> &sizeList);

  // migrate chunk meta store.
  Result<Void> migrate(const kv::KVStore::Config &config, const PhysicalConfig &targetConfig);

  // get metadata of chunk. [thread-safe]
  Result<Void> get(const ChunkId &chunkId, ChunkMetadata &meta);

  // set metadata of chunk. [thread-safe]
  Result<Void> set(const ChunkId &chunkId, const ChunkMetadata &meta);

  // remove metadata of chunk. [thread-safe]
  Result<Void> remove(const ChunkId &chunkId, const ChunkMetadata &meta);

  // create a chunk. [thread-safe]
  Result<Void> createChunk(const ChunkId &chunkId,
                           ChunkMetadata &meta,
                           uint32_t chunkSize,
                           folly::CPUThreadPoolExecutor &executor,
                           bool allowToAllocate);

  // recycle a batch of chunks, return true if has more. [thread-safe]
  Result<bool> punchHole();

  // sync the LOG of kv.
  Result<Void> sync();

  // get used size.
  uint64_t usedSize() const { return std::max(int64_t(createdSize_.load() - removedSize_.load()), 0l); }

  // get reserved and unrecycled size.
  Result<Void> unusedSize(int64_t &reservedSize, int64_t &unrecycledSize);

  // get all uncommitted chunk ids.
  auto &uncommitted() { return uncommitted_; }

  // enable or disable emergency recycling.
  void setEmergencyRecycling(bool enable) { emergencyRecycling_ = enable; }

  // iterator.
  class Iterator {
   public:
    explicit Iterator(kv::KVStore::IteratorPtr it, std::string_view chunkIdPrefix);
    // seek a chunk id prefix.
    void seek(std::string_view chunkIdPrefix);
    // return valid or not.
    bool valid() const;
    // get current chunk id.
    ChunkId chunkId() const;
    // get current metadata.
    Result<ChunkMetadata> meta() const;
    // next metadata.
    void next();
    // check status.
    Result<Void> status() const;

   private:
    kv::KVStore::IteratorPtr it_;
  };
  Result<Iterator> iterator(std::string_view chunkIdPrefix = {});

 protected:
  Result<Void> checkSentinel(std::string_view key);

  Result<Void> getSize(std::string_view key, std::atomic<uint64_t> &size);

  struct AllocateState {
    std::mutex createMutex;
    std::mutex recycleMutex;
    std::mutex allocateMutex;
    std::atomic<bool> loaded{};
    std::atomic<bool> allocating{};
    std::atomic<bool> recycling{};
    uint32_t chunkSize{};
    uint32_t allocateIndex{};               // createMutex.
    std::atomic<uint64_t> startingPoint{};  // createMutex.
    std::atomic<uint64_t> createdCount{};   // createMutex.
    std::atomic<uint64_t> usedCount{};      // createMutex.
    std::atomic<uint64_t> removedCount{};
    std::atomic<uint64_t> recycledCount{};                 // recycleMutex
    std::atomic<uint64_t> reusedCount{};                   // createMutex
    std::atomic<uint64_t> holeCount{};                     // recycleMutex
    std::atomic<UtcTime> oldestRemovedTimestamp{};         // recycleMutex
    std::vector<ChunkPosition> createdChunks;              // createMutex
    std::vector<ChunkPosition> recycledChunks;             // createMutex
    robin_hood::unordered_map<uint32_t, size_t> fileSize;  // createMutex
  };
  void createAllocateState(uint32_t chunkSize);

  Result<AllocateState *> loadAllocateState(uint32_t chunkSize);

  Result<Void> allocateChunks(AllocateState &state, bool withLock = false);

  bool needRecycleRemovedChunks(AllocateState &state);

  Result<Void> recycleRemovedChunks(AllocateState &state, bool withLock = false);

  Result<bool> punchHoleRemovedChunks(AllocateState &state, uint64_t expirationUs);

 private:
  const Config &config_;
  ChunkFileStore &fileStore_;

  std::unique_ptr<kv::KVStore> kv_;
  std::string sentinel_;
  std::string kvName_;
  bool hasSentinel_ = false;
  uint32_t physicalFileCount_ = 256;

  std::atomic<uint64_t> createdSize_ = 0;
  std::atomic<uint64_t> removedSize_ = 0;
  std::vector<ChunkId> uncommitted_;

  std::atomic<bool> emergencyRecycling_ = false;

  folly::AtomicUnorderedInsertMap<uint32_t, std::unique_ptr<AllocateState>> allocateState_;
};

}  // namespace hf3fs::storage

#pragma once

#include <cstddef>
#include <tuple>
#include <type_traits>
#include <utility>

#include "common/utils/TypeTraits.h"

namespace hf3fs::refl {

template <size_t N = 64>
struct Rank : Rank<N - 1> {};
template <>
struct Rank<0> {};

template <class Tuple, class T>
struct Append;
template <class... Ts, class T>
struct Append<std::tuple<Ts...>, T> {
  using type = std::tuple<Ts..., T>;
};
template <class Tuple, class T>
using Append_t = typename Append<Tuple, T>::type;

// REFL_ADD find this function by ADL.
[[maybe_unused]] static std::tuple<> CollectField(::hf3fs::refl::Rank<0>);

#define REFL_NOW decltype(CollectField(::hf3fs::refl::Rank<>{}))
#define REFL_ADD(info)                                                   \
  static ::hf3fs::refl::Append_t<REFL_NOW, decltype(info)> CollectField( \
      ::hf3fs::refl::Rank<std::tuple_size_v<REFL_NOW> + 1>)
#define REFL_ADD_SAFE(info, t) \
  static ::hf3fs::refl::Append_t<t, decltype(info)> CollectField(::hf3fs::refl::Rank<std::tuple_size_v<t> + 1>)

struct Helper {
  template <class T>
  static decltype(CollectField(refl::Rank<>{})) getFieldInfo();
  template <class T>
  requires requires {
    { T::CollectField(refl::Rank<>{}) } -> is_specialization<std::tuple>;
  }
  static decltype(T::CollectField(refl::Rank<>{})) getFieldInfo();

  template <class T>
  using FieldInfoList = decltype(getFieldInfo<T>());

  template <typename T>
  static constexpr auto Size = std::tuple_size_v<FieldInfoList<T>>;

  template <class T, size_t I>
  using FieldInfo = std::tuple_element_t<I, FieldInfoList<T>>;

  template <typename T, bool Backwards = false, size_t I = 0>
  static constexpr auto iterate(auto &&f, auto &&...typeChanged) requires(Size<T> > 0 && I < Size<T>) {
    constexpr auto idx = Backwards ? Size<T> - 1 - I : I;
    auto t = FieldInfo<T, idx>{};
    if constexpr (I > 0 && sizeof...(typeChanged) > 0) {
      constexpr auto pre = Backwards ? Size<T> - I : I - 1;
      if constexpr (!std::is_same_v<member_pointer_to_class_t<FieldInfo<T, idx>::getter>,
                                    member_pointer_to_class_t<FieldInfo<T, pre>::getter>>) {
        auto &&first = getFirstParameter(typeChanged...);
        if constexpr (requires { bool(first()); }) {
          auto result = first();
          if (UNLIKELY(!result)) {
            return result;
          }
        } else {
          first();
        }
      }
    }
    static_assert(requires { f(t); });
    if constexpr (I + 1 < Size<T>) {
      if constexpr (requires { bool(f(t)); }) {
        auto result = f(t);
        if (UNLIKELY(!result)) {
          return result;
        }
      } else {
        f(t);
      }
      return iterate<T, Backwards, I + 1>(f, typeChanged...);
    } else {
      return f(t);
    }
  }

  template <typename T>
  static auto iterate(auto &&) requires(Size<T> == 0) {
    return;
  }

  template <typename T, bool Backwards = false, size_t I = 0>
  static auto visit(auto &&f) requires(Size<T> > 0 && I < Size<T>) {
    constexpr auto idx = Backwards ? Size<T> - 1 - I : I;
    auto t = FieldInfo<T, idx>{};
    if constexpr (requires { f(t); }) {
      return f(t);
    } else {
      return visit<T, Backwards, I + 1>(std::forward<decltype(f)>(f));
    }
  }

  template <typename T>
  static auto visit(auto &&) requires(Size<T> == 0) {
    return;
  }

  template <typename T, typename F, typename Generator, std::size_t... I>
  static auto applyImpl(F &&f, Generator &&gen, std::index_sequence<I...>) {
    return f(gen(FieldInfo<T, I>{})...);
  }

  template <typename T, typename F, typename Generator>
  static auto apply(F &&f, Generator &&gen) {
    return applyImpl<T>(std::forward<F>(f), std::forward<Generator>(gen), std::make_index_sequence<Size<T>>{});
  }
};

}  // namespace hf3fs::refl

#pragma once

#include <fcntl.h>
#include <folly/logging/xlog.h>
#include <functional>
#include <optional>
#include <type_traits>
#include <variant>
#include <vector>

#include "common/app/ClientId.h"
#include "common/app/NodeId.h"
#include "common/serde/Serde.h"
#include "common/serde/Service.h"
#include "common/utils/Result.h"
#include "common/utils/Status.h"
#include "common/utils/StrongType.h"
#include "common/utils/Uuid.h"
#include "fbs/core/user/User.h"
#include "fbs/meta/Common.h"
#include "fbs/meta/Schema.h"
#include "fbs/mgmtd/MgmtdTypes.h"

#define META_SERVICE_VERSION 1

#define CHECK_SESSION(session)             \
  do {                                     \
    if (!session.has_value()) {            \
      return INVALID(#session " not set"); \
    }                                      \
    if (!session->valid()) {               \
      return INVALID(#session " invalid"); \
    }                                      \
  } while (0)

namespace hf3fs::meta {

struct ReqBase {
  SERDE_STRUCT_FIELD(user, UserInfo{});
  SERDE_STRUCT_FIELD(client, ClientId{Uuid::zero()});
  SERDE_STRUCT_FIELD(forward, flat::NodeId(0));
  SERDE_STRUCT_FIELD(uuid, Uuid::zero());

 public:
  // set client id in unittest
  static std::optional<ClientId> &currentClientId() {
    static std::optional<ClientId> clientId;
    return clientId;
  }

  ReqBase(UserInfo user = {}, Uuid uuid = Uuid::zero())
      : user(std::move(user)),
        uuid(uuid) {
    if (currentClientId()) {
      client = *currentClientId();
    }
  }

  Result<Void> checkUuid() const {
    if (client.uuid == Uuid::zero()) return INVALID("Invalid client uuid");
    if (uuid == Uuid::zero()) return INVALID("Invalid request uuid");
    return VALID;
  }
};
struct RspBase {
  SERDE_STRUCT_FIELD(dummy, Void{});
};

// authentication
struct AuthReq : ReqBase {
  SERDE_STRUCT_FIELD(dummy, Void{});

 public:
  AuthReq() = default;
  AuthReq(UserInfo user)
      : ReqBase(std::move(user)) {}
  Result<Void> valid() const { return VALID; }
};

struct AuthRsp : RspBase {
  SERDE_STRUCT_FIELD(user, UserInfo{});

 public:
  AuthRsp() = default;
  AuthRsp(UserInfo user)
      : user(std::move(user)) {}
};

// statFs
struct StatFsReq : ReqBase {
 public:
  StatFsReq() = default;
  StatFsReq(UserInfo user)
      : ReqBase(std::move(user)) {}
  Result<Void> valid() const { return VALID; }
};
struct StatFsRsp : RspBase {
  SERDE_STRUCT_FIELD(capacity, uint64_t(0));
  SERDE_STRUCT_FIELD(used, uint64_t(0));
  SERDE_STRUCT_FIELD(free, uint64_t(0));

 public:
  StatFsRsp() = default;
  StatFsRsp(uint64_t capacity, uint64_t used, uint64_t free)
      : capacity(capacity),
        used(used),
        free(free) {}
};

// stat
struct StatReq : ReqBase {
  SERDE_STRUCT_FIELD(path, PathAt());
  SERDE_STRUCT_FIELD(flags, AtFlags());

 public:
  StatReq() = default;
  StatReq(UserInfo user, PathAt path, AtFlags flags)
      : ReqBase(std::move(user)),
        path(std::move(path)),
        flags(flags) {}
  Result<Void> valid() const { return flags.valid(); }
};
struct StatRsp : RspBase {
  SERDE_STRUCT_FIELD(stat, Inode());

 public:
  StatRsp() = default;
  StatRsp(Inode stat)
      : stat(std::move(stat)) {}
};

// batchStat & batchStatByPath
struct BatchStatReq : ReqBase {
  SERDE_STRUCT_FIELD(inodeIds, std::vector<InodeId>());

 public:
  BatchStatReq() = default;
  BatchStatReq(UserInfo user, std::vector<InodeId> inodeIds)
      : ReqBase(std::move(user)),
        inodeIds(std::move(inodeIds)) {}

  Result<Void> valid() const { return VALID; }
};

struct BatchStatRsp : RspBase {
  SERDE_STRUCT_FIELD(inodes, std::vector<std::optional<Inode>>());

 public:
  BatchStatRsp() = default;
  BatchStatRsp(std::vector<std::optional<Inode>> inodes)
      : inodes(std::move(inodes)) {}
};

struct BatchStatByPathReq : ReqBase {
  SERDE_STRUCT_FIELD(paths, std::vector<PathAt>());
  SERDE_STRUCT_FIELD(flags, AtFlags());

 public:
  BatchStatByPathReq() = default;
  BatchStatByPathReq(UserInfo user, std::vector<PathAt> paths, AtFlags flags)
      : ReqBase(std::move(user)),
        paths(std::move(paths)),
        flags(flags) {}

  Result<Void> valid() const { return flags.valid(); }
};

struct BatchStatByPathRsp : RspBase {
  SERDE_STRUCT_FIELD(inodes, std::vector<Result<Inode>>());

 public:
  BatchStatByPathRsp() = default;
  BatchStatByPathRsp(std::vector<Result<Inode>> inodes)
      : inodes(std::move(inodes)) {}
};

// create
struct CreateReq : ReqBase {
  SERDE_STRUCT_FIELD(path, PathAt());
  SERDE_STRUCT_FIELD(session, std::optional<SessionInfo>());
  SERDE_STRUCT_FIELD(flags, OpenFlags());
  SERDE_STRUCT_FIELD(perm, Permission());
  SERDE_STRUCT_FIELD(layout, std::optional<Layout>());
  SERDE_STRUCT_FIELD(removeChunksBatchSize, uint32_t(0));
  SERDE_STRUCT_FIELD(dynStripe, false);

 public:
  CreateReq() = default;
  CreateReq(UserInfo user,
            PathAt path,
            std::optional<SessionInfo> session,
            OpenFlags flags,
            Permission perm,
            std::optional<Layout> layout = std::nullopt,
            bool dynStripe = false)
      : ReqBase(std::move(user), Uuid::random()),
        path(std::move(path)),
        session(session),
        flags(flags),
        perm(perm),
        layout(std::move(layout)),
        removeChunksBatchSize(32),
        dynStripe(dynStripe) {}

  Result<Void> valid() const {
    RETURN_ON_ERROR(path.validForCreate());
    RETURN_ON_ERROR(flags.valid());
    if (flags.accessType() != AccessType::READ) CHECK_SESSION(session);
    if (layout.has_value()) RETURN_ON_ERROR(layout->valid(true));
    if (flags.contains(O_TRUNC) && removeChunksBatchSize == 0) return INVALID("removeChunksBatchSize == 0");
    return VALID;
  }
};
struct CreateRsp : RspBase {
  SERDE_STRUCT_FIELD(stat, Inode());
  SERDE_STRUCT_FIELD(needTruncate, false);

 public:
  CreateRsp() = default;
  CreateRsp(Inode stat, bool needTruncate)
      : stat(std::move(stat)),
        needTruncate(needTruncate) {}
};

// mkdirs
struct MkdirsReq : ReqBase {
  SERDE_STRUCT_FIELD(path, PathAt());
  SERDE_STRUCT_FIELD(perm, Permission());
  SERDE_STRUCT_FIELD(recursive, false);
  SERDE_STRUCT_FIELD(layout, std::optional<Layout>());

 public:
  MkdirsReq() = default;
  MkdirsReq(UserInfo user, PathAt path, Permission perm, bool recursive, std::optional<Layout> layout = std::nullopt)
      : ReqBase(std::move(user), Uuid::random()),
        path(std::move(path)),
        perm(perm),
        recursive(recursive),
        layout(std::move(layout)) {}
  Result<Void> valid() const {
    if (!path.path.has_value() || path.path->empty()) return INVALID("path not set");
    if (layout.has_value()) RETURN_ON_ERROR(layout->valid(true));
    return Void{};
  }
};

struct MkdirsRsp : RspBase {
  SERDE_STRUCT_FIELD(stat, Inode());

 public:
  MkdirsRsp() = default;
  MkdirsRsp(Inode stat)
      : stat(std::move(stat)) {}
};

// symlink
struct SymlinkReq : ReqBase {
  SERDE_STRUCT_FIELD(path, PathAt());
  SERDE_STRUCT_FIELD(target, Path());

 public:
  SymlinkReq() = default;
  SymlinkReq(UserInfo user, PathAt path, Path target)
      : ReqBase(std::move(user), Uuid::random()),
        path(std::move(path)),
        target(std::move(target)) {}
  Result<Void> valid() const { return path.validForCreate(); }
};
struct SymlinkRsp : RspBase {
  SERDE_STRUCT_FIELD(stat, Inode());

 public:
  SymlinkRsp() = default;
  SymlinkRsp(Inode stat)
      : stat(std::move(stat)) {}
};

// hardlink
struct HardLinkReq : ReqBase {
  SERDE_STRUCT_FIELD(oldPath, PathAt());
  SERDE_STRUCT_FIELD(newPath, PathAt());
  SERDE_STRUCT_FIELD(flags, AtFlags());

 public:
  HardLinkReq() = default;
  HardLinkReq(UserInfo user, PathAt oldPath, PathAt newPath, AtFlags flags)
      : ReqBase(std::move(user), Uuid::random()),
        oldPath(std::move(oldPath)),
        newPath(std::move(newPath)),
        flags(flags) {}
  Result<Void> valid() const {
    RETURN_ON_ERROR(newPath.validForCreate());
    RETURN_ON_ERROR(flags.valid());
    return VALID;
  }
};
struct HardLinkRsp : RspBase {
  SERDE_STRUCT_FIELD(stat, Inode());

 public:
  HardLinkRsp() = default;
  HardLinkRsp(Inode stat)
      : stat(std::move(stat)) {}
};

// remove
struct RemoveReq : ReqBase {
  SERDE_STRUCT_FIELD(path, PathAt());
  SERDE_STRUCT_FIELD(atFlags, AtFlags());
  SERDE_STRUCT_FIELD(recursive, false);
  SERDE_STRUCT_FIELD(checkType, false);
  SERDE_STRUCT_FIELD(inodeId, std::optional<InodeId>());

 public:
  RemoveReq() = default;
  RemoveReq(UserInfo user,
            PathAt path,
            AtFlags flags,
            bool recursive,
            bool checkType = false,
            std::optional<InodeId> inodeId = std::nullopt)
      : ReqBase(std::move(user), Uuid::random()),
        path(std::move(path)),
        atFlags(flags),
        recursive(recursive),
        checkType(checkType),
        inodeId(inodeId) {}
  Result<Void> valid() const {
    if (path.path.has_value()) RETURN_ON_ERROR(path.validForCreate());
    if (recursive) RETURN_ON_ERROR(checkUuid());
    return VALID;
  }
};

struct RemoveRsp : RspBase {
  SERDE_STRUCT_FIELD(dummy, Void{});
};

// open
struct OpenReq : ReqBase {
  SERDE_STRUCT_FIELD(path, PathAt());
  SERDE_STRUCT_FIELD(session, std::optional<SessionInfo>());
  SERDE_STRUCT_FIELD(flags, OpenFlags());
  SERDE_STRUCT_FIELD(removeChunksBatchSize, uint32_t(0));
  SERDE_STRUCT_FIELD(dynStripe, false);

 public:
  OpenReq() = default;
  OpenReq(UserInfo user, PathAt path, std::optional<SessionInfo> session, OpenFlags flags, bool dynStripe = false)
      : ReqBase(std::move(user)),
        path(std::move(path)),
        session(session),
        flags(flags),
        removeChunksBatchSize(32),
        dynStripe(dynStripe) {}
  Result<Void> valid() const {
    RETURN_ON_ERROR(flags.valid());
    if (flags.accessType() != AccessType::READ) CHECK_SESSION(session);
    if (flags.accessType() == AccessType::READ && (flags.contains(O_TRUNC) || flags.contains(O_APPEND)))
      return INVALID("O_RDONLY with O_TRUNC or O_APPEND");
    if (flags.contains(O_TRUNC) && removeChunksBatchSize == 0) return INVALID("removeChunksBatchSize == 0");
    return VALID;
  }
};
struct OpenRsp : RspBase {
  SERDE_STRUCT_FIELD(_unused, (uint32_t)0);
  SERDE_STRUCT_FIELD(stat, Inode());
  SERDE_STRUCT_FIELD(needTruncate, false);

 public:
  OpenRsp() = default;
  OpenRsp(Inode stat, bool needTruncate)
      : stat(std::move(stat)),
        needTruncate(needTruncate) {}
};

// sync
struct SyncReq : ReqBase {
  SERDE_STRUCT_FIELD(inode, InodeId());
  SERDE_STRUCT_FIELD(updateLength, false);
  SERDE_STRUCT_FIELD(atime, std::optional<UtcTime>());
  SERDE_STRUCT_FIELD(mtime, std::optional<UtcTime>());
  SERDE_STRUCT_FIELD(truncated, false);
  SERDE_STRUCT_FIELD(lengthHint, std::optional<VersionedLength>());

 public:
  SyncReq() = default;
  SyncReq(UserInfo user,
          InodeId inode,
          bool updateLength,
          std::optional<UtcTime> atime,
          std::optional<UtcTime> mtime,
          bool truncated = false,
          std::optional<VersionedLength> hint = std::nullopt)
      : ReqBase(std::move(user)),
        inode(inode),
        updateLength(updateLength),
        atime(atime),
        mtime(mtime),
        truncated(truncated),
        lengthHint(hint) {}
  Result<Void> valid() const {
    if (truncated && !updateLength) return INVALID("truncate but not updateLength");
    return VALID;
  }
};
struct SyncRsp : RspBase {
  SERDE_STRUCT_FIELD(_unused, (uint32_t)0);
  SERDE_STRUCT_FIELD(stat, Inode());

 public:
  SyncRsp() = default;
  SyncRsp(Inode stat)
      : stat(std::move(stat)) {}
};

// close
struct CloseReq : ReqBase {
  SERDE_STRUCT_FIELD(inode, InodeId());
  SERDE_STRUCT_FIELD(session, std::optional<SessionInfo>());
  SERDE_STRUCT_FIELD(updateLength, false);
  SERDE_STRUCT_FIELD(atime, std::optional<UtcTime>());
  SERDE_STRUCT_FIELD(mtime, std::optional<UtcTime>());
  SERDE_STRUCT_FIELD(lengthHint, std::optional<VersionedLength>());

 public:
  bool pruneSession = false;

  CloseReq() = default;
  CloseReq(UserInfo user,
           InodeId inode,
           std::optional<SessionInfo> session,
           bool updateLength,
           std::optional<UtcTime> atime,
           std::optional<UtcTime> mtime)
      : ReqBase(std::move(user)),
        inode(inode),
        session(session),
        updateLength(updateLength),
        atime(atime),
        mtime(mtime) {}
  Result<Void> valid() const {
    if (updateLength || mtime.has_value()) CHECK_SESSION(session);
    return VALID;
  }
};
struct CloseRsp : RspBase {
  SERDE_STRUCT_FIELD(_unused, (uint32_t)0);
  SERDE_STRUCT_FIELD(stat, Inode());

 public:
  CloseRsp() = default;
  CloseRsp(Inode stat)
      : stat(std::move(stat)) {}
};

// rename
struct RenameReq : ReqBase {
  SERDE_STRUCT_FIELD(src, PathAt());
  SERDE_STRUCT_FIELD(dest, PathAt());
  SERDE_STRUCT_FIELD(moveToTrash, false);
  SERDE_STRUCT_FIELD(inodeId, std::optional<InodeId>());

 public:
  RenameReq() = default;
  RenameReq(UserInfo user,
            PathAt src,
            PathAt dest,
            bool moveToTrash = false,
            std::optional<InodeId> inodeId = std::nullopt)
      : ReqBase(std::move(user), Uuid::random()),
        src(std::move(src)),
        dest(std::move(dest)),
        moveToTrash(moveToTrash),
        inodeId(inodeId) {}
  Result<Void> valid() const {
    RETURN_ON_ERROR(src.validForCreate());
    RETURN_ON_ERROR(dest.validForCreate());
    if (moveToTrash) RETURN_ON_ERROR(checkUuid());
    return VALID;
  }
};
struct RenameRsp : RspBase {
  SERDE_STRUCT_FIELD(dummy, Void{});
  SERDE_STRUCT_FIELD(stat, std::optional<Inode>());

 public:
  RenameRsp() = default;
  RenameRsp(Inode stat)
      : stat(std::move(stat)) {}
};

// list
struct ListReq : ReqBase {
  SERDE_STRUCT_FIELD(path, PathAt());
  SERDE_STRUCT_FIELD(prev, std::string());
  SERDE_STRUCT_FIELD(limit, (int32_t)-1);
  SERDE_STRUCT_FIELD(status, false);

 public:
  ListReq() = default;
  ListReq(UserInfo user, PathAt path, std::string prev = {}, int32_t limit = -1, bool status = false)
      : ReqBase(std::move(user)),
        path(std::move(path)),
        prev(std::move(prev)),
        limit(limit),
        status(status) {}
  Result<Void> valid() const { return VALID; }
};
struct ListRsp : RspBase {
  SERDE_STRUCT_FIELD(entries, std::vector<DirEntry>());
  SERDE_STRUCT_FIELD(inodes, std::vector<Inode>());
  SERDE_STRUCT_FIELD(more, false);

 public:
  ListRsp() = default;
  ListRsp(std::vector<DirEntry> entries, std::vector<Inode> inodes, bool more)
      : entries(std::move(entries)),
        inodes(std::move(inodes)),
        more(more) {}
};

// truncate
struct TruncateReq : ReqBase {
  SERDE_STRUCT_FIELD(inode, InodeId(0));
  SERDE_STRUCT_FIELD(length, uint64_t(0));
  SERDE_STRUCT_FIELD(removeChunksBatchSize, uint32_t(0));

 public:
  TruncateReq() = default;
  TruncateReq(UserInfo user, InodeId inode, uint64_t length, uint32_t removeChunksBatchSize)
      : ReqBase(std::move(user)),
        inode(inode),
        length(length),
        removeChunksBatchSize(removeChunksBatchSize) {}
  Result<Void> valid() const {
    if (removeChunksBatchSize == 0) return INVALID("removeChunksBatchSize == 0");
    return VALID;
  }
};
struct TruncateRsp : RspBase {
  SERDE_STRUCT_FIELD(chunksRemoved, uint32_t(0));
  SERDE_STRUCT_FIELD(stat, Inode());
  SERDE_STRUCT_FIELD(finished, true);

 public:
  TruncateRsp() = default;
  TruncateRsp(Inode stat)
      : TruncateRsp(std::move(stat), 0, true) {}
  TruncateRsp(Inode stat, uint32_t chunksRemoved, bool finished)
      : chunksRemoved(chunksRemoved),
        stat(std::move(stat)),
        finished(finished) {}
};

// getRealPath
struct GetRealPathReq : ReqBase {
  SERDE_STRUCT_FIELD(path, PathAt());
  SERDE_STRUCT_FIELD(absolute, false);

 public:
  GetRealPathReq() = default;
  GetRealPathReq(UserInfo user, PathAt path, bool absolute)
      : ReqBase(std::move(user)),
        path(std::move(path)),
        absolute(absolute) {}
  Result<Void> valid() const { return VALID; }
};
struct GetRealPathRsp : RspBase {
  SERDE_STRUCT_FIELD(path, Path());

 public:
  GetRealPathRsp() = default;
  GetRealPathRsp(Path path)
      : path(std::move(path)) {}
};

// setAttr
struct SetAttrReq : ReqBase {
  SERDE_STRUCT_FIELD(path, PathAt());
  SERDE_STRUCT_FIELD(flags, AtFlags());
  SERDE_STRUCT_FIELD(uid, std::optional<Uid>());
  SERDE_STRUCT_FIELD(gid, std::optional<Gid>());
  SERDE_STRUCT_FIELD(perm, std::optional<Permission>());
  SERDE_STRUCT_FIELD(atime, std::optional<UtcTime>());
  SERDE_STRUCT_FIELD(mtime, std::optional<UtcTime>());
  SERDE_STRUCT_FIELD(layout, std::optional<Layout>());
  SERDE_STRUCT_FIELD(iflags, std::optional<IFlags>());
  SERDE_STRUCT_FIELD(dynStripe, uint32_t(0));

 public:
  static SetAttrReq setLayout(UserInfo user, PathAt path, AtFlags flags, Layout layout) {
    return {std::move(user), std::move(path), flags, {}, {}, {}, {}, {}, std::move(layout)};
  }
  static SetAttrReq setPermission(UserInfo user,
                                  PathAt path,
                                  AtFlags flags,
                                  std::optional<Uid> uid,
                                  std::optional<Gid> gid,
                                  std::optional<Permission> perm,
                                  std::optional<IFlags> iflags = std::nullopt) {
    return {std::move(user), std::move(path), flags, uid, gid, perm, {}, {}, {}, iflags};
  }
  static SetAttrReq setIFlags(UserInfo user, PathAt path, IFlags iflags) {
    return {std::move(user), std::move(path), AtFlags(), {}, {}, {}, {}, {}, {}, iflags};
  }
  static SetAttrReq utimes(UserInfo user,
                           PathAt path,
                           AtFlags flags,
                           std::optional<UtcTime> atime,
                           std::optional<UtcTime> mtime) {
    return {std::move(user), std::move(path), flags, {}, {}, {}, atime, mtime, {}};
  }
  static SetAttrReq extendStripe(UserInfo user, InodeId inode, uint32_t stripe) {
    return {std::move(user), PathAt(inode), AtFlags{}, {}, {}, {}, {}, {}, {}, {}, stripe};
  }
  Result<Void> valid() const {
    if (layout.has_value()) RETURN_ON_ERROR(layout->valid(true));
    if (iflags && (*iflags & ~FS_FL_SUPPORTED)) {
      return MAKE_ERROR_F(StatusCode::kInvalidArg, "only support {:x}", (uint32_t)FS_FL_SUPPORTED);
    }
    return VALID;
  }
};

struct SetAttrRsp : RspBase {
  SERDE_STRUCT_FIELD(stat, Inode());

 public:
  SetAttrRsp() = default;
  SetAttrRsp(Inode stat)
      : stat(std::move(stat)) {}
};

// pruneSession
struct PruneSessionReq : ReqBase {
  SERDE_STRUCT_FIELD(client, ClientId::zero());
  SERDE_STRUCT_FIELD(sessions, std::vector<Uuid>());
  SERDE_STRUCT_FIELD(needSync, std::vector<bool>());  // deperated

 public:
  PruneSessionReq() = default;
  PruneSessionReq(ClientId client, std::vector<Uuid> sessions)
      : ReqBase(),
        client(client),
        sessions(std::move(sessions)) {}
  Result<Void> valid() const { return VALID; }
};
struct PruneSessionRsp : RspBase {
  SERDE_STRUCT_FIELD(dummy, Void{});
};

// dropUserCache
struct DropUserCacheReq : ReqBase {
  SERDE_STRUCT_FIELD(uid, std::optional<Uid>());
  SERDE_STRUCT_FIELD(dropAll, false);

 public:
  DropUserCacheReq() = default;
  DropUserCacheReq(std::optional<Uid> u, bool d)
      : uid(std::move(u)),
        dropAll(d) {}
  Result<Void> valid() const { return VALID; }
};
struct DropUserCacheRsp : RspBase {
  SERDE_STRUCT_FIELD(dummy, Void{});
};

// lockDirectory
struct LockDirectoryReq : ReqBase {
  enum class LockAction : uint8_t {
    TryLock,
    PreemptLock,
    UnLock,
    Clear,
  };
  SERDE_STRUCT_FIELD(inode, InodeId());
  SERDE_STRUCT_FIELD(action, LockAction::TryLock);

 public:
  LockDirectoryReq() = default;
  LockDirectoryReq(UserInfo user, InodeId inode, LockAction action)
      : ReqBase(user),
        inode(inode),
        action(action) {}
  Result<Void> valid() const { return VALID; }
};

struct LockDirectoryRsp : RspBase {
  SERDE_STRUCT_FIELD(dummy, Void{});

 public:
  LockDirectoryRsp() = default;
};

// testRpc
struct TestRpcReq : ReqBase {
  SERDE_STRUCT_FIELD(path, PathAt());
  SERDE_STRUCT_FIELD(flags, uint32_t(0));

 public:
  Result<Void> valid() const { return VALID; }
};
struct TestRpcRsp : RspBase {
  SERDE_STRUCT_FIELD(stat, Inode());
};

/* MetaSerde service */
SERDE_SERVICE(MetaSerde, 4) {
#define META_SERVICE_METHOD(NAME, CODE, REQ, RSP)                                   \
  SERDE_SERVICE_METHOD(NAME, CODE, REQ, RSP);                                       \
                                                                                \
 public:                                                                            \
  static constexpr std::string_view getRpcName(const REQ &) { return #NAME; }       \
  static_assert(serde::SerializableToBytes<REQ> && serde::SerializableToJson<REQ>); \
  static_assert(serde::SerializableToBytes<RSP> && serde::SerializableToJson<RSP>)

  META_SERVICE_METHOD(statFs, 1, StatFsReq, StatFsRsp);
  META_SERVICE_METHOD(stat, 2, StatReq, StatRsp);
  META_SERVICE_METHOD(create, 3, CreateReq, CreateRsp);
  META_SERVICE_METHOD(mkdirs, 4, MkdirsReq, MkdirsRsp);
  META_SERVICE_METHOD(symlink, 5, SymlinkReq, SymlinkRsp);
  META_SERVICE_METHOD(hardLink, 6, HardLinkReq, HardLinkRsp);
  META_SERVICE_METHOD(remove, 7, RemoveReq, RemoveRsp);
  META_SERVICE_METHOD(open, 8, OpenReq, OpenRsp);
  META_SERVICE_METHOD(sync, 9, SyncReq, SyncRsp);
  META_SERVICE_METHOD(close, 10, CloseReq, CloseRsp);
  META_SERVICE_METHOD(rename, 11, RenameReq, RenameRsp);
  META_SERVICE_METHOD(list, 12, ListReq, ListRsp);
  // deperated:
  META_SERVICE_METHOD(truncate, 13, TruncateReq, TruncateRsp);
  META_SERVICE_METHOD(getRealPath, 14, GetRealPathReq, GetRealPathRsp);
  META_SERVICE_METHOD(setAttr, 15, SetAttrReq, SetAttrRsp);
  META_SERVICE_METHOD(pruneSession, 16, PruneSessionReq, PruneSessionRsp);
  META_SERVICE_METHOD(dropUserCache, 17, DropUserCacheReq, DropUserCacheRsp);
  META_SERVICE_METHOD(authenticate, 18, AuthReq, AuthRsp);
  META_SERVICE_METHOD(lockDirectory, 19, LockDirectoryReq, LockDirectoryRsp);
  META_SERVICE_METHOD(batchStat, 20, BatchStatReq, BatchStatRsp);
  META_SERVICE_METHOD(batchStatByPath, 21, BatchStatByPathReq, BatchStatByPathRsp);

  META_SERVICE_METHOD(testRpc, 50, TestRpcReq, TestRpcRsp);

#undef META_SERVICE_METHOD
};

}  // namespace hf3fs::meta

#pragma once

#include <folly/Synchronized.h>
#include <folly/Utility.h>
#include <folly/experimental/coro/Mutex.h>
#include <gtest/gtest_prod.h>
#include <map>
#include <mutex>
#include <span>
#include <unordered_map>
#include <vector>

#include "StorageClient.h"
#include "StorageMessenger.h"
#include "common/net/Client.h"
#include "common/utils/Address.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Result.h"
#include "fbs/storage/Common.h"

namespace hf3fs::storage::client {

class StorageClientInMem : public StorageClient {
 public:
  StorageClientInMem(ClientId clientId, const Config &config, hf3fs::client::ICommonMgmtdClient &mgmtdClient);

  ~StorageClientInMem() override = default;

  hf3fs::client::ICommonMgmtdClient &getMgmtdClient() override { return mgmtdClient_; }

  CoTryTask<void> batchRead(std::span<ReadIO> readIOs,
                            const flat::UserInfo &userInfo,
                            const ReadOptions &options = ReadOptions(),
                            std::vector<ReadIO *> *failedIOs = nullptr) override;

  CoTryTask<void> batchWrite(std::span<WriteIO> writeIOs,
                             const flat::UserInfo &userInfo,
                             const WriteOptions &options = WriteOptions(),
                             std::vector<WriteIO *> *failedIOs = nullptr) override;

  CoTryTask<void> read(ReadIO &readIO,
                       const flat::UserInfo &userInfo,
                       const ReadOptions &options = ReadOptions()) override;

  CoTryTask<void> write(WriteIO &writeIO,
                        const flat::UserInfo &userInfo,
                        const WriteOptions &options = WriteOptions()) override;

  CoTryTask<void> queryLastChunk(std::span<QueryLastChunkOp> ops,
                                 const flat::UserInfo &userInfo,
                                 const ReadOptions &options = ReadOptions(),
                                 std::vector<QueryLastChunkOp *> *failedOps = nullptr) override;

  CoTryTask<void> removeChunks(std::span<RemoveChunksOp> ops,
                               const flat::UserInfo &userInfo,
                               const WriteOptions &options = WriteOptions(),
                               std::vector<RemoveChunksOp *> *failedOps = nullptr) override;

  CoTryTask<void> truncateChunks(std::span<TruncateChunkOp> ops,
                                 const flat::UserInfo &userInfo,
                                 const WriteOptions &options = WriteOptions(),
                                 std::vector<TruncateChunkOp *> *failedOps = nullptr) override;

  CoTryTask<SpaceInfoRsp> querySpaceInfo(NodeId) override;

  CoTryTask<CreateTargetRsp> createTarget(NodeId nodeId, const CreateTargetReq &req) override;

  CoTryTask<OfflineTargetRsp> offlineTarget(NodeId nodeId, const OfflineTargetReq &req) override;

  CoTryTask<RemoveTargetRsp> removeTarget(NodeId nodeId, const RemoveTargetReq &req) override;

  CoTryTask<std::vector<Result<QueryChunkRsp>>> queryChunk(const QueryChunkReq &req) override;

  CoTryTask<ChunkMetaVector> getAllChunkMetadata(const ChainId &chainId, const TargetId &targetId) override;

  // for meta test
  CoTask<void> injectErrorOnChain(ChainId chainId, Result<Void> error);

 private:
  struct ChunkData {
    std::vector<uint8_t> content;
    uint32_t capacity;
    uint32_t version;
  };

  struct Chain {
    folly::coro::Mutex mutex;
    std::map<ChunkId, ChunkData> chunks;
    Result<Void> error = Void{};
  };

  CoTask<std::pair<Chain *, std::unique_lock<folly::coro::Mutex>>> getChain(ChainId chainId) {
    auto &chain = (*chains_.lock())[chainId];
    auto guard = co_await chain.mutex.co_scoped_lock();
    co_return std::make_pair(&chain, std::move(guard));
  }

  using ChunkDataProcessor = std::function<CoTryTask<void>(const ChunkId &, const ChunkData &)>;

  CoTryTask<std::vector<std::pair<ChunkId, ChunkData>>> doQuery(const ChainId &chainId, const ChunkIdRange &range);

  CoTryTask<uint32_t> processQueryResults(const ChainId chainId,
                                          const ChunkIdRange &range,
                                          ChunkDataProcessor processor,
                                          bool &moreChunksInRange);

  hf3fs::client::ICommonMgmtdClient &mgmtdClient_;
  folly::Synchronized<std::unordered_map<ChainId, Chain>, std::mutex> chains_;
};

}  // namespace hf3fs::storage::client

#pragma once

#include <compare>
#include <scn/tuple_return.h>

#include "common/app/NodeId.h"
#include "common/serde/SerdeComparisons.h"
#include "common/serde/SerdeHelper.h"
#include "common/utils/Address.h"
#include "common/utils/UtcTime.h"
#include "common/utils/VersionInfo.h"

namespace hf3fs::flat {

struct ServiceGroupInfo : public serde::SerdeHelper<ServiceGroupInfo> {
  ServiceGroupInfo() = default;
  ServiceGroupInfo(std::set<String> ss, std::vector<net::Address> es);

  bool operator==(const ServiceGroupInfo &other) const;

  SERDE_STRUCT_FIELD(services, std::set<String>{});
  SERDE_STRUCT_FIELD(endpoints, std::vector<net::Address>{});
};

struct TagPair : public serde::SerdeHelper<TagPair> {
  TagPair() = default;
  explicit TagPair(String k);
  TagPair(String k, String v);

  bool operator==(const TagPair &other) const;

  SERDE_STRUCT_FIELD(key, String{});
  SERDE_STRUCT_FIELD(value, String{});
};

struct ReleaseVersion : public serde::SerdeHelper<ReleaseVersion> {
  ReleaseVersion() = default;

  ReleaseVersion(bool isReleaseVersion,
                 uint32_t tagDate,
                 uint16_t patchVersion,
                 uint32_t pipelineId,
                 uint32_t commitHashShort);

  static ReleaseVersion fromV0(uint8_t majorv,
                               uint8_t minorv,
                               uint8_t patchv,
                               uint32_t shortH,
                               uint64_t buildTimeInSec,
                               uint32_t buildPipelineId);

  static ReleaseVersion fromVersionInfoV0();
  static ReleaseVersion fromVersionInfo();

  std::strong_ordering operator<=>(const ReleaseVersion &other) const;
  bool operator==(const ReleaseVersion &other) const;

  uint32_t getTagDate() const { return HelperV1::getTagDate(*this); }

  uint32_t getPatchVersion() const { return HelperV1::getPatchVersion(*this); }

  bool getIsReleaseVersion() const { return HelperV1::getIsReleaseVersion(*this); }

  uint32_t getPipelineId() const { return HelperV1::getPipelineId(*this); }

  uint32_t getCommitHashShort() const { return HelperV1::getCommitHashShort(*this); }

  String toString() const;

 private:
  struct HelperV1 {
    static bool getIsReleaseVersion(const ReleaseVersion &rv);
    static void setIsReleaseVersion(ReleaseVersion &rv, bool isReleaseVersion);
    static uint32_t getTagDate(const ReleaseVersion &rv);
    static void setTagDate(ReleaseVersion &rv, uint32_t tagDate);
    static uint32_t getPatchVersion(const ReleaseVersion &rv);
    static void setPatchVersion(ReleaseVersion &rv, uint32_t patchVersion);
    static uint32_t getPipelineId(const ReleaseVersion &rv);
    static void setPipelineId(ReleaseVersion &rv, uint32_t pipelineId);
    static uint32_t getCommitHashShort(const ReleaseVersion &rv);
    static void setCommitHashShort(ReleaseVersion &rv, uint32_t commitHashShort);

   private:
    // just for demonstration
    // structVersion_ <=> rv.majorVersion
    // isReleaseVersion_ <=> rv.minorVersion
    // <tagDate_, patchVersion_> <=> rv.buildTimeInSeconds
    // pipelineId_ <=> rv.pipelineId
    // commitHashShort_ <=> rv.commitHashShort
    const uint8_t structVersion_ = 1;
    bool isReleaseVersion_ = false;
    uint32_t patchVersion_ = 0;
    uint32_t tagDate_ = 0;
    uint32_t pipelineId_ = 0;
    uint32_t commitHashShort_ = 0;
  };

  // NOTE: other fields are just for storage when majorVersion != 0
  SERDE_STRUCT_FIELD(majorVersion, uint8_t(1));
  SERDE_STRUCT_FIELD(minorVersion, uint8_t(0));
  SERDE_STRUCT_FIELD(patchVersion, uint8_t(0));
  SERDE_STRUCT_FIELD(commitHashShort, uint32_t(0));
  SERDE_STRUCT_FIELD(buildTimeInSeconds, uint64_t(0));
  SERDE_STRUCT_FIELD(buildPipelineId, uint32_t(0));
};

struct FbsAppInfo : public serde::SerdeHelper<FbsAppInfo> {
  bool operator==(const FbsAppInfo &other) const;

  SERDE_STRUCT_FIELD(nodeId, NodeId(0));
  SERDE_STRUCT_FIELD(hostname, String{});
  SERDE_STRUCT_FIELD(pid, uint32_t(0));
  SERDE_STRUCT_FIELD(serviceGroups, std::vector<ServiceGroupInfo>{});
  SERDE_STRUCT_FIELD(releaseVersion, ReleaseVersion{});
  SERDE_STRUCT_FIELD(podname, String{});
};

struct AppInfo : FbsAppInfo {
  bool operator==(const AppInfo &other) const;

  String clusterId;
  std::vector<TagPair> tags;
};

std::vector<net::Address> extractAddresses(const std::vector<ServiceGroupInfo> &serviceGroups,
                                           const String &serviceName,
                                           std::optional<net::Address::Type> addressType = std::nullopt);

inline constexpr auto kDisabledTagKey = "Disable";
inline constexpr auto kTrafficZoneTagKey = "TrafficZone";

int findTag(const std::vector<TagPair> &tags, std::string_view tag);
bool removeTag(std::vector<TagPair> &tags, std::string_view tag);
}  // namespace hf3fs::flat

FMT_BEGIN_NAMESPACE

template <>
struct formatter<hf3fs::flat::ReleaseVersion> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(const hf3fs::flat::ReleaseVersion val, FormatContext &ctx) const -> decltype(ctx.out()) {
    return fmt::format_to(ctx.out(), "{}", val.toString());
  }
};

template <>
struct formatter<hf3fs::flat::TagPair> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(const hf3fs::flat::TagPair val, FormatContext &ctx) const -> decltype(ctx.out()) {
    if (val.value.empty()) {
      return fmt::format_to(ctx.out(), "{}", val.key);
    }
    return fmt::format_to(ctx.out(), "{}={}", val.key, val.value);
  }
};

FMT_END_NAMESPACE

#pragma once

#include <cassert>
#include <folly/Likely.h>
#include <folly/Synchronized.h>
#include <folly/Utility.h>
#include <folly/experimental/coro/Baton.h>
#include <folly/futures/Future.h>
#include <folly/futures/Promise.h>
#include <folly/io/async/Request.h>
#include <folly/logging/xlog.h>
#include <memory>
#include <mutex>
#include <optional>
#include <string>
#include <utility>
#include <variant>

#include "common/utils/Coroutine.h"
#include "common/utils/Result.h"
#include "fbs/meta/Common.h"
#include "fbs/meta/Schema.h"
#include "meta/store/DirEntry.h"
#include "meta/store/Inode.h"
namespace hf3fs::meta::server {

class BatchContext : public folly::RequestData {
 public:
  template <typename T>
  struct SharedFuture : folly::NonCopyableNonMovable {
    Result<T> value = makeError(StatusCode::kUnknown);
    folly::coro::Baton baton;
  };

  template <typename T>
  struct LoadGuard {
    bool needLoad;
    std::shared_ptr<SharedFuture<T>> future;

    LoadGuard(bool needLoad, std::shared_ptr<SharedFuture<T>> future)
        : needLoad(needLoad),
          future(future) {}

    ~LoadGuard() {
      if (needLoad && !future->baton.ready()) {
        future->value = makeError(StatusCode::kUnknown, "load failed in BatchContext");
        future->baton.post();
      }
    }

    void set(const Result<T> &r) {
      assert(!future->baton.ready());
      future->value = r;
      future->baton.post();
    }

    CoTryTask<T> coAwait() {
      co_await future->baton;
      co_return future->value;
    }
  };

  static folly::ShallowCopyRequestContextScopeGuard create() {
    return folly::ShallowCopyRequestContextScopeGuard{token(), std::make_unique<BatchContext>()};
  }

  static inline BatchContext *get() {
    auto requestContext = folly::RequestContext::try_get();
    if (LIKELY(requestContext == nullptr)) {
      return nullptr;
    }
    return dynamic_cast<BatchContext *>(requestContext->getContextData(token()));
  }

  LoadGuard<std::optional<Inode>> loadInode(InodeId inodeId) {
    return loadImpl<InodeId, std::optional<Inode>>(inodes_, inodeId);
  }

  LoadGuard<std::optional<DirEntry>> loadDirEntry(InodeId parent, std::string name) {
    return loadImpl<std::pair<InodeId, std::string>, std::optional<DirEntry>>(entries_, {parent, std::move(name)});
  }

  bool hasCallback() override { return false; }

 private:
  static constexpr const char *kTokenName = "hf3fs::meta::server::BatchContext";

  static folly::RequestToken const &token() {
    static folly::RequestToken const token(kTokenName);
    return token;
  }

  template <typename K, typename T>
  using SynchronizedFutureMap = folly::Synchronized<std::map<K, std::shared_ptr<SharedFuture<T>>>, std::mutex>;

  template <typename K, typename T>
  LoadGuard<T> loadImpl(SynchronizedFutureMap<K, T> &map, K key) {
    auto guard = map.lock();
    auto iter = guard->find(key);
    if (iter != guard->end()) {
      return LoadGuard<T>(false, iter->second);
    }
    auto future = std::make_shared<SharedFuture<T>>();
    guard->emplace(std::move(key), future);
    return LoadGuard<T>(true, future);
  }

  SynchronizedFutureMap<InodeId, std::optional<Inode>> inodes_;
  SynchronizedFutureMap<std::pair<InodeId, std::string>, std::optional<DirEntry>> entries_;
};

}  // namespace hf3fs::meta::server

#pragma once

#include <rocksdb/options.h>
#include <rocksdb/table.h>
#include <string>
#include <string_view>

#include "common/serde/Serde.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/Duration.h"
#include "common/utils/Path.h"
#include "common/utils/Result.h"
#include "common/utils/Size.h"

namespace hf3fs::kv {

class KVStore {
 public:
  enum class Type { LevelDB, RocksDB, MemDB };
  class Config : public ConfigBase<Config> {
    CONFIG_ITEM(type, Type::LevelDB);
    CONFIG_ITEM(create_if_missing, false);
    CONFIG_HOT_UPDATED_ITEM(sync_when_write, true);

    // for leveldb.
    CONFIG_ITEM(leveldb_sst_file_size, 16_MB, ConfigCheckers::checkGE<size_t, 4_MB>);
    CONFIG_ITEM(leveldb_write_buffer_size, 16_MB, ConfigCheckers::checkGE<size_t, 4_MB>);
    CONFIG_ITEM(leveldb_block_cache_size, 8_GB, ConfigCheckers::checkGE<size_t, 4_MB>);
    CONFIG_ITEM(leveldb_shared_block_cache, true);
    CONFIG_HOT_UPDATED_ITEM(leveldb_iterator_fill_cache, true);
    CONFIG_ITEM(integrate_leveldb_log, false);

    // for rocksdb
    CONFIG_ITEM(rocksdb_max_manifest_file_size, 64_MB, ConfigCheckers::checkGE<size_t, 4_MB>);
    CONFIG_ITEM(rocksdb_stats_dump_period, 2_min);
    CONFIG_ITEM(rocksdb_enable_pipelined_write, false);
    CONFIG_ITEM(rocksdb_unordered_write, false);
    CONFIG_ITEM(rocksdb_avoid_flush_during_recovery, false);
    CONFIG_ITEM(rocksdb_avoid_flush_during_shutdown, false);
    CONFIG_ITEM(rocksdb_avoid_unnecessary_blocking_io, false);
    CONFIG_ITEM(rocksdb_lowest_used_cache_tier, rocksdb_internal::CacheTier::kNonVolatileBlockTier);
    CONFIG_ITEM(rocksdb_write_buffer_size, 16_MB, ConfigCheckers::checkGE<size_t, 4_MB>);
    CONFIG_ITEM(rocksdb_compression, rocksdb_internal::CompressionType::kNoCompression);
    CONFIG_ITEM(rocksdb_level0_file_num_compaction_trigger, 4, ConfigCheckers::checkGE<int, 2>);
    CONFIG_ITEM(rocksdb_enable_prefix_transform, true);
    CONFIG_ITEM(rocksdb_enable_bloom_filter, true);
    CONFIG_ITEM(rocksdb_bloom_filter_bits_per_key, 10);
    CONFIG_ITEM(rocksdb_num_levels, 7, ConfigCheckers::checkGE<int, 3>);
    CONFIG_ITEM(rocksdb_target_file_size_base, 64_MB, ConfigCheckers::checkGE<size_t, 4_MB>);
    CONFIG_ITEM(rocksdb_target_file_size_multiplier, 1, ConfigCheckers::checkPositive);
    CONFIG_ITEM(rocksdb_block_cache_size, 8_GB, ConfigCheckers::checkGE<size_t, 4_MB>);
    CONFIG_ITEM(rocksdb_shared_block_cache, true);
    CONFIG_ITEM(rocksdb_block_size, 4_KB, ConfigCheckers::checkGE<size_t, 4_KB>);
    CONFIG_ITEM(rocksdb_prepopulate_block_cache,
                rocksdb_internal::BlockBasedTableOptions::PrepopulateBlockCache::kDisable);
    CONFIG_ITEM(rocksdb_threads_num, 8u, ConfigCheckers::checkPositive);
    CONFIG_ITEM(rocksdb_wal_recovery_mode, rocksdb_internal::WALRecoveryMode::kTolerateCorruptedTailRecords);
    CONFIG_ITEM(rocksdb_keep_log_file_num, 10u);
    CONFIG_HOT_UPDATED_ITEM(rocksdb_readahead_size, 2_MB);
  };

  struct Options {
    SERDE_STRUCT_FIELD(type, Type::RocksDB);
    SERDE_STRUCT_FIELD(path, Path{});
    SERDE_STRUCT_FIELD(createIfMissing, false);
  };

  virtual ~KVStore() = default;

  // get value corresponding to key.
  virtual Result<std::string> get(std::string_view key) = 0;

  // get the first key which is greater than input key.
  using IterateFunc = std::function<Result<Void>(std::string_view, std::string_view)>;
  virtual Result<Void> iterateKeysWithPrefix(std::string_view prefix,
                                             uint32_t limit,
                                             IterateFunc func,
                                             std::optional<std::string> *nextValidKey = nullptr) = 0;

  // put a key-value pair.
  virtual Result<Void> put(std::string_view key, std::string_view value, bool sync = false) = 0;

  // remove a key-value pair.
  virtual Result<Void> remove(std::string_view key) = 0;

  // batch operations.
  class BatchOperations {
   public:
    virtual ~BatchOperations() = default;
    // put a key-value pair.
    virtual void put(std::string_view key, std::string_view value) = 0;
    // remove a key.
    virtual void remove(std::string_view key) = 0;
    // clear a batch operations.
    virtual void clear() = 0;
    // commit a batch of operations.
    virtual Result<Void> commit() = 0;
    // destroy self.
    virtual void destroy() = 0;
    // deleter for std::unique_ptr.
    struct Deleter {
      void operator()(BatchOperations *b) { b->destroy(); }
    };
  };
  using BatchOptionsPtr = std::unique_ptr<BatchOperations, BatchOperations::Deleter>;
  virtual BatchOptionsPtr createBatchOps() = 0;

  // iterator.
  class Iterator {
   public:
    virtual ~Iterator() = default;
    virtual void seek(std::string_view key) = 0;
    virtual void seekToFirst() = 0;
    virtual void seekToLast() = 0;
    virtual void next() = 0;
    virtual Result<Void> status() const = 0;
    virtual bool valid() const = 0;
    virtual std::string_view key() const = 0;
    virtual std::string_view value() const = 0;
    virtual void destroy() = 0;
    struct Deleter {
      void operator()(Iterator *it) { it->destroy(); }
    };
  };
  using IteratorPtr = std::unique_ptr<Iterator, Iterator::Deleter>;
  virtual IteratorPtr createIterator() = 0;

  // create a KVStore instance.
  static std::unique_ptr<KVStore> create(const Config &config, const Options &options);
};

}  // namespace hf3fs::kv

#pragma once

#include <folly/concurrency/AtomicSharedPtr.h>
#include <optional>

#include "common/net/IOWorker.h"
#include "common/net/Processor.h"
#include "common/net/RDMAControl.h"
#include "common/net/ThreadPoolGroup.h"
#include "common/serde/ClientContext.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/Result.h"

namespace hf3fs::net {

class Client {
 public:
  struct Config : public ConfigBase<Config> {
    CONFIG_OBJ(thread_pool, ThreadPoolGroup::Config);
    CONFIG_OBJ(processor, Processor::Config);
    CONFIG_OBJ(io_worker, IOWorker::Config);
    CONFIG_OBJ(rdma_control, RDMAControlImpl::Config);
    CONFIG_HOT_UPDATED_ITEM(default_timeout, kClientRequestDefaultTimeout);
    CONFIG_HOT_UPDATED_ITEM(default_log_long_running_threshold, kClientRequestLogLongRunningThreshold);
    CONFIG_HOT_UPDATED_ITEM(default_send_retry_times, kDefaultMaxRetryTimes, ConfigCheckers::checkPositive);
    CONFIG_HOT_UPDATED_ITEM(default_compression_level, 0u);
    CONFIG_HOT_UPDATED_ITEM(default_compression_threshold, 128_KB);
    CONFIG_HOT_UPDATED_ITEM(enable_rdma_control, false);
    CONFIG_HOT_UPDATED_ITEM(force_use_tcp, false);
    CONFIG_HOT_UPDATED_ITEM(default_report_metrics, false);
  };

  explicit Client(const Config &config, const std::string &name = "Cli")
      : config_(config),
        tpg_(name, config_.thread_pool()),
        processor_(serdeServices_, tpg_.procThreadPool(), config_.processor()),
        ioWorker_(processor_, tpg_.ioThreadPool(), tpg_.connThreadPool(), config_.io_worker()),
        clientConfigGuard_(config_.addCallbackGuard([&] { updateDefaultOptions(); })) {
    updateDefaultOptions();
  }
  ~Client() { stopAndJoin(); }

  Result<Void> start(const std::string &name = "Cli") {
    RETURN_AND_LOG_ON_ERROR(processor_.start(name));
    RETURN_AND_LOG_ON_ERROR(serdeServices_.addService(std::make_unique<RDMAControlImpl>(config_.rdma_control()), true));
    return ioWorker_.start(name);
  }

  void stopAndJoin() {
    processor_.stopAndJoin();
    ioWorker_.stopAndJoin();
    tpg_.stopAndJoin();
  }

  serde::ClientContext serdeCtx(Address serverAddr) {
    return serde::ClientContext(ioWorker_, config_.force_use_tcp() ? serverAddr.tcp() : serverAddr, options_);
  }

  auto &tpg() { return tpg_; }

  auto options() const { return options_.load(); }

  void dropConnections(Address addr) { ioWorker_.dropConnections(addr); }

  void checkConnections(Address addr, Duration expiredTime) { ioWorker_.checkConnections(addr, expiredTime); }

 protected:
  void updateDefaultOptions() {
    auto options = std::make_shared<CoreRequestOptions>();
    options->timeout = config_.default_timeout();
    options->logLongRunningThreshold = config_.default_log_long_running_threshold();
    options->sendRetryTimes = config_.default_send_retry_times();
    options->compression = {config_.default_compression_level(), config_.default_compression_threshold()};
    options->enableRDMAControl = config_.enable_rdma_control();
    options->reportMetrics = config_.default_report_metrics();
    options_ = std::move(options);
  }

 private:
  serde::Services serdeServices_;
  const Config &config_;

  ThreadPoolGroup tpg_;
  Processor processor_;
  IOWorker ioWorker_;
  std::unique_ptr<ConfigCallbackGuard> clientConfigGuard_;
  folly::atomic_shared_ptr<const CoreRequestOptions> options_{std::make_shared<CoreRequestOptions>()};
};

}  // namespace hf3fs::net

#pragma once

#include <folly/logging/xlog.h>
#include <numeric>
#include <parquet/exception.h>
#include <parquet/schema.h>

#include "SerdeStructVisitor.h"
#include "common/serde/Serde.h"
#include "common/utils/Result.h"
#include "common/utils/TypeTraits.h"

namespace hf3fs::analytics {

template <serde::SerdeType SerdeType>
class SerdeSchemaBuilder : public BaseStructVisitor<SerdeSchemaBuilder<SerdeType>> {
 public:
  std::shared_ptr<parquet::schema::GroupNode> getSchema() {
    try {
      fields_.clear();
      fieldNameParts_.clear();
      this->visit<SerdeType>("");
      return std::static_pointer_cast<parquet::schema::GroupNode>(
          parquet::schema::GroupNode::Make("schema", parquet::Repetition::REQUIRED, fields_));
    } catch (const parquet::ParquetException &ex) {
      XLOGF(CRITICAL, "Failed to build schema of type {}, error: {}", nameof::nameof_full_type<SerdeType>(), ex.what());
      return nullptr;
    }
  }

 public:
  // default
  template <typename T>
  void visit(std::string_view k) = delete;

  template <>
  void visit<bool>(std::string_view k) {
    XLOGF(DBG3, "bool visit({}), fullname: '{}'", k, getFieldFullName(k));
    fields_.push_back(parquet::schema::PrimitiveNode::Make(getFieldFullName(k),
                                                           parquet::Repetition::REQUIRED,
                                                           parquet::LogicalType::None(),
                                                           parquet::Type::BOOLEAN));
  }

  template <>
  void visit<int16_t>(std::string_view k) {
    XLOGF(DBG3, "int16_t visit({}), fullname: '{}'", k, getFieldFullName(k));
    fields_.push_back(parquet::schema::PrimitiveNode::Make(getFieldFullName(k),
                                                           parquet::Repetition::REQUIRED,
                                                           parquet::LogicalType::Int(16, true),
                                                           parquet::Type::INT32));
  }

  template <>
  void visit<uint16_t>(std::string_view k) {
    XLOGF(DBG3, "uint16_t visit({}), fullname: '{}'", k, getFieldFullName(k));
    fields_.push_back(parquet::schema::PrimitiveNode::Make(getFieldFullName(k),
                                                           parquet::Repetition::REQUIRED,
                                                           parquet::LogicalType::Int(16, false),
                                                           parquet::Type::INT32));
  }

  template <>
  void visit<int32_t>(std::string_view k) {
    XLOGF(DBG3, "int32_t visit({}), fullname: '{}'", k, getFieldFullName(k));
    fields_.push_back(parquet::schema::PrimitiveNode::Make(getFieldFullName(k),
                                                           parquet::Repetition::REQUIRED,
                                                           parquet::LogicalType::Int(32, true),
                                                           parquet::Type::INT32));
  }

  template <>
  void visit<uint32_t>(std::string_view k) {
    XLOGF(DBG3, "uint32_t visit({}), fullname: '{}'", k, getFieldFullName(k));
    fields_.push_back(parquet::schema::PrimitiveNode::Make(getFieldFullName(k),
                                                           parquet::Repetition::REQUIRED,
                                                           parquet::LogicalType::Int(32, false),
                                                           parquet::Type::INT32));
  }

  template <>
  void visit<int64_t>(std::string_view k) {
    XLOGF(DBG3, "int64_t visit({}), fullname: '{}'", k, getFieldFullName(k));
    fields_.push_back(parquet::schema::PrimitiveNode::Make(getFieldFullName(k),
                                                           parquet::Repetition::REQUIRED,
                                                           parquet::LogicalType::Int(64, true),
                                                           parquet::Type::INT64));
  }

  template <>
  void visit<uint64_t>(std::string_view k) {
    XLOGF(DBG3, "uint64_t visit({}), fullname: '{}'", k, getFieldFullName(k));
    fields_.push_back(parquet::schema::PrimitiveNode::Make(getFieldFullName(k),
                                                           parquet::Repetition::REQUIRED,
                                                           parquet::LogicalType::Int(64, false),
                                                           parquet::Type::INT64));
  }

  template <typename T>
  requires std::is_enum_v<T>
  void visit(std::string_view k) {
    XLOGF(DBG3, "enum visit({}), fullname: '{}'", k, getFieldFullName(k));
    fields_.push_back(parquet::schema::PrimitiveNode::Make(getFieldFullName(k),
                                                           parquet::Repetition::REQUIRED,
                                                           parquet::LogicalType::Int(32, true),
                                                           parquet::Type::INT32));
  }

  template <serde::ConvertibleToString T>
  void visit(std::string_view k) {
    XLOGF(DBG3, "string visit({}), fullname: '{}'", k, getFieldFullName(k));
    fields_.push_back(parquet::schema::PrimitiveNode::Make(getFieldFullName(k),
                                                           parquet::Repetition::REQUIRED,
                                                           parquet::LogicalType::String(),
                                                           parquet::Type::BYTE_ARRAY));
  }

  template <StrongTyped T>
  void visit(std::string_view k) {
    XLOGF(DBG3, "strongtyped visit({}), fullname: '{}'", k, getFieldFullName(k));
    BaseStructVisitor<SerdeSchemaBuilder>::template visit<T>(k);
  }

  template <serde::WithReadableSerdeMethod T>
  void visit(std::string_view k) {
    XLOGF(DBG3, "WithReadableSerdeMethod visit({})", k);
    visit<serde::SerdeToReadableReturnType<T>>(k);
  }

  template <serde::WithSerdeMethod T>
  void visit(std::string_view k) {
    XLOGF(DBG3, "WithSerdeMethod visit({})", k);
    visit<serde::SerdeToReturnType<T>>(k);
  }

  template <serde::WithReadableSerdeMemberMethod T>
  void visit(std::string_view k) {
    XLOGF(DBG3, "WithReadableSerdeMemberMethod visit({})", k);
    visit<serde::SerdeToReadableMemberMethodReturnType<T>>(k);
  }

  template <serde::WithSerdeMemberMethod T>
  void visit(std::string_view k) {
    XLOGF(DBG3, "WithSerdeMemberMethod visit({})", k);
    visit<serde::SerdeToMemberMethodReturnType<T>>(k);
  }

  template <serde::SerdeTypeWithoutSpecializedSerdeMethod T>
  void visit(std::string_view k) {
    XLOGF(DBG3, "serdetype visit({}), fullname: '{}'", k, getFieldFullName(k));
    if (!k.empty()) fieldNameParts_.push_back(filterOutInvalidChars(k));
    BaseStructVisitor<SerdeSchemaBuilder>::template visit<T>(k);
    if (!k.empty()) fieldNameParts_.pop_back();
  }

  template <typename T>
  requires is_specialization_of_v<T, folly::Expected>
  void visit(std::string_view k) {
    XLOGF(DBG3, "result visit({}), fullname: '{}'", k, getFieldFullName(k));
    std::string errorColumnName = std::string{k} + kResultErrorTypeColumnSuffix;
    visit<typename T::error_type>(errorColumnName);
    visit<typename T::value_type>(k);
  }

  template <typename T>
  requires is_variant_v<T>
  void visit(std::string_view k) {
    XLOGF(DBG3, "variant visit({}), fullname: '{}'", k, getFieldFullName(k));
    std::string valIdxColumnName = std::string{k} + kVariantValueIndexColumnSuffix;
    visit<uint32_t>(valIdxColumnName);
    BaseStructVisitor<SerdeSchemaBuilder>::template visit<T>(k);
  }

  template <typename T>
  requires is_vector_v<T> || is_set_v<T>
  void visit(std::string_view k) {
    XLOGF(DBG3, "container visit({}), fullname: '{}'", k, getFieldFullName(k));
    fields_.push_back(parquet::schema::PrimitiveNode::Make(getFieldFullName(k),
                                                           parquet::Repetition::REQUIRED,
                                                           parquet::LogicalType::String(),
                                                           parquet::Type::BYTE_ARRAY));
  }

  template <typename T>
  requires is_optional_v<T>
  void visit(std::string_view k) {
    XLOGF(DBG3, "container visit({}), fullname: '{}'", k, getFieldFullName(k));
    fields_.push_back(parquet::schema::PrimitiveNode::Make(getFieldFullName(k),
                                                           parquet::Repetition::REQUIRED,
                                                           parquet::LogicalType::String(),
                                                           parquet::Type::BYTE_ARRAY));
  }

 private:
  std::string getFieldFullName(std::string_view k) {
    fieldNameParts_.push_back(filterOutInvalidChars(k));
    auto fieldFullName = std::accumulate(fieldNameParts_.begin(),
                                         fieldNameParts_.end(),
                                         std::string{},
                                         [](const std::string &a, const std::string &b) {
                                           return a + (a.empty() ? std::string{} : std::string("_")) + b;
                                         });
    fieldNameParts_.pop_back();
    return fieldFullName;
  }

  std::string filterOutInvalidChars(std::string_view k) {
    return std::accumulate(k.begin(), k.end(), std::string{}, [](const std::string &a, const char b) {
      if (('a' <= b && b <= 'z') || ('A' <= b && b <= 'Z') || ('0' <= b && b <= '9')) return a + b;
      return a;
    });
  };

 private:
  parquet::schema::NodeVector fields_;
  std::vector<std::string> fieldNameParts_;
};

}  // namespace hf3fs::analytics

#pragma once

#include <utility>

#include "common/serde/Serde.h"
#include "common/utils/Duration.h"
#include "common/utils/Size.h"

namespace hf3fs::net {

constexpr Duration kClientRequestDefaultTimeout = 1000_ms;
constexpr Duration kClientRequestLogLongRunningThreshold = 0_ms;
constexpr uint32_t kDefaultMaxRetryTimes = 1;

struct CompressionOptions {
  operator bool() const noexcept { return level; }
  bool enable(size_t size) const noexcept { return level && size >= threshold; }
  SERDE_STRUCT_FIELD(level, 0u);
  SERDE_STRUCT_FIELD(threshold, 0_B);
};

struct UserRequestOptions {
  SERDE_STRUCT_FIELD(timeout, std::optional<Duration>{});
  SERDE_STRUCT_FIELD(sendRetryTimes, std::optional<uint32_t>{});
  SERDE_STRUCT_FIELD(compression, std::optional<CompressionOptions>{});
  SERDE_STRUCT_FIELD(logLongRunningThreshold, std::optional<Duration>{});
  SERDE_STRUCT_FIELD(reportMetrics, std::optional<bool>{});
};

struct CoreRequestOptions {
  void merge(const UserRequestOptions &o) {
    if (o.timeout) {
      timeout = o.timeout.value();
    }
    if (o.sendRetryTimes) {
      sendRetryTimes = o.sendRetryTimes.value();
    }
    if (o.compression) {
      compression = o.compression.value();
    }
    if (o.logLongRunningThreshold) {
      logLongRunningThreshold = o.logLongRunningThreshold.value();
    }
    if (o.reportMetrics) {
      reportMetrics = o.reportMetrics.value();
    }
  }

  SERDE_STRUCT_FIELD(timeout, kClientRequestDefaultTimeout);
  SERDE_STRUCT_FIELD(sendRetryTimes, kDefaultMaxRetryTimes);
  SERDE_STRUCT_FIELD(compression, CompressionOptions{});
  SERDE_STRUCT_FIELD(enableRDMAControl, false);
  SERDE_STRUCT_FIELD(logLongRunningThreshold, kClientRequestLogLongRunningThreshold);
  SERDE_STRUCT_FIELD(reportMetrics, false);
};

}  // namespace hf3fs::net

#pragma once

#include <type_traits>
#include <utility>
#include <variant>

#include "common/serde/Serde.h"
#include "common/utils/Nameof.hpp"

namespace hf3fs::serde {

template <class T>
class VisitOut {
 public:
  auto tableBegin();               // begin a table.
  void tableEnd();                 // end a table.
  void arrayBegin();               // begin an array.
  void arrayEnd();                 // end an array.
  void variantBegin();             // begin a variant.
  void variantEnd();               // end a variant.
  void key(std::string_view key);  // prepare the key in a key-value pair.
  void value(auto &&value);        // prepare the value in a key-value pair or an array.
};

template <size_t I = 0>
inline void visitVariant(auto &&t, auto &&func) {
  using T = std::decay_t<decltype(t)>;
  using S = std::variant_alternative_t<I, T>;
  if (t.index() == I) {
    func(nameof::nameof_type<S>(), std::get<I>(t));
  } else {
    func(nameof::nameof_type<S>(), std::variant_alternative_t<I, T>{});
  }
  if constexpr (I + 1 < std::variant_size_v<T>) {
    visitVariant<I + 1>(t, func);
  }
}

template <class O>
inline void visit(auto &&t, VisitOut<O> &out) {
  using T = std::decay_t<decltype(t)>;
  if constexpr (SerdeType<T>) {
    out.tableBegin();
    refl::Helper::iterate<T>([&](auto type) {
      out.key(type.name);
      visit(t.*type.getter, out);
    });
    out.tableEnd();
  } else if constexpr (std::is_same_v<T, std::string> || std::is_convertible_v<T, std::string_view>) {
    out.value(std::string_view(t));
  } else if constexpr (is_variant_v<T>) {
    out.variantBegin();
    visitVariant(t, [&](std::string_view name, auto &&t) {
      out.key(name);
      visit(t, out);
    });
    out.variantEnd();
  } else if constexpr (is_vector_v<T> || is_set_v<T>) {
    out.arrayBegin();
    for (auto item : t) {
      visit(item, out);
    }
    out.arrayEnd();
  } else {
    out.value(t);
  }
}

}  // namespace hf3fs::serde

/* The latest version of this library is available on GitHub;
 * https://github.com/sheredom/utf8.h */

/* This is free and unencumbered software released into the public domain.
 *
 * Anyone is free to copy, modify, publish, use, compile, sell, or
 * distribute this software, either in source code form or as a compiled
 * binary, for any purpose, commercial or non-commercial, and by any
 * means.
 *
 * In jurisdictions that recognize copyright laws, the author or authors
 * of this software dedicate any and all copyright interest in the
 * software to the public domain. We make this dedication for the benefit
 * of the public at large and to the detriment of our heirs and
 * successors. We intend this dedication to be an overt act of
 * relinquishment in perpetuity of all present and future rights to this
 * software under copyright law.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 * For more information, please refer to <http://unlicense.org/> */

#ifndef SHEREDOM_UTF8_H_INCLUDED
#define SHEREDOM_UTF8_H_INCLUDED

#if defined(_MSC_VER)
#pragma warning(push)

/* disable warning: no function prototype given: converting '()' to '(void)' */
#pragma warning(disable : 4255)

/* disable warning: '__cplusplus' is not defined as a preprocessor macro,
 * replacing with '0' for '#if/#elif' */
#pragma warning(disable : 4668)

/* disable warning: bytes padding added after construct */
#pragma warning(disable : 4820)
#endif

#include <stddef.h>
#include <stdlib.h>

#if defined(_MSC_VER)
#pragma warning(pop)
#endif

#if defined(_MSC_VER) && (_MSC_VER < 1920)
typedef __int32 utf8_int32_t;
#else
#include <stdint.h>
typedef int32_t utf8_int32_t;
#endif

#if defined(__clang__)
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wold-style-cast"
#pragma clang diagnostic ignored "-Wcast-qual"
#endif

#ifdef __cplusplus
extern "C" {
#endif

#if defined(_MSC_VER)
#define utf8_nonnull
#define utf8_pure
#define utf8_restrict __restrict
#define utf8_weak __inline
#elif defined(__clang__) || defined(__GNUC__)
#define utf8_nonnull __attribute__((nonnull))
#define utf8_pure __attribute__((pure))
#define utf8_restrict __restrict__
#define utf8_weak __attribute__((weak))
#else
#error Non clang, non gcc, non MSVC compiler found!
#endif

#ifdef __cplusplus
#define utf8_null NULL
#else
#define utf8_null 0
#endif

#if (defined(__cplusplus) && __cplusplus >= 201402L)
#define utf8_constexpr14 constexpr
#define utf8_constexpr14_impl constexpr
#else
/* constexpr and weak are incompatible. so only enable one of them */
#define utf8_constexpr14 utf8_weak
#define utf8_constexpr14_impl
#endif

#if defined(__cplusplus) && __cplusplus >= 202002L
using utf8_int8_t = char8_t; /* Introduced in C++20 */
#else
typedef char utf8_int8_t;
#endif

/* Return less than 0, 0, greater than 0 if src1 < src2, src1 == src2, src1 >
 * src2 respectively, case insensitive. */
utf8_constexpr14 utf8_nonnull utf8_pure int utf8casecmp(const utf8_int8_t *src1, const utf8_int8_t *src2);

/* Append the utf8 string src onto the utf8 string dst. */
utf8_nonnull utf8_weak utf8_int8_t *utf8cat(utf8_int8_t *utf8_restrict dst, const utf8_int8_t *utf8_restrict src);

/* Find the first match of the utf8 codepoint chr in the utf8 string src. */
utf8_constexpr14 utf8_nonnull utf8_pure utf8_int8_t *utf8chr(const utf8_int8_t *src, utf8_int32_t chr);

/* Return less than 0, 0, greater than 0 if src1 < src2,
 * src1 == src2, src1 > src2 respectively. */
utf8_constexpr14 utf8_nonnull utf8_pure int utf8cmp(const utf8_int8_t *src1, const utf8_int8_t *src2);

/* Copy the utf8 string src onto the memory allocated in dst. */
utf8_nonnull utf8_weak utf8_int8_t *utf8cpy(utf8_int8_t *utf8_restrict dst, const utf8_int8_t *utf8_restrict src);

/* Number of utf8 codepoints in the utf8 string src that consists entirely
 * of utf8 codepoints not from the utf8 string reject. */
utf8_constexpr14 utf8_nonnull utf8_pure size_t utf8cspn(const utf8_int8_t *src, const utf8_int8_t *reject);

/* Duplicate the utf8 string src by getting its size, malloc'ing a new buffer
 * copying over the data, and returning that. Or 0 if malloc failed. */
utf8_weak utf8_int8_t *utf8dup(const utf8_int8_t *src);

/* Number of utf8 codepoints in the utf8 string str,
 * excluding the null terminating byte. */
utf8_constexpr14 utf8_nonnull utf8_pure size_t utf8len(const utf8_int8_t *str);

/* Similar to utf8len, except that only at most n bytes of src are looked. */
utf8_constexpr14 utf8_nonnull utf8_pure size_t utf8nlen(const utf8_int8_t *str, size_t n);

/* Return less than 0, 0, greater than 0 if src1 < src2, src1 == src2, src1 >
 * src2 respectively, case insensitive. Checking at most n bytes of each utf8
 * string. */
utf8_constexpr14 utf8_nonnull utf8_pure int utf8ncasecmp(const utf8_int8_t *src1, const utf8_int8_t *src2, size_t n);

/* Append the utf8 string src onto the utf8 string dst,
 * writing at most n+1 bytes. Can produce an invalid utf8
 * string if n falls partway through a utf8 codepoint. */
utf8_nonnull utf8_weak utf8_int8_t *utf8ncat(utf8_int8_t *utf8_restrict dst,
                                             const utf8_int8_t *utf8_restrict src,
                                             size_t n);

/* Return less than 0, 0, greater than 0 if src1 < src2,
 * src1 == src2, src1 > src2 respectively. Checking at most n
 * bytes of each utf8 string. */
utf8_constexpr14 utf8_nonnull utf8_pure int utf8ncmp(const utf8_int8_t *src1, const utf8_int8_t *src2, size_t n);

/* Copy the utf8 string src onto the memory allocated in dst.
 * Copies at most n bytes. If n falls partway through a utf8
 * codepoint, or if dst doesn't have enough room for a null
 * terminator, the final string will be cut short to preserve
 * utf8 validity. */

utf8_nonnull utf8_weak utf8_int8_t *utf8ncpy(utf8_int8_t *utf8_restrict dst,
                                             const utf8_int8_t *utf8_restrict src,
                                             size_t n);

/* Similar to utf8dup, except that at most n bytes of src are copied. If src is
 * longer than n, only n bytes are copied and a null byte is added.
 *
 * Returns a new string if successful, 0 otherwise */
utf8_weak utf8_int8_t *utf8ndup(const utf8_int8_t *src, size_t n);

/* Locates the first occurrence in the utf8 string str of any byte in the
 * utf8 string accept, or 0 if no match was found. */
utf8_constexpr14 utf8_nonnull utf8_pure utf8_int8_t *utf8pbrk(const utf8_int8_t *str, const utf8_int8_t *accept);

/* Find the last match of the utf8 codepoint chr in the utf8 string src. */
utf8_constexpr14 utf8_nonnull utf8_pure utf8_int8_t *utf8rchr(const utf8_int8_t *src, int chr);

/* Number of bytes in the utf8 string str,
 * including the null terminating byte. */
utf8_constexpr14 utf8_nonnull utf8_pure size_t utf8size(const utf8_int8_t *str);

/* Similar to utf8size, except that the null terminating byte is excluded. */
utf8_constexpr14 utf8_nonnull utf8_pure size_t utf8size_lazy(const utf8_int8_t *str);

/* Similar to utf8size, except that only at most n bytes of src are looked and
 * the null terminating byte is excluded. */
utf8_constexpr14 utf8_nonnull utf8_pure size_t utf8nsize_lazy(const utf8_int8_t *str, size_t n);

/* Number of utf8 codepoints in the utf8 string src that consists entirely
 * of utf8 codepoints from the utf8 string accept. */
utf8_constexpr14 utf8_nonnull utf8_pure size_t utf8spn(const utf8_int8_t *src, const utf8_int8_t *accept);

/* The position of the utf8 string needle in the utf8 string haystack. */
utf8_constexpr14 utf8_nonnull utf8_pure utf8_int8_t *utf8str(const utf8_int8_t *haystack, const utf8_int8_t *needle);

/* The position of the utf8 string needle in the utf8 string haystack, case
 * insensitive. */
utf8_constexpr14 utf8_nonnull utf8_pure utf8_int8_t *utf8casestr(const utf8_int8_t *haystack,
                                                                 const utf8_int8_t *needle);

/* Return 0 on success, or the position of the invalid
 * utf8 codepoint on failure. */
utf8_constexpr14 utf8_nonnull utf8_pure utf8_int8_t *utf8valid(const utf8_int8_t *str);

/* Similar to utf8valid, except that only at most n bytes of src are looked. */
utf8_constexpr14 utf8_nonnull utf8_pure utf8_int8_t *utf8nvalid(const utf8_int8_t *str, size_t n);

/* Given a null-terminated string, makes the string valid by replacing invalid
 * codepoints with a 1-byte replacement. Returns 0 on success. */
utf8_nonnull utf8_weak int utf8makevalid(utf8_int8_t *str, const utf8_int32_t replacement);

/* Sets out_codepoint to the current utf8 codepoint in str, and returns the
 * address of the next utf8 codepoint after the current one in str. */
utf8_constexpr14 utf8_nonnull utf8_int8_t *utf8codepoint(const utf8_int8_t *utf8_restrict str,
                                                         utf8_int32_t *utf8_restrict out_codepoint);

/* Calculates the size of the next utf8 codepoint in str. */
utf8_constexpr14 utf8_nonnull size_t utf8codepointcalcsize(const utf8_int8_t *str);

/* Returns the size of the given codepoint in bytes. */
utf8_constexpr14 size_t utf8codepointsize(utf8_int32_t chr);

/* Write a codepoint to the given string, and return the address to the next
 * place after the written codepoint. Pass how many bytes left in the buffer to
 * n. If there is not enough space for the codepoint, this function returns
 * null. */
utf8_nonnull utf8_weak utf8_int8_t *utf8catcodepoint(utf8_int8_t *str, utf8_int32_t chr, size_t n);

/* Returns 1 if the given character is lowercase, or 0 if it is not. */
utf8_constexpr14 int utf8islower(utf8_int32_t chr);

/* Returns 1 if the given character is uppercase, or 0 if it is not. */
utf8_constexpr14 int utf8isupper(utf8_int32_t chr);

/* Transform the given string into all lowercase codepoints. */
utf8_nonnull utf8_weak void utf8lwr(utf8_int8_t *utf8_restrict str);

/* Transform the given string into all uppercase codepoints. */
utf8_nonnull utf8_weak void utf8upr(utf8_int8_t *utf8_restrict str);

/* Make a codepoint lower case if possible. */
utf8_constexpr14 utf8_int32_t utf8lwrcodepoint(utf8_int32_t cp);

/* Make a codepoint upper case if possible. */
utf8_constexpr14 utf8_int32_t utf8uprcodepoint(utf8_int32_t cp);

/* Sets out_codepoint to the current utf8 codepoint in str, and returns the
 * address of the previous utf8 codepoint before the current one in str. */
utf8_constexpr14 utf8_nonnull utf8_int8_t *utf8rcodepoint(const utf8_int8_t *utf8_restrict str,
                                                          utf8_int32_t *utf8_restrict out_codepoint);

/* Duplicate the utf8 string src by getting its size, calling alloc_func_ptr to
 * copy over data to a new buffer, and returning that. Or 0 if alloc_func_ptr
 * returned null. */
utf8_weak utf8_int8_t *utf8dup_ex(const utf8_int8_t *src,
                                  utf8_int8_t *(*alloc_func_ptr)(utf8_int8_t *, size_t),
                                  utf8_int8_t *user_data);

/* Similar to utf8dup, except that at most n bytes of src are copied. If src is
 * longer than n, only n bytes are copied and a null byte is added.
 *
 * Returns a new string if successful, 0 otherwise. */
utf8_weak utf8_int8_t *utf8ndup_ex(const utf8_int8_t *src,
                                   size_t n,
                                   utf8_int8_t *(*alloc_func_ptr)(utf8_int8_t *, size_t),
                                   utf8_int8_t *user_data);

#undef utf8_weak
#undef utf8_pure
#undef utf8_nonnull

utf8_constexpr14_impl int utf8casecmp(const utf8_int8_t *src1, const utf8_int8_t *src2) {
  utf8_int32_t src1_lwr_cp = 0, src2_lwr_cp = 0, src1_upr_cp = 0, src2_upr_cp = 0, src1_orig_cp = 0, src2_orig_cp = 0;

  for (;;) {
    src1 = utf8codepoint(src1, &src1_orig_cp);
    src2 = utf8codepoint(src2, &src2_orig_cp);

    /* lower the srcs if required */
    src1_lwr_cp = utf8lwrcodepoint(src1_orig_cp);
    src2_lwr_cp = utf8lwrcodepoint(src2_orig_cp);

    /* lower the srcs if required */
    src1_upr_cp = utf8uprcodepoint(src1_orig_cp);
    src2_upr_cp = utf8uprcodepoint(src2_orig_cp);

    /* check if the lowered codepoints match */
    if ((0 == src1_orig_cp) && (0 == src2_orig_cp)) {
      return 0;
    } else if ((src1_lwr_cp == src2_lwr_cp) || (src1_upr_cp == src2_upr_cp)) {
      continue;
    }

    /* if they don't match, then we return the difference between the characters
     */
    return src1_lwr_cp - src2_lwr_cp;
  }
}

utf8_int8_t *utf8cat(utf8_int8_t *utf8_restrict dst, const utf8_int8_t *utf8_restrict src) {
  utf8_int8_t *d = dst;
  /* find the null terminating byte in dst */
  while ('\0' != *d) {
    d++;
  }

  /* overwriting the null terminating byte in dst, append src byte-by-byte */
  while ('\0' != *src) {
    *d++ = *src++;
  }

  /* write out a new null terminating byte into dst */
  *d = '\0';

  return dst;
}

utf8_constexpr14_impl utf8_int8_t *utf8chr(const utf8_int8_t *src, utf8_int32_t chr) {
  utf8_int8_t c[5] = {'\0', '\0', '\0', '\0', '\0'};

  if (0 == chr) {
    /* being asked to return position of null terminating byte, so
     * just run s to the end, and return! */
    while ('\0' != *src) {
      src++;
    }
    return (utf8_int8_t *)src;
  } else if (0 == ((utf8_int32_t)0xffffff80 & chr)) {
    /* 1-byte/7-bit ascii
     * (0b0xxxxxxx) */
    c[0] = (utf8_int8_t)chr;
  } else if (0 == ((utf8_int32_t)0xfffff800 & chr)) {
    /* 2-byte/11-bit utf8 code point
     * (0b110xxxxx 0b10xxxxxx) */
    c[0] = (utf8_int8_t)(0xc0 | (utf8_int8_t)(chr >> 6));
    c[1] = (utf8_int8_t)(0x80 | (utf8_int8_t)(chr & 0x3f));
  } else if (0 == ((utf8_int32_t)0xffff0000 & chr)) {
    /* 3-byte/16-bit utf8 code point
     * (0b1110xxxx 0b10xxxxxx 0b10xxxxxx) */
    c[0] = (utf8_int8_t)(0xe0 | (utf8_int8_t)(chr >> 12));
    c[1] = (utf8_int8_t)(0x80 | (utf8_int8_t)((chr >> 6) & 0x3f));
    c[2] = (utf8_int8_t)(0x80 | (utf8_int8_t)(chr & 0x3f));
  } else { /* if (0 == ((int)0xffe00000 & chr)) { */
    /* 4-byte/21-bit utf8 code point
     * (0b11110xxx 0b10xxxxxx 0b10xxxxxx 0b10xxxxxx) */
    c[0] = (utf8_int8_t)(0xf0 | (utf8_int8_t)(chr >> 18));
    c[1] = (utf8_int8_t)(0x80 | (utf8_int8_t)((chr >> 12) & 0x3f));
    c[2] = (utf8_int8_t)(0x80 | (utf8_int8_t)((chr >> 6) & 0x3f));
    c[3] = (utf8_int8_t)(0x80 | (utf8_int8_t)(chr & 0x3f));
  }

  /* we've made c into a 2 utf8 codepoint string, one for the chr we are
   * seeking, another for the null terminating byte. Now use utf8str to
   * search */
  return utf8str(src, c);
}

utf8_constexpr14_impl int utf8cmp(const utf8_int8_t *src1, const utf8_int8_t *src2) {
  while (('\0' != *src1) || ('\0' != *src2)) {
    if (*src1 < *src2) {
      return -1;
    } else if (*src1 > *src2) {
      return 1;
    }

    src1++;
    src2++;
  }

  /* both utf8 strings matched */
  return 0;
}

utf8_constexpr14_impl int utf8coll(const utf8_int8_t *src1, const utf8_int8_t *src2);

utf8_int8_t *utf8cpy(utf8_int8_t *utf8_restrict dst, const utf8_int8_t *utf8_restrict src) {
  utf8_int8_t *d = dst;

  /* overwriting anything previously in dst, write byte-by-byte
   * from src */
  while ('\0' != *src) {
    *d++ = *src++;
  }

  /* append null terminating byte */
  *d = '\0';

  return dst;
}

utf8_constexpr14_impl size_t utf8cspn(const utf8_int8_t *src, const utf8_int8_t *reject) {
  size_t chars = 0;

  while ('\0' != *src) {
    const utf8_int8_t *r = reject;
    size_t offset = 0;

    while ('\0' != *r) {
      /* checking that if *r is the start of a utf8 codepoint
       * (it is not 0b10xxxxxx) and we have successfully matched
       * a previous character (0 < offset) - we found a match */
      if ((0x80 != (0xc0 & *r)) && (0 < offset)) {
        return chars;
      } else {
        if (*r == src[offset]) {
          /* part of a utf8 codepoint matched, so move our checking
           * onwards to the next byte */
          offset++;
          r++;
        } else {
          /* r could be in the middle of an unmatching utf8 code point,
           * so we need to march it on to the next character beginning, */

          do {
            r++;
          } while (0x80 == (0xc0 & *r));

          /* reset offset too as we found a mismatch */
          offset = 0;
        }
      }
    }

    /* found a match at the end of *r, so didn't get a chance to test it */
    if (0 < offset) {
      return chars;
    }

    /* the current utf8 codepoint in src did not match reject, but src
     * could have been partway through a utf8 codepoint, so we need to
     * march it onto the next utf8 codepoint starting byte */
    do {
      src++;
    } while ((0x80 == (0xc0 & *src)));
    chars++;
  }

  return chars;
}

utf8_int8_t *utf8dup(const utf8_int8_t *src) { return utf8dup_ex(src, utf8_null, utf8_null); }

utf8_int8_t *utf8dup_ex(const utf8_int8_t *src,
                        utf8_int8_t *(*alloc_func_ptr)(utf8_int8_t *, size_t),
                        utf8_int8_t *user_data) {
  utf8_int8_t *n = utf8_null;

  /* figure out how many bytes (including the terminator) we need to copy first
   */
  size_t bytes = utf8size(src);

  if (alloc_func_ptr) {
    n = alloc_func_ptr(user_data, bytes);
  } else {
#if !defined(UTF8_NO_STD_MALLOC)
    n = (utf8_int8_t *)malloc(bytes);
#else
    return utf8_null;
#endif
  }

  if (utf8_null == n) {
    /* out of memory so we bail */
    return utf8_null;
  } else {
    bytes = 0;

    /* copy src byte-by-byte into our new utf8 string */
    while ('\0' != src[bytes]) {
      n[bytes] = src[bytes];
      bytes++;
    }

    /* append null terminating byte */
    n[bytes] = '\0';
    return n;
  }
}

utf8_constexpr14_impl utf8_int8_t *utf8fry(const utf8_int8_t *str);

utf8_constexpr14_impl size_t utf8len(const utf8_int8_t *str) { return utf8nlen(str, SIZE_MAX); }

utf8_constexpr14_impl size_t utf8nlen(const utf8_int8_t *str, size_t n) {
  const utf8_int8_t *t = str;
  size_t length = 0;

  while ((size_t)(str - t) < n && '\0' != *str) {
    if (0xf0 == (0xf8 & *str)) {
      /* 4-byte utf8 code point (began with 0b11110xxx) */
      str += 4;
    } else if (0xe0 == (0xf0 & *str)) {
      /* 3-byte utf8 code point (began with 0b1110xxxx) */
      str += 3;
    } else if (0xc0 == (0xe0 & *str)) {
      /* 2-byte utf8 code point (began with 0b110xxxxx) */
      str += 2;
    } else { /* if (0x00 == (0x80 & *s)) { */
      /* 1-byte ascii (began with 0b0xxxxxxx) */
      str += 1;
    }

    /* no matter the bytes we marched s forward by, it was
     * only 1 utf8 codepoint */
    length++;
  }

  if ((size_t)(str - t) > n) {
    length--;
  }
  return length;
}

utf8_constexpr14_impl int utf8ncasecmp(const utf8_int8_t *src1, const utf8_int8_t *src2, size_t n) {
  utf8_int32_t src1_lwr_cp = 0, src2_lwr_cp = 0, src1_upr_cp = 0, src2_upr_cp = 0, src1_orig_cp = 0, src2_orig_cp = 0;

  do {
    const utf8_int8_t *const s1 = src1;
    const utf8_int8_t *const s2 = src2;

    /* first check that we have enough bytes left in n to contain an entire
     * codepoint */
    if (0 == n) {
      return 0;
    }

    if ((1 == n) && ((0xc0 == (0xe0 & *s1)) || (0xc0 == (0xe0 & *s2)))) {
      const utf8_int32_t c1 = (0xe0 & *s1);
      const utf8_int32_t c2 = (0xe0 & *s2);

      if (c1 < c2) {
        return c1 - c2;
      } else {
        return 0;
      }
    }

    if ((2 >= n) && ((0xe0 == (0xf0 & *s1)) || (0xe0 == (0xf0 & *s2)))) {
      const utf8_int32_t c1 = (0xf0 & *s1);
      const utf8_int32_t c2 = (0xf0 & *s2);

      if (c1 < c2) {
        return c1 - c2;
      } else {
        return 0;
      }
    }

    if ((3 >= n) && ((0xf0 == (0xf8 & *s1)) || (0xf0 == (0xf8 & *s2)))) {
      const utf8_int32_t c1 = (0xf8 & *s1);
      const utf8_int32_t c2 = (0xf8 & *s2);

      if (c1 < c2) {
        return c1 - c2;
      } else {
        return 0;
      }
    }

    src1 = utf8codepoint(src1, &src1_orig_cp);
    src2 = utf8codepoint(src2, &src2_orig_cp);
    n -= utf8codepointsize(src1_orig_cp);

    src1_lwr_cp = utf8lwrcodepoint(src1_orig_cp);
    src2_lwr_cp = utf8lwrcodepoint(src2_orig_cp);

    src1_upr_cp = utf8uprcodepoint(src1_orig_cp);
    src2_upr_cp = utf8uprcodepoint(src2_orig_cp);

    /* check if the lowered codepoints match */
    if ((0 == src1_orig_cp) && (0 == src2_orig_cp)) {
      return 0;
    } else if ((src1_lwr_cp == src2_lwr_cp) || (src1_upr_cp == src2_upr_cp)) {
      continue;
    }

    /* if they don't match, then we return the difference between the characters
     */
    return src1_lwr_cp - src2_lwr_cp;
  } while (0 < n);

  /* both utf8 strings matched */
  return 0;
}

utf8_int8_t *utf8ncat(utf8_int8_t *utf8_restrict dst, const utf8_int8_t *utf8_restrict src, size_t n) {
  utf8_int8_t *d = dst;

  /* find the null terminating byte in dst */
  while ('\0' != *d) {
    d++;
  }

  /* overwriting the null terminating byte in dst, append src byte-by-byte
   * stopping if we run out of space */
  while (('\0' != *src) && (0 != n--)) {
    *d++ = *src++;
  }

  /* write out a new null terminating byte into dst */
  *d = '\0';

  return dst;
}

utf8_constexpr14_impl int utf8ncmp(const utf8_int8_t *src1, const utf8_int8_t *src2, size_t n) {
  while ((0 != n--) && (('\0' != *src1) || ('\0' != *src2))) {
    if (*src1 < *src2) {
      return -1;
    } else if (*src1 > *src2) {
      return 1;
    }

    src1++;
    src2++;
  }

  /* both utf8 strings matched */
  return 0;
}

utf8_int8_t *utf8ncpy(utf8_int8_t *utf8_restrict dst, const utf8_int8_t *utf8_restrict src, size_t n) {
  utf8_int8_t *d = dst;
  size_t index = 0, check_index = 0;

  if (n == 0) {
    return dst;
  }

  /* overwriting anything previously in dst, write byte-by-byte
   * from src */
  for (index = 0; index < n; index++) {
    d[index] = src[index];
    if ('\0' == src[index]) {
      break;
    }
  }

  for (check_index = index - 1; check_index > 0 && 0x80 == (0xc0 & d[check_index]); check_index--) {
    /* just moving the index */
  }

  if (check_index < index &&
      ((index - check_index) < utf8codepointcalcsize(&d[check_index]) || (index - check_index) == n)) {
    index = check_index;
  }

  /* append null terminating byte */
  for (; index < n; index++) {
    d[index] = 0;
  }

  return dst;
}

utf8_int8_t *utf8ndup(const utf8_int8_t *src, size_t n) { return utf8ndup_ex(src, n, utf8_null, utf8_null); }

utf8_int8_t *utf8ndup_ex(const utf8_int8_t *src,
                         size_t n,
                         utf8_int8_t *(*alloc_func_ptr)(utf8_int8_t *, size_t),
                         utf8_int8_t *user_data) {
  utf8_int8_t *c = utf8_null;
  size_t bytes = 0;

  /* Find the end of the string or stop when n is reached */
  while ('\0' != src[bytes] && bytes < n) {
    bytes++;
  }

  /* In case bytes is actually less than n, we need to set it
   * to be used later in the copy byte by byte. */
  n = bytes;

  if (alloc_func_ptr) {
    c = alloc_func_ptr(user_data, bytes + 1);
  } else {
#if !defined(UTF8_NO_STD_MALLOC)
    c = (utf8_int8_t *)malloc(bytes + 1);
#else
    c = utf8_null;
#endif
  }

  if (utf8_null == c) {
    /* out of memory so we bail */
    return utf8_null;
  }

  bytes = 0;

  /* copy src byte-by-byte into our new utf8 string */
  while ('\0' != src[bytes] && bytes < n) {
    c[bytes] = src[bytes];
    bytes++;
  }

  /* append null terminating byte */
  c[bytes] = '\0';
  return c;
}

utf8_constexpr14_impl utf8_int8_t *utf8rchr(const utf8_int8_t *src, int chr) {
  utf8_int8_t *match = utf8_null;
  utf8_int8_t c[5] = {'\0', '\0', '\0', '\0', '\0'};

  if (0 == chr) {
    /* being asked to return position of null terminating byte, so
     * just run s to the end, and return! */
    while ('\0' != *src) {
      src++;
    }
    return (utf8_int8_t *)src;
  } else if (0 == ((int)0xffffff80 & chr)) {
    /* 1-byte/7-bit ascii
     * (0b0xxxxxxx) */
    c[0] = (utf8_int8_t)chr;
  } else if (0 == ((int)0xfffff800 & chr)) {
    /* 2-byte/11-bit utf8 code point
     * (0b110xxxxx 0b10xxxxxx) */
    c[0] = (utf8_int8_t)(0xc0 | (utf8_int8_t)(chr >> 6));
    c[1] = (utf8_int8_t)(0x80 | (utf8_int8_t)(chr & 0x3f));
  } else if (0 == ((int)0xffff0000 & chr)) {
    /* 3-byte/16-bit utf8 code point
     * (0b1110xxxx 0b10xxxxxx 0b10xxxxxx) */
    c[0] = (utf8_int8_t)(0xe0 | (utf8_int8_t)(chr >> 12));
    c[1] = (utf8_int8_t)(0x80 | (utf8_int8_t)((chr >> 6) & 0x3f));
    c[2] = (utf8_int8_t)(0x80 | (utf8_int8_t)(chr & 0x3f));
  } else { /* if (0 == ((int)0xffe00000 & chr)) { */
    /* 4-byte/21-bit utf8 code point
     * (0b11110xxx 0b10xxxxxx 0b10xxxxxx 0b10xxxxxx) */
    c[0] = (utf8_int8_t)(0xf0 | (utf8_int8_t)(chr >> 18));
    c[1] = (utf8_int8_t)(0x80 | (utf8_int8_t)((chr >> 12) & 0x3f));
    c[2] = (utf8_int8_t)(0x80 | (utf8_int8_t)((chr >> 6) & 0x3f));
    c[3] = (utf8_int8_t)(0x80 | (utf8_int8_t)(chr & 0x3f));
  }

  /* we've created a 2 utf8 codepoint string in c that is
   * the utf8 character asked for by chr, and a null
   * terminating byte */

  while ('\0' != *src) {
    size_t offset = 0;

    while (src[offset] == c[offset]) {
      offset++;
    }

    if ('\0' == c[offset]) {
      /* we found a matching utf8 code point */
      match = (utf8_int8_t *)src;
      src += offset;
    } else {
      src += offset;

      /* need to march s along to next utf8 codepoint start
       * (the next byte that doesn't match 0b10xxxxxx) */
      if ('\0' != *src) {
        do {
          src++;
        } while (0x80 == (0xc0 & *src));
      }
    }
  }

  /* return the last match we found (or 0 if no match was found) */
  return match;
}

utf8_constexpr14_impl utf8_int8_t *utf8pbrk(const utf8_int8_t *str, const utf8_int8_t *accept) {
  while ('\0' != *str) {
    const utf8_int8_t *a = accept;
    size_t offset = 0;

    while ('\0' != *a) {
      /* checking that if *a is the start of a utf8 codepoint
       * (it is not 0b10xxxxxx) and we have successfully matched
       * a previous character (0 < offset) - we found a match */
      if ((0x80 != (0xc0 & *a)) && (0 < offset)) {
        return (utf8_int8_t *)str;
      } else {
        if (*a == str[offset]) {
          /* part of a utf8 codepoint matched, so move our checking
           * onwards to the next byte */
          offset++;
          a++;
        } else {
          /* r could be in the middle of an unmatching utf8 code point,
           * so we need to march it on to the next character beginning, */

          do {
            a++;
          } while (0x80 == (0xc0 & *a));

          /* reset offset too as we found a mismatch */
          offset = 0;
        }
      }
    }

    /* we found a match on the last utf8 codepoint */
    if (0 < offset) {
      return (utf8_int8_t *)str;
    }

    /* the current utf8 codepoint in src did not match accept, but src
     * could have been partway through a utf8 codepoint, so we need to
     * march it onto the next utf8 codepoint starting byte */
    do {
      str++;
    } while ((0x80 == (0xc0 & *str)));
  }

  return utf8_null;
}

utf8_constexpr14_impl size_t utf8size(const utf8_int8_t *str) { return utf8size_lazy(str) + 1; }

utf8_constexpr14_impl size_t utf8size_lazy(const utf8_int8_t *str) { return utf8nsize_lazy(str, SIZE_MAX); }

utf8_constexpr14_impl size_t utf8nsize_lazy(const utf8_int8_t *str, size_t n) {
  size_t size = 0;
  while (size < n && '\0' != str[size]) {
    size++;
  }
  return size;
}

utf8_constexpr14_impl size_t utf8spn(const utf8_int8_t *src, const utf8_int8_t *accept) {
  size_t chars = 0;

  while ('\0' != *src) {
    const utf8_int8_t *a = accept;
    size_t offset = 0;

    while ('\0' != *a) {
      /* checking that if *r is the start of a utf8 codepoint
       * (it is not 0b10xxxxxx) and we have successfully matched
       * a previous character (0 < offset) - we found a match */
      if ((0x80 != (0xc0 & *a)) && (0 < offset)) {
        /* found a match, so increment the number of utf8 codepoints
         * that have matched and stop checking whether any other utf8
         * codepoints in a match */
        chars++;
        src += offset;
        offset = 0;
        break;
      } else {
        if (*a == src[offset]) {
          offset++;
          a++;
        } else {
          /* a could be in the middle of an unmatching utf8 codepoint,
           * so we need to march it on to the next character beginning, */
          do {
            a++;
          } while (0x80 == (0xc0 & *a));

          /* reset offset too as we found a mismatch */
          offset = 0;
        }
      }
    }

    /* found a match at the end of *a, so didn't get a chance to test it */
    if (0 < offset) {
      chars++;
      src += offset;
      continue;
    }

    /* if a got to its terminating null byte, then we didn't find a match.
     * Return the current number of matched utf8 codepoints */
    if ('\0' == *a) {
      return chars;
    }
  }

  return chars;
}

utf8_constexpr14_impl utf8_int8_t *utf8str(const utf8_int8_t *haystack, const utf8_int8_t *needle) {
  utf8_int32_t throwaway_codepoint = 0;

  /* if needle has no utf8 codepoints before the null terminating
   * byte then return haystack */
  if ('\0' == *needle) {
    return (utf8_int8_t *)haystack;
  }

  while ('\0' != *haystack) {
    const utf8_int8_t *maybeMatch = haystack;
    const utf8_int8_t *n = needle;

    while (*haystack == *n && (*haystack != '\0' && *n != '\0')) {
      n++;
      haystack++;
    }

    if ('\0' == *n) {
      /* we found the whole utf8 string for needle in haystack at
       * maybeMatch, so return it */
      return (utf8_int8_t *)maybeMatch;
    } else {
      /* h could be in the middle of an unmatching utf8 codepoint,
       * so we need to march it on to the next character beginning
       * starting from the current character */
      haystack = utf8codepoint(maybeMatch, &throwaway_codepoint);
    }
  }

  /* no match */
  return utf8_null;
}

utf8_constexpr14_impl utf8_int8_t *utf8casestr(const utf8_int8_t *haystack, const utf8_int8_t *needle) {
  /* if needle has no utf8 codepoints before the null terminating
   * byte then return haystack */
  if ('\0' == *needle) {
    return (utf8_int8_t *)haystack;
  }

  for (;;) {
    const utf8_int8_t *maybeMatch = haystack;
    const utf8_int8_t *n = needle;
    utf8_int32_t h_cp = 0, n_cp = 0;

    /* Get the next code point and track it */
    const utf8_int8_t *nextH = haystack = utf8codepoint(haystack, &h_cp);
    n = utf8codepoint(n, &n_cp);

    while ((0 != h_cp) && (0 != n_cp)) {
      h_cp = utf8lwrcodepoint(h_cp);
      n_cp = utf8lwrcodepoint(n_cp);

      /* if we find a mismatch, bail out! */
      if (h_cp != n_cp) {
        break;
      }

      haystack = utf8codepoint(haystack, &h_cp);
      n = utf8codepoint(n, &n_cp);
    }

    if (0 == n_cp) {
      /* we found the whole utf8 string for needle in haystack at
       * maybeMatch, so return it */
      return (utf8_int8_t *)maybeMatch;
    }

    if (0 == h_cp) {
      /* no match */
      return utf8_null;
    }

    /* Roll back to the next code point in the haystack to test */
    haystack = nextH;
  }
}

utf8_constexpr14_impl utf8_int8_t *utf8valid(const utf8_int8_t *str) { return utf8nvalid(str, SIZE_MAX); }

utf8_constexpr14_impl utf8_int8_t *utf8nvalid(const utf8_int8_t *str, size_t n) {
  const utf8_int8_t *t = str;
  size_t consumed = 0;

  while ((void)(consumed = (size_t)(str - t)), consumed < n && '\0' != *str) {
    const size_t remaining = n - consumed;

    if (0xf0 == (0xf8 & *str)) {
      /* ensure that there's 4 bytes or more remaining */
      if (remaining < 4) {
        return (utf8_int8_t *)str;
      }

      /* ensure each of the 3 following bytes in this 4-byte
       * utf8 codepoint began with 0b10xxxxxx */
      if ((0x80 != (0xc0 & str[1])) || (0x80 != (0xc0 & str[2])) || (0x80 != (0xc0 & str[3]))) {
        return (utf8_int8_t *)str;
      }

      /* ensure that our utf8 codepoint ended after 4 bytes */
      if ((remaining != 4) && (0x80 == (0xc0 & str[4]))) {
        return (utf8_int8_t *)str;
      }

      /* ensure that the top 5 bits of this 4-byte utf8
       * codepoint were not 0, as then we could have used
       * one of the smaller encodings */
      if ((0 == (0x07 & str[0])) && (0 == (0x30 & str[1]))) {
        return (utf8_int8_t *)str;
      }

      /* 4-byte utf8 code point (began with 0b11110xxx) */
      str += 4;
    } else if (0xe0 == (0xf0 & *str)) {
      /* ensure that there's 3 bytes or more remaining */
      if (remaining < 3) {
        return (utf8_int8_t *)str;
      }

      /* ensure each of the 2 following bytes in this 3-byte
       * utf8 codepoint began with 0b10xxxxxx */
      if ((0x80 != (0xc0 & str[1])) || (0x80 != (0xc0 & str[2]))) {
        return (utf8_int8_t *)str;
      }

      /* ensure that our utf8 codepoint ended after 3 bytes */
      if ((remaining != 3) && (0x80 == (0xc0 & str[3]))) {
        return (utf8_int8_t *)str;
      }

      /* ensure that the top 5 bits of this 3-byte utf8
       * codepoint were not 0, as then we could have used
       * one of the smaller encodings */
      if ((0 == (0x0f & str[0])) && (0 == (0x20 & str[1]))) {
        return (utf8_int8_t *)str;
      }

      /* 3-byte utf8 code point (began with 0b1110xxxx) */
      str += 3;
    } else if (0xc0 == (0xe0 & *str)) {
      /* ensure that there's 2 bytes or more remaining */
      if (remaining < 2) {
        return (utf8_int8_t *)str;
      }

      /* ensure the 1 following byte in this 2-byte
       * utf8 codepoint began with 0b10xxxxxx */
      if (0x80 != (0xc0 & str[1])) {
        return (utf8_int8_t *)str;
      }

      /* ensure that our utf8 codepoint ended after 2 bytes */
      if ((remaining != 2) && (0x80 == (0xc0 & str[2]))) {
        return (utf8_int8_t *)str;
      }

      /* ensure that the top 4 bits of this 2-byte utf8
       * codepoint were not 0, as then we could have used
       * one of the smaller encodings */
      if (0 == (0x1e & str[0])) {
        return (utf8_int8_t *)str;
      }

      /* 2-byte utf8 code point (began with 0b110xxxxx) */
      str += 2;
    } else if (0x00 == (0x80 & *str)) {
      /* 1-byte ascii (began with 0b0xxxxxxx) */
      str += 1;
    } else {
      /* we have an invalid 0b1xxxxxxx utf8 code point entry */
      return (utf8_int8_t *)str;
    }
  }

  return utf8_null;
}

int utf8makevalid(utf8_int8_t *str, const utf8_int32_t replacement) {
  utf8_int8_t *read = str;
  utf8_int8_t *write = read;
  const utf8_int8_t r = (utf8_int8_t)replacement;
  utf8_int32_t codepoint = 0;

  if (replacement > 0x7f) {
    return -1;
  }

  while ('\0' != *read) {
    if (0xf0 == (0xf8 & *read)) {
      /* ensure each of the 3 following bytes in this 4-byte
       * utf8 codepoint began with 0b10xxxxxx */
      if ((0x80 != (0xc0 & read[1])) || (0x80 != (0xc0 & read[2])) || (0x80 != (0xc0 & read[3]))) {
        *write++ = r;
        read++;
        continue;
      }

      /* 4-byte utf8 code point (began with 0b11110xxx) */
      read = utf8codepoint(read, &codepoint);
      write = utf8catcodepoint(write, codepoint, 4);
    } else if (0xe0 == (0xf0 & *read)) {
      /* ensure each of the 2 following bytes in this 3-byte
       * utf8 codepoint began with 0b10xxxxxx */
      if ((0x80 != (0xc0 & read[1])) || (0x80 != (0xc0 & read[2]))) {
        *write++ = r;
        read++;
        continue;
      }

      /* 3-byte utf8 code point (began with 0b1110xxxx) */
      read = utf8codepoint(read, &codepoint);
      write = utf8catcodepoint(write, codepoint, 3);
    } else if (0xc0 == (0xe0 & *read)) {
      /* ensure the 1 following byte in this 2-byte
       * utf8 codepoint began with 0b10xxxxxx */
      if (0x80 != (0xc0 & read[1])) {
        *write++ = r;
        read++;
        continue;
      }

      /* 2-byte utf8 code point (began with 0b110xxxxx) */
      read = utf8codepoint(read, &codepoint);
      write = utf8catcodepoint(write, codepoint, 2);
    } else if (0x00 == (0x80 & *read)) {
      /* 1-byte ascii (began with 0b0xxxxxxx) */
      read = utf8codepoint(read, &codepoint);
      write = utf8catcodepoint(write, codepoint, 1);
    } else {
      /* if we got here then we've got a dangling continuation (0b10xxxxxx) */
      *write++ = r;
      read++;
      continue;
    }
  }

  *write = '\0';

  return 0;
}

utf8_constexpr14_impl utf8_int8_t *utf8codepoint(const utf8_int8_t *utf8_restrict str,
                                                 utf8_int32_t *utf8_restrict out_codepoint) {
  if (0xf0 == (0xf8 & str[0])) {
    /* 4 byte utf8 codepoint */
    *out_codepoint = ((0x07 & str[0]) << 18) | ((0x3f & str[1]) << 12) | ((0x3f & str[2]) << 6) | (0x3f & str[3]);
    str += 4;
  } else if (0xe0 == (0xf0 & str[0])) {
    /* 3 byte utf8 codepoint */
    *out_codepoint = ((0x0f & str[0]) << 12) | ((0x3f & str[1]) << 6) | (0x3f & str[2]);
    str += 3;
  } else if (0xc0 == (0xe0 & str[0])) {
    /* 2 byte utf8 codepoint */
    *out_codepoint = ((0x1f & str[0]) << 6) | (0x3f & str[1]);
    str += 2;
  } else {
    /* 1 byte utf8 codepoint otherwise */
    *out_codepoint = str[0];
    str += 1;
  }

  return (utf8_int8_t *)str;
}

utf8_constexpr14_impl size_t utf8codepointcalcsize(const utf8_int8_t *str) {
  if (0xf0 == (0xf8 & str[0])) {
    /* 4 byte utf8 codepoint */
    return 4;
  } else if (0xe0 == (0xf0 & str[0])) {
    /* 3 byte utf8 codepoint */
    return 3;
  } else if (0xc0 == (0xe0 & str[0])) {
    /* 2 byte utf8 codepoint */
    return 2;
  }

  /* 1 byte utf8 codepoint otherwise */
  return 1;
}

utf8_constexpr14_impl size_t utf8codepointsize(utf8_int32_t chr) {
  if (0 == ((utf8_int32_t)0xffffff80 & chr)) {
    return 1;
  } else if (0 == ((utf8_int32_t)0xfffff800 & chr)) {
    return 2;
  } else if (0 == ((utf8_int32_t)0xffff0000 & chr)) {
    return 3;
  } else { /* if (0 == ((int)0xffe00000 & chr)) { */
    return 4;
  }
}

utf8_int8_t *utf8catcodepoint(utf8_int8_t *str, utf8_int32_t chr, size_t n) {
  if (0 == ((utf8_int32_t)0xffffff80 & chr)) {
    /* 1-byte/7-bit ascii
     * (0b0xxxxxxx) */
    if (n < 1) {
      return utf8_null;
    }
    str[0] = (utf8_int8_t)chr;
    str += 1;
  } else if (0 == ((utf8_int32_t)0xfffff800 & chr)) {
    /* 2-byte/11-bit utf8 code point
     * (0b110xxxxx 0b10xxxxxx) */
    if (n < 2) {
      return utf8_null;
    }
    str[0] = (utf8_int8_t)(0xc0 | (utf8_int8_t)((chr >> 6) & 0x1f));
    str[1] = (utf8_int8_t)(0x80 | (utf8_int8_t)(chr & 0x3f));
    str += 2;
  } else if (0 == ((utf8_int32_t)0xffff0000 & chr)) {
    /* 3-byte/16-bit utf8 code point
     * (0b1110xxxx 0b10xxxxxx 0b10xxxxxx) */
    if (n < 3) {
      return utf8_null;
    }
    str[0] = (utf8_int8_t)(0xe0 | (utf8_int8_t)((chr >> 12) & 0x0f));
    str[1] = (utf8_int8_t)(0x80 | (utf8_int8_t)((chr >> 6) & 0x3f));
    str[2] = (utf8_int8_t)(0x80 | (utf8_int8_t)(chr & 0x3f));
    str += 3;
  } else { /* if (0 == ((int)0xffe00000 & chr)) { */
    /* 4-byte/21-bit utf8 code point
     * (0b11110xxx 0b10xxxxxx 0b10xxxxxx 0b10xxxxxx) */
    if (n < 4) {
      return utf8_null;
    }
    str[0] = (utf8_int8_t)(0xf0 | (utf8_int8_t)((chr >> 18) & 0x07));
    str[1] = (utf8_int8_t)(0x80 | (utf8_int8_t)((chr >> 12) & 0x3f));
    str[2] = (utf8_int8_t)(0x80 | (utf8_int8_t)((chr >> 6) & 0x3f));
    str[3] = (utf8_int8_t)(0x80 | (utf8_int8_t)(chr & 0x3f));
    str += 4;
  }

  return str;
}

utf8_constexpr14_impl int utf8islower(utf8_int32_t chr) { return chr != utf8uprcodepoint(chr); }

utf8_constexpr14_impl int utf8isupper(utf8_int32_t chr) { return chr != utf8lwrcodepoint(chr); }

void utf8lwr(utf8_int8_t *utf8_restrict str) {
  utf8_int32_t cp = 0;
  utf8_int8_t *pn = utf8codepoint(str, &cp);

  while (cp != 0) {
    const utf8_int32_t lwr_cp = utf8lwrcodepoint(cp);
    const size_t size = utf8codepointsize(lwr_cp);

    if (lwr_cp != cp) {
      utf8catcodepoint(str, lwr_cp, size);
    }

    str = pn;
    pn = utf8codepoint(str, &cp);
  }
}

void utf8upr(utf8_int8_t *utf8_restrict str) {
  utf8_int32_t cp = 0;
  utf8_int8_t *pn = utf8codepoint(str, &cp);

  while (cp != 0) {
    const utf8_int32_t lwr_cp = utf8uprcodepoint(cp);
    const size_t size = utf8codepointsize(lwr_cp);

    if (lwr_cp != cp) {
      utf8catcodepoint(str, lwr_cp, size);
    }

    str = pn;
    pn = utf8codepoint(str, &cp);
  }
}

utf8_constexpr14_impl utf8_int32_t utf8lwrcodepoint(utf8_int32_t cp) {
  if (((0x0041 <= cp) && (0x005a >= cp)) || ((0x00c0 <= cp) && (0x00d6 >= cp)) || ((0x00d8 <= cp) && (0x00de >= cp)) ||
      ((0x0391 <= cp) && (0x03a1 >= cp)) || ((0x03a3 <= cp) && (0x03ab >= cp)) || ((0x0410 <= cp) && (0x042f >= cp))) {
    cp += 32;
  } else if ((0x0400 <= cp) && (0x040f >= cp)) {
    cp += 80;
  } else if (((0x0100 <= cp) && (0x012f >= cp)) || ((0x0132 <= cp) && (0x0137 >= cp)) ||
             ((0x014a <= cp) && (0x0177 >= cp)) || ((0x0182 <= cp) && (0x0185 >= cp)) ||
             ((0x01a0 <= cp) && (0x01a5 >= cp)) || ((0x01de <= cp) && (0x01ef >= cp)) ||
             ((0x01f8 <= cp) && (0x021f >= cp)) || ((0x0222 <= cp) && (0x0233 >= cp)) ||
             ((0x0246 <= cp) && (0x024f >= cp)) || ((0x03d8 <= cp) && (0x03ef >= cp)) ||
             ((0x0460 <= cp) && (0x0481 >= cp)) || ((0x048a <= cp) && (0x04ff >= cp))) {
    cp |= 0x1;
  } else if (((0x0139 <= cp) && (0x0148 >= cp)) || ((0x0179 <= cp) && (0x017e >= cp)) ||
             ((0x01af <= cp) && (0x01b0 >= cp)) || ((0x01b3 <= cp) && (0x01b6 >= cp)) ||
             ((0x01cd <= cp) && (0x01dc >= cp))) {
    cp += 1;
    cp &= ~0x1;
  } else {
    switch (cp) {
      default:
        break;
      case 0x0178:
        cp = 0x00ff;
        break;
      case 0x0243:
        cp = 0x0180;
        break;
      case 0x018e:
        cp = 0x01dd;
        break;
      case 0x023d:
        cp = 0x019a;
        break;
      case 0x0220:
        cp = 0x019e;
        break;
      case 0x01b7:
        cp = 0x0292;
        break;
      case 0x01c4:
        cp = 0x01c6;
        break;
      case 0x01c7:
        cp = 0x01c9;
        break;
      case 0x01ca:
        cp = 0x01cc;
        break;
      case 0x01f1:
        cp = 0x01f3;
        break;
      case 0x01f7:
        cp = 0x01bf;
        break;
      case 0x0187:
        cp = 0x0188;
        break;
      case 0x018b:
        cp = 0x018c;
        break;
      case 0x0191:
        cp = 0x0192;
        break;
      case 0x0198:
        cp = 0x0199;
        break;
      case 0x01a7:
        cp = 0x01a8;
        break;
      case 0x01ac:
        cp = 0x01ad;
        break;
      case 0x01af:
        cp = 0x01b0;
        break;
      case 0x01b8:
        cp = 0x01b9;
        break;
      case 0x01bc:
        cp = 0x01bd;
        break;
      case 0x01f4:
        cp = 0x01f5;
        break;
      case 0x023b:
        cp = 0x023c;
        break;
      case 0x0241:
        cp = 0x0242;
        break;
      case 0x03fd:
        cp = 0x037b;
        break;
      case 0x03fe:
        cp = 0x037c;
        break;
      case 0x03ff:
        cp = 0x037d;
        break;
      case 0x037f:
        cp = 0x03f3;
        break;
      case 0x0386:
        cp = 0x03ac;
        break;
      case 0x0388:
        cp = 0x03ad;
        break;
      case 0x0389:
        cp = 0x03ae;
        break;
      case 0x038a:
        cp = 0x03af;
        break;
      case 0x038c:
        cp = 0x03cc;
        break;
      case 0x038e:
        cp = 0x03cd;
        break;
      case 0x038f:
        cp = 0x03ce;
        break;
      case 0x0370:
        cp = 0x0371;
        break;
      case 0x0372:
        cp = 0x0373;
        break;
      case 0x0376:
        cp = 0x0377;
        break;
      case 0x03f4:
        cp = 0x03b8;
        break;
      case 0x03cf:
        cp = 0x03d7;
        break;
      case 0x03f9:
        cp = 0x03f2;
        break;
      case 0x03f7:
        cp = 0x03f8;
        break;
      case 0x03fa:
        cp = 0x03fb;
        break;
    }
  }

  return cp;
}

utf8_constexpr14_impl utf8_int32_t utf8uprcodepoint(utf8_int32_t cp) {
  if (((0x0061 <= cp) && (0x007a >= cp)) || ((0x00e0 <= cp) && (0x00f6 >= cp)) || ((0x00f8 <= cp) && (0x00fe >= cp)) ||
      ((0x03b1 <= cp) && (0x03c1 >= cp)) || ((0x03c3 <= cp) && (0x03cb >= cp)) || ((0x0430 <= cp) && (0x044f >= cp))) {
    cp -= 32;
  } else if ((0x0450 <= cp) && (0x045f >= cp)) {
    cp -= 80;
  } else if (((0x0100 <= cp) && (0x012f >= cp)) || ((0x0132 <= cp) && (0x0137 >= cp)) ||
             ((0x014a <= cp) && (0x0177 >= cp)) || ((0x0182 <= cp) && (0x0185 >= cp)) ||
             ((0x01a0 <= cp) && (0x01a5 >= cp)) || ((0x01de <= cp) && (0x01ef >= cp)) ||
             ((0x01f8 <= cp) && (0x021f >= cp)) || ((0x0222 <= cp) && (0x0233 >= cp)) ||
             ((0x0246 <= cp) && (0x024f >= cp)) || ((0x03d8 <= cp) && (0x03ef >= cp)) ||
             ((0x0460 <= cp) && (0x0481 >= cp)) || ((0x048a <= cp) && (0x04ff >= cp))) {
    cp &= ~0x1;
  } else if (((0x0139 <= cp) && (0x0148 >= cp)) || ((0x0179 <= cp) && (0x017e >= cp)) ||
             ((0x01af <= cp) && (0x01b0 >= cp)) || ((0x01b3 <= cp) && (0x01b6 >= cp)) ||
             ((0x01cd <= cp) && (0x01dc >= cp))) {
    cp -= 1;
    cp |= 0x1;
  } else {
    switch (cp) {
      default:
        break;
      case 0x00ff:
        cp = 0x0178;
        break;
      case 0x0180:
        cp = 0x0243;
        break;
      case 0x01dd:
        cp = 0x018e;
        break;
      case 0x019a:
        cp = 0x023d;
        break;
      case 0x019e:
        cp = 0x0220;
        break;
      case 0x0292:
        cp = 0x01b7;
        break;
      case 0x01c6:
        cp = 0x01c4;
        break;
      case 0x01c9:
        cp = 0x01c7;
        break;
      case 0x01cc:
        cp = 0x01ca;
        break;
      case 0x01f3:
        cp = 0x01f1;
        break;
      case 0x01bf:
        cp = 0x01f7;
        break;
      case 0x0188:
        cp = 0x0187;
        break;
      case 0x018c:
        cp = 0x018b;
        break;
      case 0x0192:
        cp = 0x0191;
        break;
      case 0x0199:
        cp = 0x0198;
        break;
      case 0x01a8:
        cp = 0x01a7;
        break;
      case 0x01ad:
        cp = 0x01ac;
        break;
      case 0x01b0:
        cp = 0x01af;
        break;
      case 0x01b9:
        cp = 0x01b8;
        break;
      case 0x01bd:
        cp = 0x01bc;
        break;
      case 0x01f5:
        cp = 0x01f4;
        break;
      case 0x023c:
        cp = 0x023b;
        break;
      case 0x0242:
        cp = 0x0241;
        break;
      case 0x037b:
        cp = 0x03fd;
        break;
      case 0x037c:
        cp = 0x03fe;
        break;
      case 0x037d:
        cp = 0x03ff;
        break;
      case 0x03f3:
        cp = 0x037f;
        break;
      case 0x03ac:
        cp = 0x0386;
        break;
      case 0x03ad:
        cp = 0x0388;
        break;
      case 0x03ae:
        cp = 0x0389;
        break;
      case 0x03af:
        cp = 0x038a;
        break;
      case 0x03cc:
        cp = 0x038c;
        break;
      case 0x03cd:
        cp = 0x038e;
        break;
      case 0x03ce:
        cp = 0x038f;
        break;
      case 0x0371:
        cp = 0x0370;
        break;
      case 0x0373:
        cp = 0x0372;
        break;
      case 0x0377:
        cp = 0x0376;
        break;
      case 0x03d1:
        cp = 0x0398;
        break;
      case 0x03d7:
        cp = 0x03cf;
        break;
      case 0x03f2:
        cp = 0x03f9;
        break;
      case 0x03f8:
        cp = 0x03f7;
        break;
      case 0x03fb:
        cp = 0x03fa;
        break;
    }
  }

  return cp;
}

utf8_constexpr14_impl utf8_int8_t *utf8rcodepoint(const utf8_int8_t *utf8_restrict str,
                                                  utf8_int32_t *utf8_restrict out_codepoint) {
  const utf8_int8_t *s = (const utf8_int8_t *)str;

  if (0xf0 == (0xf8 & s[0])) {
    /* 4 byte utf8 codepoint */
    *out_codepoint = ((0x07 & s[0]) << 18) | ((0x3f & s[1]) << 12) | ((0x3f & s[2]) << 6) | (0x3f & s[3]);
  } else if (0xe0 == (0xf0 & s[0])) {
    /* 3 byte utf8 codepoint */
    *out_codepoint = ((0x0f & s[0]) << 12) | ((0x3f & s[1]) << 6) | (0x3f & s[2]);
  } else if (0xc0 == (0xe0 & s[0])) {
    /* 2 byte utf8 codepoint */
    *out_codepoint = ((0x1f & s[0]) << 6) | (0x3f & s[1]);
  } else {
    /* 1 byte utf8 codepoint otherwise */
    *out_codepoint = s[0];
  }

  do {
    s--;
  } while ((0 != (0x80 & s[0])) && (0x80 == (0xc0 & s[0])));

  return (utf8_int8_t *)s;
}

#undef utf8_restrict
#undef utf8_constexpr14
#undef utf8_null

#ifdef __cplusplus
} /* extern "C" */
#endif

#if defined(__clang__)
#pragma clang diagnostic pop
#endif

#endif /* SHEREDOM_UTF8_H_INCLUDED */

#pragma once

#include "common/net/sync/ConnectionPool.h"
#include "common/serde/ClientContext.h"

namespace hf3fs::net::sync {

class Client {
 public:
  struct Config : public ConfigBase<Config> {
    CONFIG_HOT_UPDATED_ITEM(default_timeout, kClientRequestDefaultTimeout);
    CONFIG_HOT_UPDATED_ITEM(default_send_retry_times, kDefaultMaxRetryTimes, ConfigCheckers::checkPositive);
    CONFIG_HOT_UPDATED_ITEM(default_compression_level, 0u);
    CONFIG_HOT_UPDATED_ITEM(default_compression_threshold, 128_KB);
    CONFIG_OBJ(connection_pool, ConnectionPool::Config);
  };

  explicit Client(const Config &config)
      : config_(config),
        connectionPool_(config_.connection_pool()),
        clientConfigGuard_(config_.addCallbackGuard([&] { updateDefaultOptions(); })) {
    updateDefaultOptions();
  }

  auto serdeCtx(Address serverAddr) { return serde::ClientContext(connectionPool_, serverAddr, options_); }

 protected:
  void updateDefaultOptions() {
    auto options = std::make_shared<CoreRequestOptions>();
    options->timeout = config_.default_timeout();
    options->sendRetryTimes = config_.default_send_retry_times();
    options->compression = {config_.default_compression_level(), config_.default_compression_threshold()};
    options_ = std::move(options);
  }

 private:
  const Config &config_;
  ConnectionPool connectionPool_;
  std::unique_ptr<ConfigCallbackGuard> clientConfigGuard_;
  folly::atomic_shared_ptr<const CoreRequestOptions> options_{std::make_shared<CoreRequestOptions>()};
};

}  // namespace hf3fs::net::sync

#pragma once

#include <condition_variable>
#include <folly/Hash.h>
#include <map>
#include <memory>
#include <mutex>
#include <shared_mutex>
#include <vector>

#include "common/utils/RobinHood.h"

namespace hf3fs {

class ReentrantLockManager {
  enum class LockType { NONE, READ, WRITE };

 public:
  void init(uint32_t shardsNum = 4099u);

  class LockGuard {
   public:
    LockGuard() = default;
    LockGuard(ReentrantLockManager *locker, uint64_t id, LockType type);
    LockGuard(const LockGuard &) = delete;
    LockGuard(LockGuard &&o);
    ~LockGuard() { unlock(); }
    void unlock();
    bool locked() const { return type_ != LockType::NONE; }
    bool readLocked() const { return type_ == LockType::READ; }
    bool writeLocked() const { return type_ == LockType::WRITE; }
    LockGuard &operator=(LockGuard &&o);

   private:
    ReentrantLockManager *locker_ = nullptr;
    uint64_t id_ = 0;
    LockType type_ = LockType::NONE;
  };

  LockGuard readLock(uint64_t id);
  LockGuard tryReadLock(uint64_t id);
  void readUnlock(uint64_t id);
  LockGuard writeLock(uint64_t id);
  LockGuard tryWriteLock(uint64_t id);
  void writeUnlock(uint64_t id);

 private:
  struct Status {
    int currentLock = 0;  // positive for read locked, negative for write locked.
    int waitingReadLock = 0;
    int waitingWriteLock = 0;

    bool readyToErase() const { return currentLock == 0 && waitingReadLock == 0 && waitingWriteLock == 0; }
    bool readyToReadLock() const { return currentLock >= 0 && waitingWriteLock == 0; }
    void readLock() { ++currentLock; }
    void readUnlock() { --currentLock; }
    bool wakeUpWaitingReadLock() { return currentLock >= 0 && waitingReadLock > 0; }
    bool readyToWriteLock() const { return currentLock == 0; }
    void writeLock() { --currentLock; }
    void writeUnlock() { ++currentLock; }
    bool wakeUpWaitingWriteLock() { return currentLock == 0 && waitingWriteLock > 0; }
  };
  struct Shard {
    std::mutex mutex;
    std::condition_variable readCond;
    std::condition_variable writeCond;
    robin_hood::unordered_map<uint64_t, Status> map;
  };
  Shard &getShard(uint64_t id) { return shards_[folly::hash::twang_32from64(id) % shardsNum_]; }

 private:
  uint32_t shardsNum_;
  std::unique_ptr<Shard[]> shards_;
};

}  // namespace hf3fs

#pragma once

#include <folly/CancellationToken.h>
#include <folly/Random.h>
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <folly/experimental/coro/Collect.h>
#include <folly/experimental/coro/Sleep.h>
#include <folly/logging/xlog.h>

#include "common/app/ApplicationBase.h"
#include "common/utils/CPUExecutorGroup.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/Coroutine.h"
#include "common/utils/PriorityUnboundedQueue.h"

namespace hf3fs {

/** Like CoroutinePool, but use PriorityUnboundedQueue in order to support tasks with priority. */
struct PriorityCoroutinePoolConfig : public ConfigBase<PriorityCoroutinePoolConfig> {
  // todo: support hot update coroutine numbers
  CONFIG_HOT_UPDATED_ITEM(coroutines_num, 64u, ConfigCheckers::checkPositive);
  // unused, just for compatibility
  CONFIG_HOT_UPDATED_ITEM(queue_size, 1024u);
  CONFIG_HOT_UPDATED_ITEM(enable_work_stealing, false);
};

template <class Job>
class PriorityCoroutinePool {
 public:
  using Config = PriorityCoroutinePoolConfig;

  PriorityCoroutinePool(const Config &config, uint8_t numPriorities = 3)
      : config_(config),
        queue_(numPriorities),
        cancel_(),
        futures_() {}
  ~PriorityCoroutinePool() { stopAndJoin(); }

  using Handler = std::function<CoTask<void>(Job job)>;
  Result<Void> start(Handler handler, CPUExecutorGroup &grp) {
    auto coroutines = config_.coroutines_num();
    for (auto i = 0u; i < coroutines; ++i) {
      futures_.push_back(run(handler, i).scheduleOn(&grp.pickNext()).start());
    }
    return Void{};
  }

  void stopAndJoin() {
    if (stopped_.test_and_set()) {
      XLOGF(DBG, "coroutines pool {} is already stopped.", fmt::ptr(this));
      return;
    }
    cancel_.requestCancellation();
    for (auto &future : futures_) {
      future.wait();
    }
    futures_.clear();
    XLOGF(INFO, "coroutines pool {} is stopped", fmt::ptr(this));
  }

  void enqueue(Job job, int8_t priority) { queue_.addWithPriority(std::move(job), priority); }

 private:
  CoTask<void> run(Handler handler, size_t queueIndex) {
    while (true) {
      auto result = co_await co_awaitTry(co_withCancellation(cancel_.getToken(), queue_.co_dequeue()));
      if (UNLIKELY(result.template hasException<OperationCancelled>())) {
        XLOGF(DBG, "a coroutine in pool {} is stopped", fmt::ptr(this));
        break;
      } else if (result.hasException()) {
        XLOGF(ERR, "a coroutine in pool {} is stopped (exception: {})", fmt::ptr(this), result.exception().what());
        break;
      }
      co_await handler(std::move(*result));
    }
  }

  const Config &config_;
  PriorityUnboundedQueue<Job> queue_;
  CancellationSource cancel_;
  std::vector<folly::SemiFuture<Void>> futures_;
  std::atomic_flag stopped_{false};
};

}  // namespace hf3fs

#pragma once

#include <fmt/core.h>
#include <folly/Function.h>
#include <folly/Synchronized.h>
#include <folly/TimeoutQueue.h>
#include <folly/concurrency/AtomicSharedPtr.h>
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <folly/experimental/coro/Invoke.h>
#include <folly/experimental/coro/Sleep.h>
#include <folly/logging/xlog.h>
#include <functional>
#include <map>
#include <memory>
#include <mutex>
#include <optional>
#include <set>
#include <type_traits>
#include <utility>
#include <variant>
#include <vector>

#include "client/meta/ServerSelectionStrategy.h"
#include "client/mgmtd/ICommonMgmtdClient.h"
#include "client/storage/StorageClient.h"
#include "common/app/ClientId.h"
#include "common/app/NodeId.h"
#include "common/serde/MessagePacket.h"
#include "common/utils/BackgroundRunner.h"
#include "common/utils/CPUExecutorGroup.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/Coroutine.h"
#include "common/utils/CoroutinesPool.h"
#include "common/utils/Duration.h"
#include "common/utils/ExponentialBackoffRetry.h"
#include "common/utils/Path.h"
#include "common/utils/Result.h"
#include "common/utils/Semaphore.h"
#include "common/utils/StatusCode.h"
#include "common/utils/UtcTime.h"
#include "common/utils/Uuid.h"
#include "fbs/meta/Common.h"
#include "fbs/meta/Service.h"
#include "stubs/MetaService/IMetaServiceStub.h"
#include "stubs/MetaService/MetaServiceStub.h"
#include "stubs/MetaService/MockMetaServiceStub.h"
#include "stubs/common/StubFactory.h"

namespace hf3fs::meta::client {

using SessionId = hf3fs::Uuid;

using hf3fs::client::ICommonMgmtdClient;

class MetaClient {
 public:
  using StubFactory = stubs::SerdeStubFactory<MetaServiceStub, true>;
  using Stub = StubFactory::Stub;

  class RetryConfig : public ConfigBase<RetryConfig> {
    CONFIG_HOT_UPDATED_ITEM(rpc_timeout, 5_s);
    CONFIG_HOT_UPDATED_ITEM(retry_send, 1u, ConfigCheckers::checkPositive);
    CONFIG_HOT_UPDATED_ITEM(retry_fast, 1_s);
    CONFIG_HOT_UPDATED_ITEM(retry_init_wait, 500_ms);
    CONFIG_HOT_UPDATED_ITEM(retry_max_wait, 5_s);
    CONFIG_HOT_UPDATED_ITEM(retry_total_time, 1_min);
    CONFIG_HOT_UPDATED_ITEM(max_failures_before_failover, 1u, ConfigCheckers::checkPositive);
  };

  class CloserConfig : public ConfigBase<CloserConfig> {
    CONFIG_HOT_UPDATED_ITEM(task_scan, 50_ms);
    CONFIG_HOT_UPDATED_ITEM(prune_session_batch_interval, 10_s);
    CONFIG_HOT_UPDATED_ITEM(prune_session_batch_count, 128);
    CONFIG_HOT_UPDATED_ITEM(retry_first_wait, 100_ms);
    CONFIG_HOT_UPDATED_ITEM(retry_max_wait, 10_s);
    CONFIG_OBJ(coroutine_pool, CoroutinesPoolBase::Config, [](CoroutinesPoolBase::Config &c) {
      c.set_coroutines_num(8);
      c.set_queue_size(128);
    });
  };

  struct Config : public ConfigBase<Config> {
    CONFIG_HOT_UPDATED_ITEM(selection_mode, ServerSelectionMode::RandomFollow);
    CONFIG_HOT_UPDATED_ITEM(network_type, net::Address::Type::RDMA);
    CONFIG_HOT_UPDATED_ITEM(check_server_interval, 5_s);
    CONFIG_HOT_UPDATED_ITEM(max_concurrent_requests, 128u, ConfigCheckers::checkPositive);
    CONFIG_HOT_UPDATED_ITEM(remove_chunks_batch_size, uint32_t(32), ConfigCheckers::checkPositive);
    CONFIG_HOT_UPDATED_ITEM(remove_chunks_max_iters, uint32_t(1024), ConfigCheckers::checkPositive);  // deprecated

    CONFIG_HOT_UPDATED_ITEM(dynamic_stripe, false);

    CONFIG_OBJ(retry_default, RetryConfig);
    CONFIG_OBJ(background_closer, CloserConfig);
  };

  MetaClient(ClientId clientId,
             const Config &config,
             std::unique_ptr<StubFactory> factory,
             std::shared_ptr<ICommonMgmtdClient> mgmtd,
             std::shared_ptr<storage::client::StorageClient> storage,
             bool dynStripe);

  void start(CPUExecutorGroup &exec);
  void stop();

  CoTryTask<UserInfo> authenticate(const UserInfo &userInfo);

  CoTryTask<Inode> stat(const UserInfo &userInfo,
                        InodeId inodeId,
                        const std::optional<Path> &path,
                        bool followLastSymlink);

  CoTryTask<std::vector<std::optional<Inode>>> batchStat(const UserInfo &userInfo, std::vector<InodeId> inodeIds);

  CoTryTask<std::vector<Result<Inode>>> batchStatByPath(const UserInfo &userInfo,
                                                        std::vector<PathAt> paths,
                                                        bool followLastSymlink);

  CoTryTask<StatFsRsp> statFs(const UserInfo &userInfo);

  CoTryTask<Path> getRealPath(const UserInfo &userInfo, InodeId parent, const std::optional<Path> &path, bool absolute);

  CoTryTask<Inode> open(const UserInfo &userInfo,
                        InodeId inodeId,
                        const std::optional<Path> &path,
                        std::optional<SessionId> sessionId,
                        int flags);

  CoTryTask<Inode> close(const UserInfo &userInfo,
                         InodeId inodeId,
                         std::optional<SessionId> sessionId,
                         bool read,
                         bool written);
  CoTryTask<Inode> close(const UserInfo &userInfo,
                         InodeId inodeId,
                         std::optional<SessionId> sessionId,
                         bool updateLength,
                         const std::optional<UtcTime> atime,
                         const std::optional<UtcTime> mtime);

  CoTryTask<Inode> setPermission(const UserInfo &userInfo,
                                 InodeId parent,
                                 const std::optional<Path> &path,
                                 bool followLastSymlink,
                                 std::optional<Uid> uid,
                                 std::optional<Gid> gid,
                                 std::optional<Permission> perm,
                                 std::optional<IFlags> iflags = std::nullopt);

  CoTryTask<Inode> setIFlags(const UserInfo &userInfo, InodeId inode, IFlags iflags);

  CoTryTask<Inode> utimes(const UserInfo &userInfo,
                          InodeId inodeId,
                          const std::optional<Path> &path,
                          bool followLastSymlink,
                          std::optional<UtcTime> atime,
                          std::optional<UtcTime> mtime);

  CoTryTask<Inode> create(const UserInfo &userInfo,
                          InodeId parent,
                          const Path &path,
                          std::optional<SessionId> sessionId,
                          Permission perm,
                          int flags,
                          std::optional<Layout> layout = std::nullopt);

  CoTryTask<Inode> mkdirs(const UserInfo &userInfo,
                          InodeId parent,
                          const Path &path,
                          Permission perm,
                          bool recursive,
                          std::optional<Layout> layout = std::nullopt);

  CoTryTask<Inode> symlink(const UserInfo &userInfo, InodeId parent, const Path &path, const Path &target);

  // remove without check inode type
  CoTryTask<void> remove(const UserInfo &userInfo, InodeId parent, const std::optional<Path> &path, bool recursive);

  // only remove directory
  CoTryTask<void> rmdir(const UserInfo &userInfo, InodeId parent, const std::optional<Path> &path, bool recursive);

  // unlink file
  CoTryTask<void> unlink(const UserInfo &userInfo, InodeId parent, const Path &path);

  CoTryTask<Inode> rename(const UserInfo &userInfo,
                          InodeId srcParent,
                          const Path &src,
                          InodeId dstParent,
                          const Path &dst,
                          bool moveToTrash = false);

  CoTryTask<ListRsp> list(const UserInfo &userInfo,
                          InodeId inodeId,
                          const std::optional<Path> &path,
                          std::string_view prev,
                          int32_t limit,
                          bool needStatus);

  CoTryTask<Inode> setLayout(const UserInfo &userInfo, InodeId inodeId, const std::optional<Path> &path, Layout layout);

  CoTryTask<Inode> truncate(const UserInfo &userInfo, InodeId inodeId, uint64_t length);

  CoTryTask<Inode> sync(const UserInfo &userInfo,
                        InodeId inode,
                        bool read,
                        bool written,
                        std::optional<VersionedLength> hint);
  CoTryTask<Inode> sync(const UserInfo &userInfo,
                        InodeId inode,
                        bool updateLength,
                        const std::optional<UtcTime> atime,
                        const std::optional<UtcTime> mtime,
                        std::optional<VersionedLength> hint);

  CoTryTask<Inode> hardLink(const UserInfo &userInfo,
                            InodeId oldParent,
                            const std::optional<Path> &oldPath,
                            InodeId newParent,
                            const Path &newPath,
                            bool followLastSymlink);

  CoTryTask<void> lockDirectory(const UserInfo &userInfo, InodeId inode, LockDirectoryReq::LockAction action);

  CoTryTask<Inode> extendStripe(const UserInfo &userInfo, InodeId inodeId, uint32_t stripe);

  CoTryTask<Void> testRpc();

 private:
  CoTryTask<Inode> openCreate(auto func, auto req);

  CoTryTask<void> removeImpl(RemoveReq &req);

  struct CloseTask {
    std::variant<CloseReq, PruneSessionReq> req;
    Duration backoff;

    std::string describe() const {
      return fmt::format("{{{}, backoff {}}}",
                         std::visit([](const auto &req) -> std::string { return fmt::format("{}", req); }, req),
                         backoff);
    }
  };

  class PrunSessionBatch {
    SteadyTime lastTime;
    std::vector<SessionId> sessions;

   public:
    std::vector<SessionId> take(size_t batchSize, Duration batchDur) {
      if (sessions.size() < batchSize && (sessions.empty() || SteadyClock::now() - lastTime < batchDur)) {
        return {};
      }
      return std::exchange(sessions, {});
    }

    std::vector<SessionId> push(SessionId session, size_t batchSize) {
      if (sessions.empty()) {
        lastTime = SteadyClock::now();
        sessions.reserve(batchSize);
      }
      sessions.push_back(session);

      if (sessions.size() >= batchSize) {
        return std::exchange(sessions, {});
      }
      return {};
    }
  };

  CoTryTask<Inode> tryClose(const CloseReq &req, bool &needRetry);
  CoTryTask<void> tryPrune(const PruneSessionReq &req, bool &needRetry);
  CoTryTask<void> updateLayout(Layout &layout);

  CoTask<void> pruneSession(SessionId session);
  void enqueueCloseTask(CloseTask task);
  CoTask<void> scanCloseTask();
  CoTask<void> runCloseTask(CloseTask task);

  folly::Synchronized<PrunSessionBatch, std::mutex> batchPrune_;
  std::unique_ptr<BackgroundRunner> bgRunner_;
  std::unique_ptr<CoroutinesPool<CloseTask>> bgCloser_;
  folly::Synchronized<std::multimap<SteadyTime, CloseTask>, std::mutex> bgCloseTasks_;

  CoTryTask<Inode> truncateImpl(const UserInfo &userInfo, const Inode &inode, size_t targetLength);

 private:
  struct ServerNode {
    ServerSelectionStrategy::NodeInfo node;
    StubFactory::Stub stub;
    size_t failure;

    ServerNode(ServerSelectionStrategy::NodeInfo node, StubFactory::Stub stub)
        : node(std::move(node)),
          stub(std::move(stub)),
          failure(0) {}
  };
  folly::Synchronized<std::set<flat::NodeId>> errNodes_;
  CoTryTask<Stub> getStub();
  CoTryTask<ServerNode> getServerNode();
  CoTask<void> checkServers();

  template <typename Func, typename Req>
  auto retry(Func &&func, Req &&req)
      -> std::invoke_result_t<Func, Stub::IStub, Req &&, const net::UserRequestOptions &, serde::Timestamp *> {
    co_return co_await retry(func, req, config_.retry_default());
  }
  template <typename Func, typename Req>
  auto retry(Func &&func, Req &&req, RetryConfig retryConfig, std::function<void(const Status &)> onError = {})
      -> std::invoke_result_t<Func, Stub::IStub, Req &&, const net::UserRequestOptions &, serde::Timestamp *>;

  template <typename Rsp>
  CoTryTask<Void> waitRoutingInfo(const Rsp &rsp, const RetryConfig &retryConfig);

 private:
  [[maybe_unused]] ClientId clientId_;
  const Config &config_;
  const bool dynStripe_;  // only fuse client support dynamic stripe.
  std::unique_ptr<StubFactory> factory_;
  std::shared_ptr<ICommonMgmtdClient> mgmtd_;
  std::shared_ptr<storage::client::StorageClient> storage_;

 private:
  folly::atomic_shared_ptr<ServerSelectionStrategy> serverSelection_;
  std::unique_ptr<ConfigCallbackGuard> onConfigUpdated_;
  Semaphore concurrentReqSemaphore_;
};
}  // namespace hf3fs::meta::client

#pragma once

#include <chrono>
#include <cstdint>
#include <fmt/chrono.h>
#include <folly/Math.h>
#include <folly/io/async/HHWheelTimer.h>

#include "common/utils/Duration.h"

namespace hf3fs {
class UtcTime;
// UtcClock is similar to std::chrono::system_clock except that it's based on UTC.
// UtcClock only provides microsecond-level timestamps.
class UtcClock {
 public:
  using rep = int64_t;
  using period = std::ratio<1, 1000000>;
  using duration = std::chrono::duration<rep, period>;
  using time_point = UtcTime;
  static constexpr bool is_steady = false;

  static time_point now() noexcept;

  static std::time_t secondsSinceEpoch() noexcept;

  static std::time_t to_time_t(const time_point &t) noexcept;
};

class UtcTime : public std::chrono::time_point<UtcClock> {
  using Base = std::chrono::time_point<UtcClock>;

 public:
  using Base::Base;
  using is_serde_copyable = void;
  constexpr UtcTime()
      : Base(std::chrono::microseconds(0)) {}
  constexpr UtcTime(const Base &base)
      : Base(base) {}

  static constexpr UtcTime from(std::chrono::microseconds us) { return UtcTime(us); }
  static constexpr UtcTime fromMicroseconds(int64_t us) { return UtcTime(std::chrono::microseconds(us)); }
  constexpr int64_t toMicroseconds() const {
    return std::chrono::duration_cast<std::chrono::microseconds>(time_since_epoch()).count();
  }

  UtcTime castGranularity(Duration gran) const {
    auto us = toMicroseconds();
    auto res = std::max((int64_t)1, gran.asUs().count());
    return UtcTime::fromMicroseconds(us / res * res);
  }

  static UtcTime now() noexcept;

  std::string serdeToReadable() const;

  std::string YmdHMS() const;

  bool isZero() const;
};

using SteadyClock = std::chrono::steady_clock;
using SteadyTime = SteadyClock::time_point;
}  // namespace hf3fs

FMT_BEGIN_NAMESPACE

template <>
struct formatter<hf3fs::UtcTime> : formatter<std::tm> {
  FMT_CONSTEXPR formatter() { this->format_str = "%F %T"; }

  template <typename FormatContext>
  auto format(hf3fs::UtcTime val, FormatContext &ctx) const -> decltype(ctx.out()) {
    return formatter<std::tm>::format(localtime(hf3fs::UtcClock::to_time_t(val)), ctx);
  }
};

FMT_END_NAMESPACE

#pragma once

#include <folly/Likely.h>
#include <folly/ThreadLocal.h>

#include "common/net/Transport.h"
#include "common/utils/RobinHood.h"
#include "common/utils/Shards.h"

namespace hf3fs::net {

class IOWorker;

// A set of transports to same address for load balance.
class TransportSet {
 public:
  static constexpr uint32_t kDefaultMaxConnections = 1;

  // return a transport randomly.
  TransportPtr acquire(uint32_t idx);

  // add a new transport to this set.
  void add(TransportPtr tr, uint32_t idx);

  // remove a transport from this set.
  bool remove(TransportPtr tr);

  // get all transports.
  auto &transports() { return transports_; }

  // check all transports.
  uint32_t checkAll(Duration expiredTime);

  // drop all transports.
  uint32_t dropAll();

 protected:
  using TransportMap = std::map<TransportPtr, uint32_t>;

  void ensureSize(uint32_t idx);

  TransportMap::iterator erase(TransportMap::iterator it);

 private:
  TransportMap transports_{};
  std::vector<TransportMap::iterator> idxToTransport_{kDefaultMaxConnections, transports_.end()};
};

struct TransportCacheKey {
  Address addr;
  uint32_t idx;
  bool operator==(const TransportCacheKey &o) const { return addr == o.addr && idx == o.idx; }
};

}  // namespace hf3fs::net

template <>
struct std::hash<hf3fs::net::TransportCacheKey> {
  auto operator()(const hf3fs::net::TransportCacheKey &key) const {
    return std::hash<hf3fs::net::Address>{}(key.addr) ^ key.idx;
  }
};

namespace hf3fs::net {

// Maintain a map of addresses and transport.
class TransportPool {
 public:
  class Config : public ConfigBase<Config> {
    CONFIG_HOT_UPDATED_ITEM(max_connections, TransportSet::kDefaultMaxConnections);
  };
  TransportPool(const Config &config)
      : config_(config) {}

  ~TransportPool();

  // add a transport into pool. [thread-safe]
  void add(TransportPtr tr);

  // remove a transport from pool. [thread-safe]
  void remove(TransportPtr tr);

  // get the transport corresponding to the specified address. [thread-safe]
  // the bool value in pair indicates whether connecting is required.
  std::pair<TransportPtr, bool> get(Address addr, IOWorker &io_worker);

  // drop all connections.
  void dropConnections(bool dropAll = true, bool dropIncome = false);

  // drop all connections to this addr.
  void dropConnections(Address addr);

  // drop connetions to peer.
  void checkConnections(Address addr, Duration expiredTime);

 private:
  const Config &config_;

  // sharding by address for better performance.
  constexpr static auto kShardsSize = 32u;
  using Map = robin_hood::unordered_map<Address, TransportSet>;
  Shards<Map, kShardsSize> shards_;

  // thread local cache for better performance.
  folly::ThreadLocal<robin_hood::unordered_map<TransportCacheKey, std::weak_ptr<Transport>>> tlsCache_;
};

}  // namespace hf3fs::net

#pragma once

#include "common/utils/ConfigBase.h"
#include "common/utils/LockManager.h"
#include "common/utils/RobinHood.h"
#include "common/utils/UtcTime.h"
#include "fbs/core/user/User.h"

namespace hf3fs::core {
using flat::Gid;
using flat::Uid;
using flat::UserAttr;
using flat::UserInfo;

class UserCache {
 public:
  struct Config : ConfigBase<Config> {
    CONFIG_ITEM(buckets, 127u, ConfigCheckers::isPositivePrime<uint32_t>);
    CONFIG_HOT_UPDATED_ITEM(exist_ttl, 5_min);
    CONFIG_HOT_UPDATED_ITEM(inexist_ttl, 10_s);
  };

  explicit UserCache(const Config &cfg)
      : cfg_(cfg),
        lockManager_(cfg.buckets()),
        buckets_(cfg.buckets()) {}

  struct GetResult {
    std::optional<UserAttr> attr;
    bool marked = false;

    static GetResult unmarkedNotExist() { return {std::nullopt, false}; }
    static GetResult markedNotExist() { return {std::nullopt, true}; }
    static GetResult exist(UserAttr attr) { return {std::move(attr), false}; }
  };

  GetResult get(Uid id) {
    auto lock = lockManager_.lock(id);
    auto now = SteadyClock::now();
    auto &bucket = buckets_[lockManager_.idx(id)];

    auto it = bucket.find(id);
    if (it == bucket.end()) return GetResult::unmarkedNotExist();

    auto &entry = it->second;
    if (expired(entry, now)) {
      bucket.erase(it);
      return GetResult::unmarkedNotExist();
    }

    if (entry.attr)
      return GetResult::exist(*entry.attr);
    else
      return GetResult::markedNotExist();
  }

  void set(Uid id, std::optional<UserAttr> attr) {
    auto lock = lockManager_.lock(id);
    auto &bucket = buckets_[lockManager_.idx(id)];
    auto &entry = bucket[id];
    auto now = SteadyClock::now();
    entry.setTs = now;
    entry.attr = std::move(attr);
  }

  void clear(Uid id) {
    auto lock = lockManager_.lock(id);
    auto &bucket = buckets_[lockManager_.idx(id)];
    bucket.erase(id);
  }

  void clear() {
    for (uint32_t i = 0; i < lockManager_.numBuckets(); ++i) {
      auto lock = lockManager_.lock_at(i);
      auto &bucket = buckets_[i];
      bucket.clear();
    }
  }

  void clearRetired() {
    std::vector<Uid> uids;
    for (uint32_t i = 0; i < lockManager_.numBuckets(); ++i) {
      auto lock = lockManager_.lock_at(i);
      auto &bucket = buckets_[i];
      auto now = SteadyClock::now();
      for (const auto &[id, entry] : bucket) {
        if (expired(entry, now)) uids.push_back(id);
      }
    }
    if (!uids.empty()) {
      for (uint32_t i = 0; i < lockManager_.numBuckets(); ++i) {
        auto lock = lockManager_.lock_at(i);
        auto &bucket = buckets_[i];
        auto now = SteadyClock::now();
        for (auto id : uids) {
          auto it = bucket.find(id);
          if (expired(it->second, now)) bucket.erase(it);
        }
      }
    }
  }

 private:
  struct Entry {
    std::optional<UserAttr> attr;
    SteadyTime setTs;
  };

  bool expired(const Entry &entry, SteadyTime now) const {
    auto timeout = entry.attr ? cfg_.exist_ttl().asUs() : cfg_.inexist_ttl().asUs();
    return now - entry.setTs >= timeout;
  }

  using Bucket = robin_hood::unordered_node_map<Uid, Entry>;

  const Config &cfg_;
  UniqueLockManager lockManager_;
  std::vector<Bucket> buckets_;
};
}  // namespace hf3fs::core

#pragma once

#include <folly/Range.h>

#include "common/utils/Result.h"
#include "fbs/storage/Common.h"

namespace hf3fs::storage {

class ChunkFileView {
 public:
  // read a piece of data.
  Result<uint32_t> read(uint8_t *buf, size_t size, size_t offset, bool direct = false) const;

  // write a piece of data.
  Result<uint32_t> write(const uint8_t *buf, size_t size, size_t offset, const ChunkMetadata &meta);

  // calculate the chunk checksum
  Result<ChecksumInfo> checksum(ChecksumType type, size_t size, size_t offset, const ChunkMetadata &meta);

  // get direct fd for aio read.
  int directFD() const { return direct_; }

  // get fd index in list.
  auto &index() const { return index_; }

 private:
  friend class ChunkFileStore;
  int normal_;
  int direct_;
  std::optional<uint32_t> index_{};
};

class ChunkDataIterator : public ChecksumInfo::DataIterator {
 public:
  ChunkDataIterator(ChunkFileView &chunkFile, size_t length, size_t offset)
      : data_((uint8_t *)memory::memalign(kAIOAlignSize, ChecksumInfo::kChunkSize)),
        chunkFile_(chunkFile),
        length_(length),
        offset_(offset) {}

  ~ChunkDataIterator() override { memory::deallocate(data_); }

  std::pair<const uint8_t *, size_t> next() override;

 private:
  uint8_t *data_;
  ChunkFileView &chunkFile_;
  size_t length_;
  size_t offset_;
};

}  // namespace hf3fs::storage

#pragma once

#include <folly/experimental/coro/BlockingWait.h>
#include <fstream>
#define OPENSSL_SUPPRESS_DEPRECATED
#include <openssl/md5.h>
#include <optional>
#include <sstream>

#include "client/cli/admin/AdminEnv.h"
#include "client/cli/admin/Layout.h"
#include "client/cli/common/Dispatcher.h"
#include "client/cli/common/Utils.h"
#include "client/storage/StorageClient.h"
#include "client/storage/TargetSelection.h"
#include "common/serde/Serde.h"
#include "common/utils/Result.h"
#include "fbs/meta/Common.h"
#include "fbs/storage/Common.h"

namespace hf3fs::client::cli {

struct Block {
  SERDE_STRUCT_FIELD(chainId, storage::ChainId{});
  SERDE_STRUCT_FIELD(chunkId, storage::ChunkId{});
  SERDE_STRUCT_FIELD(offset, uint64_t{});
  SERDE_STRUCT_FIELD(length, uint64_t{});
};

class FileWrapper {
 public:
  FileWrapper(AdminEnv &env)
      : env_(env) {}

  auto &file() { return inode_.asFile(); }
  auto &file() const { return inode_.asFile(); }
  auto length() const { return file().length; }

  auto truncate(size_t end) { return env_.metaClientGetter()->truncate(env_.userInfo, inode_.id, end); }
  auto sync() { return env_.metaClientGetter()->sync(env_.userInfo, inode_.id, false, true, std::nullopt); }

  Result<std::vector<Block>> prepareBlocks(size_t offset, size_t end, const flat::RoutingInfo &routingInfo);
  Result<uint32_t> replicasNum();
  Result<Void> showChunks(size_t offset, size_t length);

  CoTryTask<storage::ChecksumInfo> readFile(
      std::ostream &out,
      size_t length,
      size_t offset,
      bool checksum = true,
      bool hex = false,
      storage::client::TargetSelectionMode mode = storage::client::TargetSelectionMode::Default,
      MD5_CTX *md5 = nullptr,
      bool fillZero = false,
      bool verbose = false,
      uint32_t targetIndex = 0);

  static CoTryTask<storage::ChecksumInfo> readFile(
      AdminEnv &env,
      std::ostream &out,
      const std::vector<Block> &blocks,
      bool checksum = true,
      bool hex = false,
      storage::client::TargetSelectionMode mode = storage::client::TargetSelectionMode::Default,
      MD5_CTX *md5 = nullptr,
      bool fillZero = false,
      bool verbose = false,
      uint32_t targetIndex = 0);

  CoTryTask<storage::ChecksumInfo> writeFile(std::istream &in,
                                             size_t length,
                                             size_t offset,
                                             Duration timeout,
                                             bool syncAfterWrite = true);

  static CoTryTask<FileWrapper> openOrCreateFile(AdminEnv &env, const Path &src, bool createIsMissing = false);

 private:
  AdminEnv &env_;
  SERDE_CLASS_FIELD(inode, meta::Inode{});
};

}  // namespace hf3fs::client::cli

#pragma once

#include <iostream>

#include "AppInfo.h"
#include "ApplicationBase.h"
#include "Utils.h"
#include "common/logging/LogInit.h"
#include "common/net/Server.h"
#include "common/net/ib/IBDevice.h"
#include "common/utils/LogCommands.h"
#include "common/utils/SysResource.h"

DECLARE_string(app_cfg);
DECLARE_bool(dump_default_app_cfg);

DECLARE_string(cfg);
DECLARE_bool(dump_cfg);
DECLARE_bool(dump_default_cfg);

namespace hf3fs {
template <class T>
requires requires {
  typename T::Config;
  std::string_view(T::kName);
}
class OnePhaseApplication : public ApplicationBase {
 public:
  class CommonConfig : public ConfigBase<CommonConfig> {
    CONFIG_ITEM(cluster_id, "");
    CONFIG_OBJ(log, logging::LogConfig);
    CONFIG_OBJ(monitor, monitor::Monitor::Config);
    CONFIG_OBJ(ib_devices, net::IBDevice::Config);
  };

  class Config : public ConfigBase<Config> {
   public:
    CONFIG_OBJ(common, CommonConfig);
    CONFIG_OBJ(server, typename T::Config);
  };

  struct AppConfig : public ConfigBase<AppConfig> {
    CONFIG_ITEM(node_id, 0);
    CONFIG_ITEM(allow_empty_node_id, true);
  };

  OnePhaseApplication(AppConfig &appConfig, Config &config)
      : appConfig_(appConfig),
        config_(config) {}

  static OnePhaseApplication &instance() {
    static AppConfig appConfig;
    static Config config;
    static OnePhaseApplication app(appConfig, config);
    return app;
  }

  Result<Void> parseFlags(int *argc, char ***argv) final {
    static constexpr std::string_view appConfigPrefix = "--app_config.";
    static constexpr std::string_view configPrefix = "--config.";

    RETURN_ON_ERROR(ApplicationBase::parseFlags(appConfigPrefix, argc, argv, appConfigFlags_));
    RETURN_ON_ERROR(ApplicationBase::parseFlags(configPrefix, argc, argv, configFlags_));
    return Void{};
  }

  Result<Void> initApplication() final {
    if (!FLAGS_app_cfg.empty()) {
      app_detail::initConfigFromFile(appConfig_, FLAGS_app_cfg, FLAGS_dump_default_app_cfg, appConfigFlags_);
    }
    XLOGF_IF(FATAL, !appConfig_.allow_empty_node_id() && appConfig_.node_id() == 0, "node_id is not allowed to be 0");

    if (!FLAGS_cfg.empty()) {
      app_detail::initConfigFromFile(config_, FLAGS_cfg, FLAGS_dump_default_cfg, configFlags_);
    }

    if (FLAGS_dump_cfg) {
      std::cout << config_.toString() << std::endl;
      exit(0);
    }

    // init basic components
    auto ibResult = net::IBManager::start(config_.common().ib_devices());
    XLOGF_IF(FATAL, !ibResult, "Failed to start IBManager: {}", ibResult.error());

    auto logConfigStr = logging::generateLogConfig(config_.common().log(), String(T::kName));
    XLOGF(INFO, "LogConfig: {}", logConfigStr);
    logging::initOrDie(logConfigStr);
    XLOGF(INFO, "{}", VersionInfo::full());
    XLOGF(INFO, "Full AppConfig:\n{}", config_.toString());
    XLOGF(INFO, "Full Config:\n{}", config_.toString());

    XLOGF(INFO, "Init waiter singleton {}", fmt::ptr(&net::Waiter::instance()));

    auto monitorResult = monitor::Monitor::start(config_.common().monitor());
    XLOGF_IF(FATAL, !monitorResult, "Parse config file from flags failed: {}", monitorResult.error());

    // init server and node info.
    server_ = std::make_unique<T>(config_.server());
    auto setupResult = server_->setup();
    XLOGF_IF(FATAL, !setupResult, "Setup server failed: {}", setupResult.error());

    auto hostnameResult = SysResource::hostname(/*physicalMachineName=*/true);
    XLOGF_IF(FATAL, !hostnameResult, "Get hostname failed: {}", hostnameResult.error());

    auto podnameResult = SysResource::hostname(/*physicalMachineName=*/false);
    XLOGF_IF(FATAL, !podnameResult, "Get podname failed: {}", podnameResult.error());

    info_.nodeId = flat::NodeId(appConfig_.node_id());
    info_.clusterId = config_.common().cluster_id();
    info_.hostname = *hostnameResult;
    info_.podname = *podnameResult;
    info_.pid = SysResource::pid();
    info_.releaseVersion = flat::ReleaseVersion::fromVersionInfo();
    for (auto &group : server_->groups()) {
      info_.serviceGroups.emplace_back(group->serviceNameList(), group->addressList());
    }
    XLOGF(INFO, "{}", server_->describe());

    // 5. start server.
    auto startResult = server_->start(info_);
    XLOGF_IF(FATAL, !startResult, "Start server failed: {}", startResult.error());

    return Void{};
  }

  config::IConfig *getConfig() final { return &config_; }

  const flat::AppInfo *info() const final { return &info_; }

  void stop() final { stopAndJoin(server_.get()); }

 private:
  OnePhaseApplication() = default;

  ConfigFlags appConfigFlags_;
  ConfigFlags configFlags_;

  AppConfig &appConfig_;
  Config &config_;
  flat::AppInfo info_;
  std::unique_ptr<net::Server> server_;
};

}  // namespace hf3fs

#pragma once

#include <folly/MPMCQueue.h>
#include <folly/ProducerConsumerQueue.h>
#include <folly/Utility.h>
#include <folly/concurrency/PriorityUnboundedQueueSet.h>
#include <folly/experimental/coro/Task.h>
#include <folly/fibers/Semaphore.h>
#include <optional>

#include "common/utils/Semaphore.h"

namespace hf3fs {

/** Like folly::PriorityUnboundedBlockingQueue, but work with coroutine. */
template <typename T>
class PriorityUnboundedQueue {
 public:
  explicit PriorityUnboundedQueue(uint8_t numPriorities)
      : sem_(0),
        queue_(numPriorities) {}

  uint8_t getNumPriorities() { return queue_.priorities(); }

  void addWithPriority(T item, int8_t priority) {
    queue_.at_priority(translatePriority(priority)).enqueue(std::move(item));
    sem_.signal();
  }

  T dequeue() {
    sem_.wait();
    return dequeueImpl();
  }

  std::optional<T> try_dequeue() {
    if (!sem_.try_wait()) {
      return std::nullopt;
    }
    return dequeueImpl();
  }

  folly::coro::Task<T> co_dequeue() {
    co_await sem_.co_wait();
    co_return dequeueImpl();
  }

  size_t size() const { return queue_.size(); }

  size_t sizeGuess() const { return queue_.size(); }

 private:
  size_t translatePriority(int8_t const priority) {
    size_t const priorities = queue_.priorities();
    assert(priorities <= 255);
    int8_t const hi = (priorities + 1) / 2 - 1;
    int8_t const lo = hi - (priorities - 1);
    return hi - folly::constexpr_clamp(priority, lo, hi);
  }

  T dequeueImpl() {
    // must follow a successful sem wait
    if (auto obj = queue_.try_dequeue()) {
      return std::move(*obj);
    }
    folly::terminate_with<std::logic_error>("bug in PriorityUnboundedQueue");
  }

  folly::fibers::Semaphore sem_;
  folly::PriorityUnboundedQueueSet<T, false, false, true /* MayBlock */> queue_;
};

}  // namespace hf3fs

#pragma once

#include "Serde.h"
#include "common/kv/ITransaction.h"

namespace hf3fs::serde {
template <typename T>
struct SerdeHelper {
  using Self = T;

  template <typename... Args>
  static Self create(Args &&...args) {
    return Self{/* base */ {}, std::forward<Args>(args)...};
  }

  CoTryTask<void> store(kv::IReadWriteTransaction &txn, std::string_view key) const {
    auto value = serialize(*static_cast<const T *>(this));
    co_return co_await txn.set(key, value);
  }

  static CoTryTask<std::optional<T>> load(kv::IReadOnlyTransaction &txn, std::string_view key) {
    co_return unpackFrom(co_await txn.get(key));
  }

  static CoTryTask<std::optional<T>> snapshotLoad(kv::IReadOnlyTransaction &txn, std::string_view key) {
    co_return unpackFrom(co_await txn.snapshotGet(key));
  }

  static Result<T> unpackFrom(std::string_view str) {
    auto o = defaultValue();
    RETURN_ON_ERROR(deserialize(o, str));
    return o;
  }

  static Result<T> unpackFrom(const String &str) { return unpackFrom(std::string_view(str)); }

  static Result<std::optional<T>> unpackFrom(const Result<std::optional<String>> &getResult) {
    RETURN_ON_ERROR(getResult);

    if (!getResult->has_value()) {
      return std::nullopt;
    } else {
      auto o = defaultValue();
      RETURN_ON_ERROR(deserialize(o, getResult->value()));
      return o;
    }
  }

  static T defaultValue() {
    if constexpr (requires { T::defaultValueForLoad(); }) {
      return T::defaultValueForLoad();
    } else {
      return T{};
    }
  }
};
}  // namespace hf3fs::serde

#define DEFINE_SERDE_HELPER_STRUCT(name) struct name : public ::hf3fs::serde::SerdeHelper<name>

#pragma once

#include <fmt/core.h>
#include <limits>
#include <string>
#include <string_view>

#include "common/utils/Result.h"

namespace hf3fs {

class Size {
 public:
  static constexpr uint64_t kUnitB = 1ul << 0;
  static constexpr uint64_t kUnitKB = 1ul << 10;
  static constexpr uint64_t kUnitMB = 1ul << 20;
  static constexpr uint64_t kUnitGB = 1ul << 30;
  static constexpr uint64_t kUnitTB = 1ul << 40;

  static constexpr uint64_t kUnitK = 1000ull;
  static constexpr uint64_t kUnitM = 1000'000ull;
  static constexpr uint64_t kUnitG = 1000'000'000ull;
  static constexpr uint64_t kUnitT = 1000'000'000'000ull;

  static constexpr std::string_view kLabelB = "B";
  static constexpr std::string_view kLabelKB = "KB";
  static constexpr std::string_view kLabelMB = "MB";
  static constexpr std::string_view kLabelGB = "GB";
  static constexpr std::string_view kLabelTB = "TB";

  static constexpr std::string_view kLabelK = "K";
  static constexpr std::string_view kLabelM = "M";
  static constexpr std::string_view kLabelG = "G";
  static constexpr std::string_view kLabelT = "T";

  using is_serde_copyable = void;

  // allow implicit conversion
  constexpr Size(uint64_t value = 0) noexcept
      : value_(value) {}
  constexpr operator uint64_t() const { return value_; }
  constexpr uint64_t toInt() const { return value_; }
  std::string toString() const { return toString(value_); }
  std::string serdeToReadable() const { return toString(); }
  static Result<Size> serdeFromReadable(std::string_view input) { return from(input); }
  std::string around() const { return around(value_); }

  static Result<Size> from(std::string_view input);
  static std::string toString(uint64_t s);
  static std::string around(uint64_t s);

  static constexpr Size infinity() { return std::numeric_limits<uint64_t>::max(); }

 private:
  uint64_t value_ = 0;
};

inline constexpr Size operator""_B(unsigned long long value) { return Size(value * Size::kUnitB); }
inline constexpr Size operator""_KB(unsigned long long value) { return Size(value * Size::kUnitKB); }
inline constexpr Size operator""_MB(unsigned long long value) { return Size(value * Size::kUnitMB); }
inline constexpr Size operator""_GB(unsigned long long value) { return Size(value * Size::kUnitGB); }
inline constexpr Size operator""_TB(unsigned long long value) { return Size(value * Size::kUnitTB); }

inline constexpr Size operator""_K(unsigned long long value) { return Size(value * Size::kUnitK); }
inline constexpr Size operator""_M(unsigned long long value) { return Size(value * Size::kUnitM); }
inline constexpr Size operator""_G(unsigned long long value) { return Size(value * Size::kUnitG); }
inline constexpr Size operator""_T(unsigned long long value) { return Size(value * Size::kUnitT); }

}  // namespace hf3fs

FMT_BEGIN_NAMESPACE

template <>
struct formatter<hf3fs::Size> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(const hf3fs::Size &size, FormatContext &ctx) const {
    return formatter<std::string_view>::format(size.toString(), ctx);
  }
};

FMT_END_NAMESPACE

class LevelDBLogger {
  public:
    ~LevelDBLogger() final = default;
    void Logv(const char *format, std::va_list ap) final {
  static constexpr int fixedBufferSize = 1024;
  thread_local char fixedBuffer[fixedBufferSize];

  // ap can only be used once, we have to prepare a copy of ap for both paths
  std::va_list apCopy;
  va_copy(apCopy, ap);
  auto count = std::vsnprintf(fixedBuffer, fixedBufferSize, format, apCopy);
  va_end(apCopy);

  if (count < 0) {
    XLOGF(WARN, "[LevelDB] Print log from LevelDB failed. ret of vsnprintf is {}", count);
  } else if (count < fixedBufferSize) {
    // fit into fixedBuffer, note that the terminating '\0' will occupy one byte but not counted in the return value.
    if (count >= 1 && fixedBuffer[count - 1] == '\n') {
      --count;
    }
    XLOGF(INFO, "[LevelDB] {}", std::string_view{fixedBuffer, static_cast<size_t>(count)});
  } else {
    // not fit into fixedBuffer, allocate a dynamic buffer large enough for the content and the terminating '\0'
    auto dynamicBuffer = std::make_unique<char[]>(count + 1);
    std::vsnprintf(dynamicBuffer.get(), count + 1, format, ap);
    if (count >= 1 && dynamicBuffer.get()[count - 1] == '\n') {
      --count;
    }
    XLOGF(INFO, "[LevelDB] {}", std::string_view{dynamicBuffer.get(), static_cast<size_t>(count)});
  }
}
};

#pragma once

#include <boost/lexical_cast.hpp>
#include <boost/lexical_cast/bad_lexical_cast.hpp>
#include <boost/uuid/nil_generator.hpp>
#include <boost/uuid/random_generator.hpp>
#include <boost/uuid/uuid.hpp>
#include <boost/uuid/uuid_io.hpp>
#include <cstring>
#include <optional>
#include <string>
#include <string_view>

#include "common/utils/Result.h"
#include "common/utils/String.h"

namespace hf3fs {
struct Uuid : boost::uuids::uuid {
  using is_serde_copyable = void;
  static thread_local boost::uuids::random_generator uuidGenerator;

  static constexpr Uuid zero() { return {{0}}; }
  static Uuid random() { return {uuidGenerator()}; }
  static Uuid max() {
    Uuid uuid;
    memset(uuid.data, 0xff, sizeof(uuid.data));
    return uuid;
  }

  static Uuid from(uint64_t low, uint64_t high) {
    Uuid uuid;
    memcpy(uuid.data, &low, sizeof(low));
    memcpy(uuid.data + sizeof(low), &high, sizeof(high));
    return uuid;
  }

  String toHexString() const { return boost::lexical_cast<String>(*this); }
  std::string serdeToReadable() const { return toHexString(); }

  std::string_view asStringView() const { return std::string_view(reinterpret_cast<const char *>(data), sizeof(data)); }

  static Result<Uuid> fromHexString(std::string_view str) {
    try {
      auto uuid = boost::lexical_cast<boost::uuids::uuid>(str);
      return Uuid{uuid};
    } catch (boost::bad_lexical_cast &ex) {
      return makeError(StatusCode::kDataCorruption, ex.what());
    }
  }

  static Result<Uuid> serdeFromReadable(std::string_view str) { return fromHexString(str); }
};
}  // namespace hf3fs

template <>
struct std::hash<hf3fs::Uuid> {
  size_t operator()(const hf3fs::Uuid &uuid) const;
};

FMT_BEGIN_NAMESPACE

template <>
struct formatter<hf3fs::Uuid> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(const hf3fs::Uuid &uuid, FormatContext &ctx) const {
    return formatter<std::string_view>::format(uuid.toHexString(), ctx);
  }
};

FMT_END_NAMESPACE

#include <atomic>
#include <folly/experimental/coro/Sleep.h>

#include "MgmtdConfig.h"
#include "MgmtdOperator.h"
#include "MgmtdState.h"
#include "common/kv/WithTransaction.h"
#include "core/utils/ServiceOperation.h"

#define RECORD_LATENCY(latency)                                                                                \
  auto FB_ANONYMOUS_VARIABLE(guard) = folly::makeGuard([lat = &(latency), begin = std::chrono::steady_clock::now()] { \
    lat->addSample(std::chrono::steady_clock::now() - begin);                                                         \
  })

namespace hf3fs::mgmtd {
template <typename T>
inline T nextVersion(T version) {
  auto v = version + 1;
  return T(v);
}

CoTryTask<Void> ensureSelfIsPrimary(MgmtdState &state);

template <typename Handler>
inline std::invoke_result_t<Handler> doAsPrimary(MgmtdState &state, Handler &&handler) {
  auto ret = co_await ensureSelfIsPrimary(state);
  CO_RETURN_ON_ERROR(ret);
  co_return co_await handler();
}

void updateMemoryRoutingInfo(RoutingInfo &alreadyLockedRoutingInfo, core::ServiceOperation &ctx);

template <typename Handler>
inline CoTask<void> updateMemoryRoutingInfo(MgmtdState &state, core::ServiceOperation &ctx, Handler &&handler) {
  auto dataPtr = co_await state.data_.coLock();
  auto &ri = dataPtr->routingInfo;
  updateMemoryRoutingInfo(ri, ctx);
  handler(ri);
}

CoTask<void> updateMemoryRoutingInfo(MgmtdState &state, core::ServiceOperation &ctx);

template <typename Handler, typename Result = std::invoke_result_t<Handler, kv::IReadWriteTransaction &>>
inline Result withReadWriteTxn(MgmtdState &state, Handler &&handler, bool expectSelfPrimary = true) {
  int maxRetryTimes = state.config_.retry_times_on_txn_errors();
  constexpr auto retryInterval = std::chrono::milliseconds(1000);
  auto strategy = state.createRetryStrategy();
  for (int i = 0;; ++i) {
    auto res = co_await kv::WithTransaction(strategy).run(
        state.env_->kvEngine()->createReadWriteTransaction(),
        [&](kv::IReadWriteTransaction &txn) -> Result {
          if (expectSelfPrimary && state.config_.validate_lease_on_write()) {
            CO_RETURN_ON_ERROR(co_await state.store_.ensureLeaseValid(txn, state.selfId(), state.utcNow()));
          }
          co_return co_await handler(txn);
        });
    if (res || i == maxRetryTimes || StatusCode::typeOf(res.error().code()) != StatusCodeType::Transaction)
      co_return res;
    XLOGF(CRITICAL, "Transaction failed: {}\nretryCount: {}", res.error(), i);
    co_await folly::coro::sleep(retryInterval);
  }
}

template <typename Handler, typename Result = std::invoke_result_t<Handler, kv::IReadOnlyTransaction &>>
inline Result withReadOnlyTxn(MgmtdState &state, Handler &&handler) {
  auto strategy = state.createRetryStrategy();
  co_return co_await kv::WithTransaction(strategy).run(
      state.env_->kvEngine()->createReadonlyTransaction(),
      [&](kv::IReadOnlyTransaction &txn) -> Result { co_return co_await handler(txn); });
}

template <typename Handler>
inline CoTryTask<void> updateStoredRoutingInfo(MgmtdState &state, core::ServiceOperation &ctx, Handler &&handler) {
  auto dataPtr = co_await state.data_.coSharedLock();
  auto nextv = nextVersion(dataPtr->routingInfo.routingInfoVersion);
  co_return co_await withReadWriteTxn(state, [&](kv::IReadWriteTransaction &txn) -> CoTryTask<void> {
    CO_RETURN_ON_ERROR(co_await state.store_.storeRoutingInfoVersion(txn, nextv));
    LOG_OP_INFO(ctx, "RoutingInfo: bump storage version to {}", nextv);
    co_return co_await handler(txn);
  });
}

CoTryTask<void> updateStoredRoutingInfo(MgmtdState &state, core::ServiceOperation &ctx);

flat::TargetInfo makeTargetInfo(flat::ChainId chainId, const flat::ChainTargetInfo &info);

Result<Void> updateSelfConfig(MgmtdState &state, const flat::ConfigInfo &cfg);

using MgmtdStub = mgmtd::IMgmtdServiceStub;
using MgmtdStubFactory = stubs::IStubFactory<MgmtdStub>;

Result<std::vector<flat::TagPair>> updateTags(core::ServiceOperation &op,
                                              flat::SetTagMode mode,
                                              const std::vector<flat::TagPair> &oldTags,
                                              const std::vector<flat::TagPair> &updates);
}  // namespace hf3fs::mgmtd

#pragma once

#include <chrono>
#include <folly/experimental/coro/Sleep.h>

#include "common/kv/ITransaction.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Result.h"
#include "common/utils/Status.h"
#include "common/utils/UtcTime.h"
#include "fdb/FDBTransaction.h"

namespace hf3fs {
struct CoroSleeper {
  CoTask<void> operator()(std::chrono::milliseconds d) const { co_await folly::coro::sleep(d); }
};

struct RetryConfig {
  using TimeUnit = std::chrono::milliseconds;
  TimeUnit minRetryInterval{0};
  TimeUnit maxRetryInterval{0};
  size_t maxRetryCount{0};
};

struct RetryState {
  using TimeUnit = std::chrono::milliseconds;

  const RetryConfig config;
  TimeUnit retryInterval;
  size_t retryCount = 0;

  RetryState(const RetryConfig &config)
      : config(config),
        retryInterval(config.minRetryInterval) {}

  bool canRetry() const { return retryCount < config.maxRetryCount; }

  void reset() {
    retryInterval = config.minRetryInterval;
    retryCount = 0;
  }

  void next() {
    ++retryCount;
    retryInterval = std::min(2 * retryInterval, config.maxRetryInterval);
  }
};

template <typename Sleeper = CoroSleeper>
class DefaultRetryStrategy {
 public:
  using TimeUnit = std::chrono::milliseconds;
  DefaultRetryStrategy(RetryConfig config, Sleeper sleeper = Sleeper{})
      : state_(config),
        sleeper_(std::move(sleeper)) {}

  template <typename Txn>
  Result<Void> init(Txn * /* txn */) {
    state_.reset();
    return Void{};
  }

  template <typename Txn>
  CoTryTask<void> onError(Txn *txn, Status error) {
    CO_RETURN_ON_ERROR(co_await onError(error));
    txn->reset();
    co_return Void{};
  }

  CoTryTask<void> onError(Status error) {
    if (!kv::TransactionHelper::isRetryable(error, false) || !state_.canRetry()) {
      co_return makeError(std::move(error));
    }
    co_await sleeper_(state_.retryInterval);
    state_.next();
    co_return Void{};
  }

  Sleeper &getSleeper() { return sleeper_; }

 private:
  RetryState state_;
  Sleeper sleeper_;
};
}  // namespace hf3fs

#pragma once

#include <functional>

#include "common/serde/ClientContext.h"
#include "common/utils/RobinHood.h"
#include "stubs/common/Stub.h"

namespace hf3fs::stubs {

template <template <typename Ctx> typename StubImpl, bool SupportsStubStub = false>
class SerdeStubFactory {
 public:
  using Stub = stubs::SerdeStub<StubImpl, SupportsStubStub>;
  using MockStub = typename Stub::MockStub;
  using IStub = typename Stub::IStub;
  using MockContext = typename Stub::MockContext;
  using StubStub = typename Stub::StubStub;

  using SerdeContextCreator = std::function<serde::ClientContext(net::Address)>;
  explicit SerdeStubFactory(SerdeContextCreator creator)
      : ctx_(std::move(creator)) {}
  explicit SerdeStubFactory(serde::ClientMockContext context)
      : ctx_(std::move(context)) {}
  explicit SerdeStubFactory(robin_hood::unordered_map<net::Address, serde::ClientMockContext> contextMap)
      : ctx_(std::move(contextMap)) {}
  explicit SerdeStubFactory(MockContext ctx)
      : ctx_(std::move(ctx)) {}

  Stub create(net::Address addr, NodeId node = NodeId(0), std::string_view host = "mock") {
    struct MakeStub {
      MakeStub(net::Address addr, NodeId node, std::string_view host)
          : addr_(addr),
            node_(node),
            host_(host) {}
      Stub operator()(SerdeContextCreator &creator) { return Stub(creator(addr_), node_, host_); }
      Stub operator()(serde::ClientMockContext &ctx) { return Stub(ctx); }
      Stub operator()(robin_hood::unordered_map<net::Address, serde::ClientMockContext> &map) {
        return Stub(map[addr_], node_, host_);
      }
      Stub operator()(MockContext &ctx) {
        if constexpr (SupportsStubStub) {
          return Stub(ctx);
        } else {
          return Stub{};
        }
      }

     private:
      net::Address addr_;
      NodeId node_;
      std::string_view host_;
    };

    return std::visit(MakeStub(addr, node, host), ctx_);
  }

 private:
  std::conditional_t<SupportsStubStub,
                     std::variant<SerdeContextCreator,
                                  serde::ClientMockContext,
                                  robin_hood::unordered_map<net::Address, serde::ClientMockContext>,
                                  MockContext>,
                     std::variant<SerdeContextCreator,
                                  serde::ClientMockContext,
                                  robin_hood::unordered_map<net::Address, serde::ClientMockContext>>>
      ctx_;
};

}  // namespace hf3fs::stubs

#pragma once

#include <exception>
#include <unistd.h>
#include <utility>

#include "common/utils/Coroutine.h"

namespace hf3fs {

class FdWrapper {
 public:
  static constexpr int kInvalidFd = -1;

  explicit FdWrapper(int fd = kInvalidFd)
      : fd_(fd) {}
  FdWrapper(const FdWrapper &) = delete;
  FdWrapper(FdWrapper &&o)
      : fd_(o.release()) {}
  ~FdWrapper() { close(); }

  // convert to fd implicit.
  operator int() const { return fd_; }
  bool valid() const { return fd_ != kInvalidFd; }

  // get the ownership of other fd.
  FdWrapper &operator=(const FdWrapper &) = delete;
  FdWrapper &operator=(FdWrapper &&o) {
    close();
    fd_ = o.release();
    return *this;
  }
  FdWrapper &operator=(int fd) {
    close();
    fd_ = fd;
    return *this;
  }

  void close() {
    if (valid()) {
      ::close(fd_);
      fd_ = kInvalidFd;
    }
  }

  // release the ownership of current fd.
  int release() { return std::exchange(fd_, kInvalidFd); }

 private:
  int fd_ = kInvalidFd;
};

}  // namespace hf3fs

FMT_BEGIN_NAMESPACE

template <>
struct formatter<hf3fs::FdWrapper> : formatter<int> {
  template <typename FormatContext>
  auto format(const hf3fs::FdWrapper &val, FormatContext &ctx) const {
    return formatter<int>::format(static_cast<int>(val), ctx);
  }
};

FMT_END_NAMESPACE

#pragma once

#include <folly/MPMCQueue.h>
#include <folly/ProducerConsumerQueue.h>
#include <folly/Utility.h>
#include <folly/experimental/coro/Task.h>
#include <folly/fibers/Semaphore.h>

namespace hf3fs {

template <typename T>
class BoundedQueue {
 public:
  explicit BoundedQueue(uint32_t capacity)
      : queue_(capacity),
        enqueueSemaphore_{capacity},
        dequeueSemaphore_{0} {}
  BoundedQueue(const BoundedQueue &) = delete;
  BoundedQueue &operator=(const BoundedQueue &) = delete;

  template <typename U = T>
  void enqueue(U &&item) {
    enqueueSemaphore_.wait();
    queue_.writeIfNotFull(std::forward<U>(item));
    dequeueSemaphore_.signal();
  }

  template <typename U = T>
  folly::coro::Task<void> co_enqueue(U &&item) {
    co_await enqueueSemaphore_.co_wait();
    queue_.writeIfNotFull(std::forward<U>(item));
    dequeueSemaphore_.signal();
  }

  T dequeue() {
    dequeueSemaphore_.wait();
    T item;
    queue_.readIfNotEmpty(item);
    enqueueSemaphore_.signal();
    return item;
  }

  folly::coro::Task<T> co_dequeue() {
    co_await dequeueSemaphore_.co_wait();
    T item;
    queue_.readIfNotEmpty(item);
    enqueueSemaphore_.signal();
    co_return item;
  }

  void dequeue(T &item) {
    dequeueSemaphore_.wait();
    queue_.readIfNotEmpty(item);
    enqueueSemaphore_.signal();
  }

  folly::coro::Task<void> co_dequeue(T &item) {
    co_await dequeueSemaphore_.co_wait();
    queue_.readIfNotEmpty(item);
    enqueueSemaphore_.signal();
  }

  template <typename U = T>
  bool try_enqueue(U &&item) {
    auto waitSuccess = enqueueSemaphore_.try_wait();
    if (!waitSuccess) {
      return false;
    }
    queue_.writeIfNotFull(std::forward<U>(item));
    dequeueSemaphore_.signal();
    return true;
  }

  std::optional<T> try_dequeue() {
    T item;
    if (try_dequeue(item)) {
      return item;
    }
    return std::nullopt;
  }

  bool try_dequeue(T &item) {
    auto waitSuccess = dequeueSemaphore_.try_wait();
    if (!waitSuccess) {
      return false;
    }
    queue_.readIfNotEmpty(item);
    enqueueSemaphore_.signal();
    return true;
  }

  bool empty() const { return queue_.isEmpty(); }
  bool full() const { return queue_.isFull(); }
  size_t size() const { return queue_.size(); }

 private:
  folly::MPMCQueue<T> queue_;
  folly::fibers::Semaphore enqueueSemaphore_;
  folly::fibers::Semaphore dequeueSemaphore_;
};

}  // namespace hf3fs

#pragma once

#include <folly/concurrency/AtomicSharedPtr.h>

#include "client/mgmtd/MgmtdClientForServer.h"
#include "client/storage/StorageMessenger.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/DynamicCoroutinesPool.h"
#include "common/utils/LockManager.h"
#include "common/utils/RobinHood.h"
#include "fbs/storage/Service.h"
#include "storage/aio/AioReadWorker.h"
#include "storage/service/BufferPool.h"
#include "storage/service/StorageOperator.h"
#include "storage/service/TargetMap.h"
#include "storage/store/StorageTargets.h"
#include "storage/sync/ResyncWorker.h"
#include "storage/worker/AllocateWorker.h"
#include "storage/worker/CheckWorker.h"
#include "storage/worker/DumpWorker.h"
#include "storage/worker/PunchHoleWorker.h"
#include "storage/worker/SyncMetaKvWorker.h"

namespace hf3fs::storage {

class ReliableForwarding;

struct Components {
  struct Config : public ConfigBase<Config> {
    CONFIG_OBJ(base, net::Server::Config, [](net::Server::Config &c) {
      c.set_groups_length(2);
      c.groups(0).listener().set_listen_port(8000);
      c.groups(0).set_network_type(net::Address::RDMA);
      c.groups(0).set_services({"StorageSerde"});

      c.groups(1).set_network_type(net::Address::TCP);
      c.groups(1).listener().set_listen_port(9000);
      c.groups(1).set_use_independent_thread_pool(true);
      c.groups(1).set_services({"Core"});

      c.thread_pool().set_num_io_threads(32);
      c.thread_pool().set_num_proc_threads(32);
    });

    CONFIG_OBJ(client, net::Client::Config);
    CONFIG_OBJ(mgmtd, hf3fs::client::MgmtdClientForServer::Config);
    CONFIG_OBJ(targets, StorageTargets::Config);
    CONFIG_OBJ(storage, StorageOperator::Config);
    CONFIG_OBJ(reliable_forwarding, ReliableForwarding::Config);
    CONFIG_OBJ(reliable_update, ReliableUpdate::Config);
    CONFIG_OBJ(buffer_pool, BufferPool::Config);
    CONFIG_OBJ(aio_read_worker, AioReadWorker::Config);
    CONFIG_OBJ(sync_worker, ResyncWorker::Config);
    CONFIG_OBJ(check_worker, CheckWorker::Config);
    CONFIG_OBJ(dump_worker, DumpWorker::Config);
    CONFIG_OBJ(allocate_worker, AllocateWorker::Config);
    CONFIG_OBJ(sync_meta_kv_worker, SyncMetaKvWorker::Config);
    CONFIG_OBJ(forward_client, net::Client::Config);
    CONFIG_OBJ(coroutines_pool_read, DynamicCoroutinesPool::Config);
    CONFIG_OBJ(coroutines_pool_update, DynamicCoroutinesPool::Config);
    CONFIG_OBJ(coroutines_pool_sync, DynamicCoroutinesPool::Config);
    CONFIG_OBJ(coroutines_pool_default, DynamicCoroutinesPool::Config);
    CONFIG_HOT_UPDATED_ITEM(use_coroutines_pool_read, true);
    CONFIG_HOT_UPDATED_ITEM(use_coroutines_pool_update, true);
    CONFIG_HOT_UPDATED_ITEM(speed_up_quit, true);
  };

  Components(const Config &config);

  Result<Void> start(const flat::AppInfo &appInfo, net::ThreadPoolGroup &tpg);
  Result<Void> waitRoutingInfo(const flat::AppInfo &appInfo, folly::CPUThreadPoolExecutor &executor);
  Result<Void> refreshRoutingInfo();
  Result<Void> stopAndJoin(CPUExecutorGroup &executor);
  Result<Void> stopMgmtdClient();
  const flat::AppInfo &getAppInfo() const { return appInfo; }

  Result<robin_hood::unordered_set<std::string>> getActiveClientsList();
  void triggerHeartbeatIfNeed();

  inline DynamicCoroutinesPool &getCoroutinesPool(uint16_t methodId) {
    if (LIKELY(config.use_coroutines_pool_read()) && methodId == StorageSerde<>::batchReadMethodId) {
      return readPool;
    }
    if (LIKELY(config.use_coroutines_pool_update()) &&
        (methodId == StorageSerde<>::writeMethodId || methodId == StorageSerde<>::updateMethodId)) {
      return updatePool;
    }
    if (methodId == StorageSerde<>::syncStartMethodId || methodId == StorageSerde<>::getAllChunkMetadataMethodId) {
      return syncPool;
    }
    return defaultPool;
  }

 protected:
  void updateHeartbeatPayload(const TargetMap &map, bool offline = false);

 public:
  ConstructLog<"storage::Components"> constructLog_;
  const Config &config;
  flat::AppInfo appInfo;
  std::unique_ptr<net::Client> netClient;
  folly::atomic_shared_ptr<hf3fs::client::IMgmtdClientForServer> mgmtdClient;
  BufferPool rdmabufPool;
  AtomicallyTargetMap targetMap;
  StorageTargets storageTargets;
  AioReadWorker aioReadWorker;
  client::StorageMessenger messenger;
  ResyncWorker resyncWorker;
  CheckWorker checkWorker;
  DumpWorker dumpWorker;
  AllocateWorker allocateWorker;
  PunchHoleWorker punchHoleWorker;
  SyncMetaKvWorker syncMetaKvWorker;
  ReliableForwarding reliableForwarding;
  DynamicCoroutinesPool readPool;
  DynamicCoroutinesPool updatePool;
  DynamicCoroutinesPool syncPool;
  DynamicCoroutinesPool defaultPool;
  StorageOperator storageOperator;
  ReliableUpdate reliableUpdate;
  std::atomic<uint32_t> triggerHeartbeatFlag{};
};

}  // namespace hf3fs::storage

#pragma once

#include <folly/sorted_vector_types.h>
#include <map>
#include <optional>
#include <set>
#include <type_traits>
#include <unordered_map>
#include <unordered_set>
#include <variant>
#include <vector>

#include "common/utils/RobinHood.h"

namespace hf3fs {

template <class T, template <class...> class Primary>
struct is_specialization_of : std::false_type {};

template <template <class...> class Primary, class... Args>
struct is_specialization_of<Primary<Args...>, Primary> : std::true_type {};

template <class T, template <class...> class Primary>
inline constexpr bool is_specialization_of_v = is_specialization_of<T, Primary>::value;

template <typename T, template <class...> class Primary>
concept is_specialization = is_specialization_of_v<T, Primary>;

template <class T>
inline constexpr bool is_vector_v = is_specialization_of_v<T, std::vector>;

template <class T>
inline constexpr bool is_set_v = is_specialization_of_v<T, std::set> || is_specialization_of_v<T, std::unordered_set>;

template <class T>
inline constexpr bool is_map_v = is_specialization_of_v<T, std::map> || is_specialization_of_v<T, std::unordered_map> ||
                                 is_specialization_of_v<T, folly::sorted_vector_map>;

template <typename Key, typename Hash, typename KeyEqual, size_t MaxLoad>
inline constexpr bool is_set_v<robin_hood::unordered_set<Key, Hash, KeyEqual, MaxLoad>> = true;

template <typename Key, typename T, typename Hash, typename KeyEqual, size_t MaxLoad>
inline constexpr bool is_map_v<robin_hood::unordered_map<Key, T, Hash, KeyEqual, MaxLoad>> = true;

template <class T>
inline constexpr bool is_bounded_array_v = std::is_bounded_array_v<T>;

template <class T, size_t N>
inline constexpr bool is_bounded_array_v<std::array<T, N>> = true;

template <typename T>
inline constexpr bool is_linear_container_v = is_bounded_array_v<T> || is_vector_v<T> || is_set_v<T>;

template <class T>
inline constexpr bool is_optional_v = is_specialization_of_v<T, std::optional>;

template <class T>
inline constexpr bool is_unique_ptr_v = is_specialization_of_v<T, std::unique_ptr>;

template <class T>
inline constexpr bool is_shared_ptr_v = is_specialization_of_v<T, std::shared_ptr>;

template <class T>
inline constexpr bool is_variant_v = is_specialization_of_v<T, std::variant>;

template <class T>
inline constexpr bool is_pair_v = is_specialization_of_v<T, std::pair>;

template <class T>
struct type_list;
template <class... Ts>
struct type_list<std::variant<Ts...>> {
  using type = std::tuple<Ts...>;
};
template <class T>
using type_list_t = typename type_list<T>::type;

template <class Ret, class... Args>
inline std::tuple<Args...> getFunctionParametersList(Ret (*)(Args...));
template <class T, class Ret, class... Args>
inline std::tuple<Args...> getFunctionParametersList(Ret (T::*)(Args...));
template <class T, class Ret, class... Args>
inline std::tuple<Args...> getFunctionParametersList(Ret (T::*)(Args...) const);
template <auto func>
using function_parameters_t = decltype(getFunctionParametersList(func));
template <auto func>
using function_first_parameter_t = std::decay_t<std::tuple_element_t<0, function_parameters_t<func>>>;

inline auto &getFirstParameter(auto &&first, auto &&...) { return first; }

template <class T, class M>
inline constexpr auto MemberPointerToClass(M T::*) -> T;
template <auto Pointer>
using member_pointer_to_class_t = decltype(MemberPointerToClass(Pointer));

template <class T>
concept GenericPair = requires {
  requires(std::is_same_v<typename T::first_type, decltype(T().first)>);
  requires(std::is_same_v<typename T::second_type, decltype(T().second)>);
};
template <class T>
inline constexpr bool is_generic_pair_v = GenericPair<T>;

template <class T>
concept Container = requires(T a, const T b) {
  requires std::regular<T>;
  requires std::swappable<T>;
  requires std::destructible<typename T::value_type>;
  requires std::forward_iterator<typename T::iterator>;
  requires std::forward_iterator<typename T::const_iterator>;
  { a.begin() } -> std::same_as<typename T::iterator>;
  { a.end() } -> std::same_as<typename T::iterator>;
  { b.begin() } -> std::same_as<typename T::const_iterator>;
  { b.end() } -> std::same_as<typename T::const_iterator>;
  { a.cbegin() } -> std::same_as<typename T::const_iterator>;
  { a.cend() } -> std::same_as<typename T::const_iterator>;
  { a.size() } -> std::same_as<typename T::size_type>;
  { a.max_size() } -> std::same_as<typename T::size_type>;
  { a.empty() } -> std::convertible_to<bool>;
};

template <auto Value>
struct value_identity {
  using type = decltype(Value);
  constexpr static auto value = Value;
};

template <class T>
struct enable_shared_from_this : public std::enable_shared_from_this<T> {
  template <class... Args>
  static std::shared_ptr<T> create(Args &&...args) {
    struct Impl : public T {
      Impl(Args &&...args)
          : T(std::forward<Args>(args)...) {}
    };
    static_assert(sizeof(Impl) == sizeof(T));
    return std::make_shared<Impl>(std::forward<Args>(args)...);
  }
};

template <class T>
constexpr auto callByIdx(auto &&f, size_t idx) {
  switch (idx) {
    case 0:
      if constexpr (0 < std::tuple_size_v<T>) return f(std::tuple_element_t<0, T>{});
    case 1:
      if constexpr (1 < std::tuple_size_v<T>) return f(std::tuple_element_t<1, T>{});
    case 2:
      if constexpr (2 < std::tuple_size_v<T>) return f(std::tuple_element_t<2, T>{});
    case 3:
      if constexpr (3 < std::tuple_size_v<T>) return f(std::tuple_element_t<3, T>{});
    case 4:
      if constexpr (4 < std::tuple_size_v<T>) return f(std::tuple_element_t<4, T>{});
    case 5:
      if constexpr (5 < std::tuple_size_v<T>) return f(std::tuple_element_t<5, T>{});
    case 6:
      if constexpr (6 < std::tuple_size_v<T>) return f(std::tuple_element_t<6, T>{});
    case 7:
      if constexpr (7 < std::tuple_size_v<T>) return f(std::tuple_element_t<7, T>{});
    case 8:
      if constexpr (8 < std::tuple_size_v<T>) return f(std::tuple_element_t<8, T>{});
    case 9:
      if constexpr (9 < std::tuple_size_v<T>) return f(std::tuple_element_t<9, T>{});
    case 10:
      if constexpr (10 < std::tuple_size_v<T>) return f(std::tuple_element_t<10, T>{});
    case 11:
      if constexpr (11 < std::tuple_size_v<T>) return f(std::tuple_element_t<11, T>{});
    case 12:
      if constexpr (12 < std::tuple_size_v<T>) return f(std::tuple_element_t<12, T>{});
    case 13:
      if constexpr (13 < std::tuple_size_v<T>) return f(std::tuple_element_t<13, T>{});
    case 14:
      if constexpr (14 < std::tuple_size_v<T>) return f(std::tuple_element_t<14, T>{});
    case 15:
      if constexpr (15 < std::tuple_size_v<T>) return f(std::tuple_element_t<15, T>{});
    case 16:
      if constexpr (16 < std::tuple_size_v<T>) return f(std::tuple_element_t<16, T>{});
    case 17:
      if constexpr (17 < std::tuple_size_v<T>) return f(std::tuple_element_t<17, T>{});
    case 18:
      if constexpr (18 < std::tuple_size_v<T>) return f(std::tuple_element_t<18, T>{});
    case 19:
      if constexpr (19 < std::tuple_size_v<T>) return f(std::tuple_element_t<19, T>{});
    case 20:
      if constexpr (20 < std::tuple_size_v<T>) return f(std::tuple_element_t<20, T>{});
    case 21:
      if constexpr (21 < std::tuple_size_v<T>) return f(std::tuple_element_t<21, T>{});
    case 22:
      if constexpr (22 < std::tuple_size_v<T>) return f(std::tuple_element_t<22, T>{});
    case 23:
      if constexpr (23 < std::tuple_size_v<T>) return f(std::tuple_element_t<23, T>{});
    case 24:
      if constexpr (24 < std::tuple_size_v<T>) return f(std::tuple_element_t<24, T>{});
    case 25:
      if constexpr (25 < std::tuple_size_v<T>) return f(std::tuple_element_t<25, T>{});
    case 26:
      if constexpr (26 < std::tuple_size_v<T>) return f(std::tuple_element_t<26, T>{});
    case 27:
      if constexpr (27 < std::tuple_size_v<T>) return f(std::tuple_element_t<27, T>{});
    case 28:
      if constexpr (28 < std::tuple_size_v<T>) return f(std::tuple_element_t<28, T>{});
    case 29:
      if constexpr (29 < std::tuple_size_v<T>) return f(std::tuple_element_t<29, T>{});
    case 30:
      if constexpr (30 < std::tuple_size_v<T>) return f(std::tuple_element_t<30, T>{});
    case 31:
      if constexpr (31 < std::tuple_size_v<T>) return f(std::tuple_element_t<31, T>{});
    case 32:
      if constexpr (32 < std::tuple_size_v<T>) return f(std::tuple_element_t<32, T>{});
    case 33:
      if constexpr (33 < std::tuple_size_v<T>) return f(std::tuple_element_t<33, T>{});
    case 34:
      if constexpr (34 < std::tuple_size_v<T>) return f(std::tuple_element_t<34, T>{});
    case 35:
      if constexpr (35 < std::tuple_size_v<T>) return f(std::tuple_element_t<35, T>{});
    case 36:
      if constexpr (36 < std::tuple_size_v<T>) return f(std::tuple_element_t<36, T>{});
    case 37:
      if constexpr (37 < std::tuple_size_v<T>) return f(std::tuple_element_t<37, T>{});
    case 38:
      if constexpr (38 < std::tuple_size_v<T>) return f(std::tuple_element_t<38, T>{});
    case 39:
      if constexpr (39 < std::tuple_size_v<T>) return f(std::tuple_element_t<39, T>{});
    case 40:
      if constexpr (40 < std::tuple_size_v<T>) return f(std::tuple_element_t<40, T>{});
    case 41:
      if constexpr (41 < std::tuple_size_v<T>) return f(std::tuple_element_t<41, T>{});
    case 42:
      if constexpr (42 < std::tuple_size_v<T>) return f(std::tuple_element_t<42, T>{});
    case 43:
      if constexpr (43 < std::tuple_size_v<T>) return f(std::tuple_element_t<43, T>{});
    case 44:
      if constexpr (44 < std::tuple_size_v<T>) return f(std::tuple_element_t<44, T>{});
    case 45:
      if constexpr (45 < std::tuple_size_v<T>) return f(std::tuple_element_t<45, T>{});
    case 46:
      if constexpr (46 < std::tuple_size_v<T>) return f(std::tuple_element_t<46, T>{});
    case 47:
      if constexpr (47 < std::tuple_size_v<T>) return f(std::tuple_element_t<47, T>{});
    case 48:
      if constexpr (48 < std::tuple_size_v<T>) return f(std::tuple_element_t<48, T>{});
    case 49:
      if constexpr (49 < std::tuple_size_v<T>) return f(std::tuple_element_t<49, T>{});
    case 50:
      if constexpr (50 < std::tuple_size_v<T>) return f(std::tuple_element_t<50, T>{});
    case 51:
      if constexpr (51 < std::tuple_size_v<T>) return f(std::tuple_element_t<51, T>{});
    case 52:
      if constexpr (52 < std::tuple_size_v<T>) return f(std::tuple_element_t<52, T>{});
    case 53:
      if constexpr (53 < std::tuple_size_v<T>) return f(std::tuple_element_t<53, T>{});
    case 54:
      if constexpr (54 < std::tuple_size_v<T>) return f(std::tuple_element_t<54, T>{});
    case 55:
      if constexpr (55 < std::tuple_size_v<T>) return f(std::tuple_element_t<55, T>{});
    case 56:
      if constexpr (56 < std::tuple_size_v<T>) return f(std::tuple_element_t<56, T>{});
    case 57:
      if constexpr (57 < std::tuple_size_v<T>) return f(std::tuple_element_t<57, T>{});
    case 58:
      if constexpr (58 < std::tuple_size_v<T>) return f(std::tuple_element_t<58, T>{});
    case 59:
      if constexpr (59 < std::tuple_size_v<T>) return f(std::tuple_element_t<59, T>{});
    case 60:
      if constexpr (60 < std::tuple_size_v<T>) return f(std::tuple_element_t<60, T>{});
    case 61:
      if constexpr (61 < std::tuple_size_v<T>) return f(std::tuple_element_t<61, T>{});
    case 62:
      if constexpr (62 < std::tuple_size_v<T>) return f(std::tuple_element_t<62, T>{});
    case 63:
      if constexpr (63 < std::tuple_size_v<T>) return f(std::tuple_element_t<63, T>{});
    default:
      return f(nullptr);
  }
}

}  // namespace hf3fs

#pragma once

#include <folly/concurrency/UnboundedQueue.h>
#include <list>
#include <memory>
#include <sys/epoll.h>
#include <thread>

#include "common/utils/FdWrapper.h"
#include "common/utils/Result.h"
#include "common/utils/TypeTraits.h"

namespace hf3fs::net {

// EventLoop provides a main loop that notifies EventHandler callback objects when I/O is ready on a file descriptor.
class EventLoop : public hf3fs::enable_shared_from_this<EventLoop> {
  struct HandlerWrapper;

 protected:
  EventLoop() = default;

 public:
  ~EventLoop() { stopAndJoin(); }

  // start and stop.
  Result<Void> start(const std::string &threadName = "EventLoop");
  Result<Void> wakeUp();
  void stopAndJoin();

  class EventHandler {
   public:
    virtual ~EventHandler() = default;
    virtual int fd() const = 0;
    virtual void handleEvents(uint32_t epollEvents) = 0;

   protected:
    friend class EventLoop;
    std::weak_ptr<EventLoop> eventLoop_;
    std::list<HandlerWrapper>::iterator it_;
  };

  // add a event handler with interest events into event loop.
  Result<Void> add(const std::shared_ptr<EventHandler> &handler, uint32_t interestEvents);

  // remove a event handler from event loop.
  Result<Void> remove(EventHandler *handler);

 private:
  struct HandlerWrapper {
    std::weak_ptr<EventHandler> handler;
  };

  void loop();

 private:
  FdWrapper epfd_;
  FdWrapper eventfd_;

  std::atomic<bool> stop_{false};
  std::jthread thread_;

  std::mutex mutex_;
  std::list<HandlerWrapper> wrapperList_;

  // wake up the event loop to do deletion if the size of delete queue greater than this threshold.
  constexpr static size_t kDeleteQueueWakeUpLoopThreshold = 128u;
  // deletion of the wrapper object is done in the loop thread.
  folly::UMPSCQueue<std::list<HandlerWrapper>::iterator, true> deleteQueue_;
};

class EventLoopPool {
 public:
  EventLoopPool(size_t numThreads);

  // start and stop.
  Result<Void> start(const std::string &threadName);
  void stopAndJoin();

  // add a event handler with interest events into event loop.
  Result<Void> add(const std::shared_ptr<EventLoop::EventHandler> &handler, uint32_t interestEvents);

 private:
  std::vector<std::shared_ptr<EventLoop>> eventLoops_;
};

}  // namespace hf3fs::net

#pragma once

#include <folly/concurrency/AtomicSharedPtr.h>
#include <folly/experimental/coro/Mutex.h>
#include <folly/hash/Hash.h>
#include <folly/logging/xlog.h>
#include <memory>
#include <mutex>
#include <shared_mutex>

#include "common/utils/Coroutine.h"
#include "common/utils/FairSharedMutex.h"
#include "common/utils/RobinHood.h"

namespace hf3fs {

template <typename Hash>
inline std::string hashKeyToString(const Hash &) noexcept {
  return "";
}

template <typename Hash, typename T, typename... Ts>
std::string hashKeyToString(const Hash &h, const T &t, const Ts &...ts) {
  if (sizeof...(ts) == 0) return fmt::format("{}#{:#x}", t, h);
  return fmt::format("{}/{}", t, hashKeyToString(h, ts...));
}

template <class Mutex = std::mutex>
class LockManager {
 public:
  LockManager(uint32_t numBuckets = 4099u)
      : numBuckets_(numBuckets),
        mutexes_(std::make_unique<Mutex[]>(numBuckets)),
        owners_(numBuckets) {
    for (auto &owner : owners_) owner = std::make_shared<std::string>("(unknown)");
  }

  auto lock(auto &&...args) { return std::unique_lock{mutexes_[idx(args...)]}; }
  auto try_lock(auto &&...args) { return std::unique_lock{mutexes_[idx(args...)], std::try_to_lock}; }

  auto lock_shared(auto &&...args) { return std::shared_lock{mutexes_[idx(args...)]}; }
  auto try_lock_shared(auto &&...args) { return std::shared_lock{mutexes_[idx(args...)], std::try_to_lock}; }

  auto co_scoped_lock(auto &&...args) { return mutexes_[idx(args...)].co_scoped_lock(); }

  CoTask<std::unique_lock<folly::coro::Mutex>> co_scoped_lock_log_owner(auto &&...args) {
    auto index = idx(args...);
    auto &mutex = mutexes_[index];
    auto &owner = owners_[index];

    std::unique_lock<folly::coro::Mutex> lock{mutex, std::try_to_lock};

    if (!lock) {
      XLOGF(DBG3, "Key {} is waiting on mutex #{} ownerd by {}", hashKeyToString(index, args...), index, *owner.load());
      auto waitLock = co_await mutex.co_scoped_lock();
      lock.swap(waitLock);
      XLOGF(DBG3, "Key {} just got a lock on mutex #{}", hashKeyToString(index, args...), index);
    }

    if (XLOG_IS_ON(DBG3)) owner = std::make_shared<std::string>(hashKeyToString(index, args...));

    co_return lock;
  }

  auto lock_at(uint64_t index) { return std::unique_lock{mutexes_[index]}; }
  auto try_lock_at(uint64_t index) { return std::unique_lock{mutexes_[index], std::try_to_lock}; }

  auto lock_shared_at(uint64_t index) { return std::shared_lock{mutexes_[index]}; }
  auto try_lock_shared_at(uint64_t index) { return std::shared_lock{mutexes_[index], std::try_to_lock}; }

  auto co_scoped_lock_at(uint64_t index) { return mutexes_[index].co_scoped_lock(); }

  uint64_t idx(auto &&...args) const {
    if constexpr (sizeof...(args) == 0) {
      return 0;
    } else {
      return folly::hash::hash_combine(args...) % numBuckets_;
    }
  }

  uint32_t numBuckets() const { return numBuckets_; }

 private:
  uint32_t numBuckets_{};
  std::unique_ptr<Mutex[]> mutexes_;
  std::vector<folly::atomic_shared_ptr<std::string>> owners_;
};

using UniqueLockManager = LockManager<std::mutex>;
using SharedLockManager = LockManager<FairSharedMutex>;
using CoUniqueLockManager = LockManager<folly::coro::Mutex>;

}  // namespace hf3fs

#pragma once

#include <functional>

#include "client/meta/MetaClient.h"
#include "client/storage/StorageClient.h"
#include "common/utils/Result.h"

namespace hf3fs::lib::agent {
using flat::UserInfo;
class PioV {
 public:
  PioV(storage::client::StorageClient &storageClient, int chunkSizeLim, std::vector<ssize_t> &res);
  hf3fs::Result<Void> addRead(size_t idx,
                              const meta::Inode &inode,
                              uint16_t track,
                              off_t off,
                              size_t len,
                              void *buf,
                              storage::client::IOBuffer &memh);
  // if metaClient and userInfo are not nullptr,
  // meta server will be contacted for latest file length if known length is shorter than off
  CoTryTask<bool> checkWriteOff(size_t idx,
                                meta::client::MetaClient *metaClient,
                                const UserInfo *userInfo,
                                const meta::Inode &inode,
                                size_t off);
  hf3fs::Result<Void> addWrite(size_t idx,
                               const meta::Inode &inode,
                               uint16_t track,
                               off_t off,
                               size_t len,
                               const void *buf,
                               storage::client::IOBuffer &memh);
  CoTryTask<void> executeRead(const UserInfo &userInfo,
                              const storage::client::ReadOptions &options = storage::client::ReadOptions());
  CoTryTask<void> executeWrite(const UserInfo &userInfo,
                               const storage::client::WriteOptions &options = storage::client::WriteOptions());
  void finishIo(bool allowHoles);

 private:
  Result<Void> chunkIo(
      const meta::Inode &inode,
      uint16_t track,
      off_t off,
      size_t len,
      std::function<void(storage::ChainId, storage::ChunkId, uint32_t, uint32_t, uint32_t)> &&consumeChunk);

 private:
  storage::client::StorageClient &storageClient_;
  int chunkSizeLim_;
  std::shared_ptr<flat::RoutingInfo> routingInfo_;
  std::vector<ssize_t> &res_;
  std::vector<storage::client::ReadIO> rios_;
  std::vector<storage::client::WriteIO> wios_;
  std::vector<storage::client::TruncateChunkOp> trops_;
  std::map<meta::InodeId, size_t> potentialLens_;
};
}  // namespace hf3fs::lib::agent

#pragma once

#include <folly/logging/xlog.h>
#include <string_view>
#include <type_traits>
#include <utility>
#include <variant>

#include "Common.h"
#include "common/serde/Serde.h"
#include "common/utils/Nameof.hpp"
#include "common/utils/StrongType.h"
#include "common/utils/TypeTraits.h"

namespace hf3fs::analytics {

class StructVisitor {
 public:
  // default
  template <typename T>
  void visit(std::string_view) = delete;

  template <typename T>
  requires std::is_arithmetic_v<T>
  void visit(std::string_view) = delete;

  template <typename T>
  requires std::is_enum_v<T>
  void visit(std::string_view) = delete;

  template <StrongTyped T>
  void visit(std::string_view) = delete;

  template <serde::SerdeTypeWithoutSpecializedSerdeMethod T>
  void visit(std::string_view) = delete;

  template <typename T>
  requires is_variant_v<T>
  void visit(std::string_view) = delete;

  template <typename T>
  requires is_vector_v<T> || is_set_v<T>
  void visit(std::string_view) = delete;

  template <typename T>
  requires is_optional_v<T>
  void visit(std::string_view) = delete;
};

template <class T, size_t I = 0>
inline void visitVariant(auto &&func) {
  using S = std::variant_alternative_t<I, T>;
  func(nameof::nameof_short_type<S>(), std::type_identity<S>{});
  if constexpr (I + 1 < std::variant_size_v<T>) {
    visitVariant<T, I + 1>(func);
  }
}

template <typename Derived>
class BaseStructVisitor : public StructVisitor {
 public:
  // default
  template <typename T>
  void visit(std::string_view k) = delete;

  template <typename T>
  requires std::is_arithmetic_v<T>
  void visit(std::string_view k) {
    XLOGF(DBG3, "arithmetic visit({})", k);
    static_cast<Derived *>(this)->template visit<T>(k);
  }

  template <typename T>
  requires std::is_enum_v<T>
  void visit(std::string_view k) = delete;

  template <serde::ConvertibleToString T>
  void visit(std::string_view k) = delete;

  template <StrongTyped T>
  void visit(std::string_view k) {
    XLOGF(DBG3, "strongtyped visit({})", k);
    static_cast<Derived *>(this)->template visit<typename T::UnderlyingType>(k);
  }

  template <serde::SerdeTypeWithoutSpecializedSerdeMethod T>
  void visit(std::string_view k) {
    XLOGF(DBG3, "serdetype visit({})", k);
    refl::Helper::iterate<T>([&](auto field) {
      using FieldType = std::decay_t<decltype(std::declval<T>().*field.getter)>;
      static_cast<Derived *>(this)->template visit<FieldType>(field.name);
    });
  }

  template <typename T>
  requires is_variant_v<T>
  void visit(std::string_view k) {
    XLOGF(DBG3, "variant visit({})", k);
    visitVariant<T>([&](std::string_view typeName, auto &&v) {
      using AlternativeType = typename std::decay_t<decltype(v)>::type;
      std::string altTypeName = std::string{k} + std::string{typeName};
      static_cast<Derived *>(this)->template visit<AlternativeType>(altTypeName);
    });
  }

  template <typename T>
  requires is_vector_v<T> || is_set_v<T>
  void visit(std::string_view k) {
    XLOGF(DBG3, "container visit({})", k);
    using ElemValueType = typename T::value_type;
    static_cast<Derived *>(this)->template visit<ElemValueType>(k);
  }

  template <typename T>
  requires is_optional_v<T>
  void visit(std::string_view k) {
    XLOGF(DBG3, "optional visit({})", k);
    using ValueType = typename T::value_type;
    static_cast<Derived *>(this)->template visit<ValueType>(k);
  }
};

}  // namespace hf3fs::analytics

#pragma once

#include <algorithm>
#include <chrono>

#include "common/utils/UtcTime.h"

namespace hf3fs {

class ExponentialBackoffRetry {
 public:
  ExponentialBackoffRetry(std::chrono::milliseconds initWaitTime,
                          std::chrono::milliseconds maxWaitTime,
                          std::chrono::milliseconds maxRetryTime)
      : startTime_(SteadyClock::now()),
        initWaitTime_(initWaitTime),
        maxWaitTime_(std::max(initWaitTime_, maxWaitTime)),
        maxRetryTime_(std::max(maxWaitTime_, maxRetryTime)),
        nextWaitTime_(initWaitTime) {}

  std::chrono::milliseconds getWaitTime() {
    auto elapsedTime = getElapsedTime();

    if (elapsedTime + initWaitTime_ > maxRetryTime_) {
      return std::chrono::milliseconds::zero();
    }

    std::chrono::milliseconds waitTime = std::min(nextWaitTime_, maxRetryTime_ - elapsedTime);
    nextWaitTime_ = std::min(nextWaitTime_ * kWaitTimeMultiplier, maxWaitTime_);

    return waitTime;
  }

  hf3fs::SteadyTime getStartTime() const { return startTime_; }

  std::chrono::milliseconds getElapsedTime() const {
    return std::chrono::duration_cast<std::chrono::milliseconds>(SteadyClock::now() - startTime_);
  }

 private:
  static constexpr uint32_t kWaitTimeMultiplier = 2;

  const hf3fs::SteadyTime startTime_;
  const std::chrono::milliseconds initWaitTime_;
  const std::chrono::milliseconds maxWaitTime_;
  const std::chrono::milliseconds maxRetryTime_;
  std::chrono::milliseconds nextWaitTime_;
};
}  // namespace hf3fs

#pragma once

#include "common/serde/CallContext.h"
#include "common/serde/Echo.h"
#include "common/serde/Service.h"
#include "common/utils/ConstructLog.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Result.h"

namespace hf3fs::serde {

class Services {
 public:
  Services() { addService(std::make_unique<echo::ServiceImpl>(), true); }

  template <class Service>
  Result<Void> addService(std::unique_ptr<Service> &&obj, bool isRDMA) {
    std::shared_ptr<Service> shared = std::move(obj);
    for (auto i = 0u; i <= uint32_t(isRDMA); ++i) {
      auto &service = services_[i][Service::kServiceID];
      if (UNLIKELY(service.object != nullptr)) {
        return makeError(StatusCode::kInvalidArg, fmt::format("redundant service id: {}", Service::kServiceID));
      }
      service.getter = &MethodExtractor<Service, CallContext, &CallContext::invalidId>::get;
      service.object = shared.get();
      service.alive = std::shared_ptr<void *>(shared, nullptr);
      if constexpr (requires { Service{}.onError(Status::OK); }) {
        service.onError = &CallContext::customOnError<Service, &Service::onError>;
      }
    }
    return Void{};
  }

  CallContext::ServiceWrapper &getServiceById(uint16_t idx, bool isRDMA) { return services_[isRDMA].at(idx); }

 private:
  ConstructLog<"serde::Services"> constructLog_;
  std::array<CallContext::ServiceWrapper, 65536> services_[2];  // 0 for TCP, 1 for RDMA.
};

}  // namespace hf3fs::serde

#pragma once

#include <atomic>
#include <cassert>
#include <folly/ScopeGuard.h>
#include <folly/Synchronized.h>
#include <folly/Utility.h>
#include <folly/experimental/coro/Baton.h>
#include <folly/logging/xlog.h>
#include <functional>
#include <map>
#include <memory>
#include <optional>
#include <type_traits>
#include <utility>
#include <vector>

#include "common/kv/ITransaction.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Result.h"
#include "common/utils/Shards.h"
#include "common/utils/UtcTime.h"
#include "fbs/core/user/User.h"
#include "fbs/meta/Common.h"
#include "fbs/meta/Schema.h"
#include "fbs/meta/Service.h"
#include "meta/store/DirEntry.h"
#include "meta/store/Inode.h"
#include "meta/store/MetaStore.h"
#include "meta/store/Operation.h"
namespace hf3fs::meta::server {

class MetaOperator;

class BatchedOp : public Operation<Inode> {
 public:
  template <typename Req, typename Rsp>
  struct Waiter : folly::NonCopyableNonMovable {
    Req req;
    std::optional<Result<Rsp>> result;
    folly::coro::Baton baton;
    bool newFile = false; /* for Create operation */

    Waiter(Req req)
        : req(std::move(req)) {}

    Result<Rsp> getResult() {
      XLOGF_IF(FATAL, !result.has_value(), "result not set");
      return *result;
    }

    bool hasError() const { return result.has_value() && result->hasError(); }

    void finish(BatchedOp &op, const Result<Inode> &r);
  };

  BatchedOp(MetaStore &meta, InodeId inodeId)
      : Operation(meta),
        inodeId_(inodeId) {}

  std::string_view name() const override { return "batchedOp"; }

  flat::Uid user() const override { return user_; }

  template <typename Req, typename Rsp>
  void add(Waiter<Req, Rsp> &waiter);

  template <>
  void add(Waiter<SyncReq, SyncRsp> &waiter) {
    XLOGF_IF(FATAL, waiter.req.inode != inodeId_, "{} != {}", waiter.req.inode, inodeId_);
    addReq(syncs_, waiter);
  }

  template <>
  void add(Waiter<CloseReq, CloseRsp> &waiter) {
    XLOGF_IF(FATAL, waiter.req.inode != inodeId_, "{} != {}", waiter.req.inode, inodeId_);
    addReq(closes_, waiter);
  }

  template <>
  void add(Waiter<SetAttrReq, SetAttrRsp> &waiter) {
    XLOGF_IF(FATAL, waiter.req.path != PathAt(inodeId_), "{} != {}", waiter.req.path, PathAt(inodeId_));
    addReq(setattrs_, waiter);
  }

  template <>
  void add(Waiter<CreateReq, CreateRsp> &waiter) {
    XLOGF_IF(FATAL,
             (waiter.req.path.parent != inodeId_ || !waiter.req.path.path || waiter.req.path.path->has_parent_path()),
             "path {}, inodeId {}",
             waiter.req.path,
             inodeId_);
    addReq(creates_, waiter);
  }

  CoTryTask<Inode> run(IReadWriteTransaction &txn) override;

  void retry(const Status &error) override;

  void finish(const Result<Inode> &result) override;

  size_t numReqs() const { return numReqs_; }

  // for test
  static CoTryTask<CreateRsp> create(MetaStore &store, IReadWriteTransaction &txn, CreateReq req) {
    Waiter<CreateReq, CreateRsp> waiter(req);
    BatchedOp op(store, req.path.parent);
    op.add(waiter);
    op.finish(co_await op.run(txn));
    co_return waiter.getResult();
  }

 private:
  friend class MetaOperator;

  template <typename Req, typename Rsp>
  using WaiterRef = std::reference_wrapper<Waiter<Req, Rsp>>;

  void addReq(auto &reqs, auto &waiter) {
    if (!user_) {
      user_ = waiter.req.user.uid;
    }
    reqs.emplace_back(waiter);
    numReqs_++;
  }

  CoTryTask<bool> setAttr(IReadWriteTransaction &txn, Inode &inode);

  CoTryTask<bool> syncAndClose(IReadWriteTransaction &txn, Inode &inode);

  CoTryTask<bool> sync(Inode &inode,
                       const SyncReq &req,
                       bool &updateLength,
                       bool &truncate,
                       std::optional<VersionedLength> &hintLength);

  CoTryTask<bool> close(Inode &inode,
                        const CloseReq &req,
                        bool &updateLength,
                        std::optional<VersionedLength> &hintLength,
                        std::vector<FileSession> &sessions);

  CoTryTask<bool> create(IReadWriteTransaction &txn, Inode &inode);

  CoTryTask<bool> create(IReadWriteTransaction &txn,
                         const Inode &parent,
                         folly::Synchronized<uint32_t> &chainAllocCounter,
                         auto begin,
                         auto end);

  CoTryTask<std::pair<Inode, DirEntry>> create(IReadWriteTransaction &txn,
                                               const Inode &parent,
                                               folly::Synchronized<uint32_t> &chainAllocCounter,
                                               const CreateReq &req);

  CoTryTask<void> openExists(IReadWriteTransaction &txn, Inode &inode, const DirEntry &entry, auto begin, auto end);

  CoTryTask<bool> openExists(IReadWriteTransaction &txn, Inode &inode, const CreateReq &req);

  CoTryTask<VersionedLength> queryLength(const Inode &inode, std::optional<VersionedLength> hintLength, bool truncate);

  // requests
  InodeId inodeId_;
  flat::Uid user_;  // use first uid
  std::vector<WaiterRef<SetAttrReq, SetAttrRsp>> setattrs_;
  std::vector<WaiterRef<SyncReq, SyncRsp>> syncs_;
  std::vector<WaiterRef<CloseReq, CloseRsp>> closes_;
  std::vector<WaiterRef<CreateReq, CreateRsp>> creates_;
  size_t numReqs_ = 0;

  // state
  std::optional<kv::Versionstamp> versionstamp_;
  std::optional<VersionedLength> currLength_;
  std::optional<VersionedLength> nextLength_;
};

}  // namespace hf3fs::meta::server

#pragma once

#include <fmt/format.h>
#include <folly/experimental/symbolizer/Symbolizer.h>
#include <folly/logging/xlog.h>

#include "common/monitor/Recorder.h"
#include "common/utils/NameWrapper.h"
#include "common/utils/String.h"

#define RETURN_AND_LOG_OP_ERR(...) RETURN_AND_LOG_OP_ERR_IMPL(return, __VA_ARGS__)
#define CO_RETURN_AND_LOG_OP_ERR(...) RETURN_AND_LOG_OP_ERR_IMPL(co_return, __VA_ARGS__)

#define RETURN_AND_LOG_OP_ERR_IMPL(RETURN, op, code, s, ...)            \
  do {                                                                  \
    auto _msg = fmt::format(s __VA_OPT__(, ) __VA_ARGS__);              \
    LOG_OP_ERR((op), "error: {} {}", StatusCode::toString(code), _msg); \
    RETURN makeError(code, _msg);                                       \
  } while (false)

#define LOG_OP_DBG(...) LOG_OP_IMPL(DBG, __VA_ARGS__)
#define LOG_OP_INFO(...) LOG_OP_IMPL(INFO, __VA_ARGS__)
#define LOG_OP_WARN(...) LOG_OP_IMPL(WARN, __VA_ARGS__)
#define LOG_OP_ERR(...) LOG_OP_IMPL(ERR, __VA_ARGS__)
#define LOG_OP_CRITICAL(...) LOG_OP_IMPL(CRITICAL, __VA_ARGS__)
#define LOG_OP_FATAL(op, fmt, ...)                  \
  XLOGF(FATAL,                                      \
        "{} " fmt ". stack trace:\n{}",             \
        (op).toString() __VA_OPT__(, ) __VA_ARGS__, \
        folly::symbolizer::getStackTraceStr())

#define LOG_OP_IMPL(LEVEL, op, fmt, ...) XLOGF(LEVEL, "{} " fmt, (op).toString() __VA_OPT__(, ) __VA_ARGS__)

namespace hf3fs::core {
struct ServiceOperation {
  virtual ~ServiceOperation() = default;

  String toString() const {
    if (!cache) {
      cache = fmt::format("[{}Op {} No.{}]", serviceNameImpl(), toStringImpl(), reqId);
    }
    return *cache;
  }

  template <typename T>
  requires(std::is_base_of_v<ServiceOperation, T>) T &as() {
    auto *p = dynamic_cast<T *>(this);
    XLOG_IF(FATAL, !p, "invalid dynamic cast from ServiceOperation to {}", typeid(T).name());
    return *p;
  }

  template <typename T>
  requires(std::is_base_of_v<ServiceOperation, T>) const T &as() const {
    auto *p = dynamic_cast<const T *>(this);
    XLOG_IF(FATAL, !p, "invalid dynamic cast from ServiceOperation to {}", typeid(T).name());
    return *p;
  }

 private:
  virtual String serviceNameImpl() const = 0;
  virtual String toStringImpl() const = 0;

  static int64_t nextReqId() {
    static std::atomic<int64_t> id{1};
    return id.fetch_add(1, std::memory_order_acq_rel);
  }

  const int64_t reqId = nextReqId();
  mutable std::optional<String> cache;
};

template <NameWrapper ServiceName, NameWrapper OpName, NameWrapper MetricNamePrefix>
struct ServiceOperationWithMetric : ServiceOperation {
  static constexpr std::string_view serviceName = ServiceName;
  static constexpr std::string_view opName = OpName;
  static constexpr std::string_view metricNamePrefix = MetricNamePrefix;

  static_assert(!serviceName.empty());
  static_assert(!opName.empty());
  static_assert(!metricNamePrefix.empty());

  static const monitor::TagSet &getTagSet() {
    static auto tagSet = [] {
      monitor::TagSet ts;
      ts.addTag("instance", String(opName));
      return ts;
    }();
    return tagSet;
  }

  static monitor::SimpleOperationRecorder::Guard startRecord() {
    static monitor::SimpleOperationRecorder recorder(fmt::format("{}.{}", serviceName, metricNamePrefix),
                                                     getTagSet(),
                                                     /*recordErrorCode=*/true);
    return recorder.record();
  }

  String serviceNameImpl() const final { return String(serviceName); }
};

}  // namespace hf3fs::core

#pragma once

#include <fmt/format.h>
#include <folly/SharedMutex.h>
#include <folly/Synchronized.h>
#include <folly/Utility.h>
#include <folly/logging/xlog.h>
#include <memory>
#include <mutex>
#include <optional>
#include <random>
#include <set>
#include <utility>
#include <vector>

#include "client/mgmtd/ICommonMgmtdClient.h"
#include "client/mgmtd/RoutingInfo.h"
#include "common/app/NodeId.h"
#include "common/utils/Address.h"
#include "common/utils/MagicEnum.hpp"

namespace hf3fs::meta::client {

using hf3fs::client::ICommonMgmtdClient;
using hf3fs::client::RoutingInfo;

enum ServerSelectionMode {
  RoundRobin,
  UniformRandom,
  RandomFollow,
};

struct ServerSelectionStrategy : folly::NonCopyableNonMovable {
  struct NodeInfo {
    flat::NodeId nodeId;
    net::Address address;
    std::string hostname;
  };

  static std::unique_ptr<ServerSelectionStrategy> create(ServerSelectionMode mode,
                                                         std::shared_ptr<ICommonMgmtdClient> mgmtd,
                                                         net::Address::Type addrType);

  virtual ~ServerSelectionStrategy() = default;

  virtual ServerSelectionMode mode() = 0;
  virtual net::Address::Type addrType() = 0;
  virtual Result<NodeInfo> select(const std::set<flat::NodeId> &skipNodes) = 0;
  virtual std::optional<NodeInfo> get(flat::NodeId node) = 0;
};

struct ServerList {
  std::shared_ptr<RoutingInfo> routing;
  std::vector<flat::NodeId> nodeList;
  std::map<flat::NodeId, net::Address> nodeAddress;

  bool update(std::shared_ptr<RoutingInfo> newRouting, net::Address::Type addrType);
  std::optional<ServerSelectionStrategy::NodeInfo> get(flat::NodeId nodeId) const;
};

class BaseSelectionStrategy : public ServerSelectionStrategy {
 public:
  BaseSelectionStrategy(std::shared_ptr<ICommonMgmtdClient> mgmtd, net::Address::Type addrType);
  ~BaseSelectionStrategy() override;

  Result<NodeInfo> select(const std::set<flat::NodeId> &skipNodes) override;
  std::optional<NodeInfo> get(flat::NodeId node) override { return servers_.rlock()->get(node); }
  net::Address::Type addrType() override { return addrType_; }

  virtual flat::NodeId selectFrom(const std::vector<flat::NodeId> &nodes, const std::set<flat::NodeId> &skipHint) = 0;
  virtual void onUpdate(ServerList &);

 protected:
  // NOTE: must call this in subclass
  void registerListener();
  void update(std::shared_ptr<RoutingInfo> routing);

  std::string name_;
  net::Address::Type addrType_;
  std::shared_ptr<ICommonMgmtdClient> mgmtd_;
  folly::Synchronized<ServerList> servers_;
};

struct RoundRobinServerSelection : public BaseSelectionStrategy {
  RoundRobinServerSelection(std::shared_ptr<ICommonMgmtdClient> mgmtd, net::Address::Type addrType)
      : BaseSelectionStrategy(mgmtd, addrType) {
    registerListener();
  }
  ~RoundRobinServerSelection() override = default;

  ServerSelectionMode mode() override { return ServerSelectionMode::RoundRobin; }
  flat::NodeId selectFrom(const std::vector<flat::NodeId> &nodes, const std::set<flat::NodeId> &skipHint) override;

 private:
  std::atomic<size_t> next_{0};
};

struct UniformRandomServerSelection : public BaseSelectionStrategy {
  UniformRandomServerSelection(std::shared_ptr<ICommonMgmtdClient> mgmtd, net::Address::Type addrType)
      : BaseSelectionStrategy(mgmtd, addrType),
        mutex_(),
        gen_(std::random_device{}()) {
    registerListener();
  }
  ~UniformRandomServerSelection() override = default;

  ServerSelectionMode mode() override { return ServerSelectionMode::UniformRandom; }
  flat::NodeId selectFrom(const std::vector<flat::NodeId> &nodes, const std::set<flat::NodeId> &skipHint) override;

 private:
  using index_range = std::uniform_int_distribution<size_t>::param_type;
  std::mutex mutex_;
  std::mt19937_64 gen_;
  std::uniform_int_distribution<size_t> dist_;
};

struct RandomFollowServerSelection : public BaseSelectionStrategy {
  RandomFollowServerSelection(std::shared_ptr<ICommonMgmtdClient> mgmtd, net::Address::Type addrType)
      : BaseSelectionStrategy(mgmtd, addrType),
        token_(Uuid::random()) {
    registerListener();
  }
  ~RandomFollowServerSelection() override = default;

  ServerSelectionMode mode() override { return ServerSelectionMode::RandomFollow; }
  flat::NodeId selectFrom(const std::vector<flat::NodeId> &nodes, const std::set<flat::NodeId> &skipHint) override;
  std::optional<NodeInfo> get(flat::NodeId node) override { return servers_.rlock()->get(node); }
  void onUpdate(ServerList &servers) override;

 private:
  Uuid token_;
  std::vector<flat::NodeId> prefer_;
};
}  // namespace hf3fs::meta::client

FMT_BEGIN_NAMESPACE

template <>
struct formatter<hf3fs::meta::client::ServerSelectionStrategy::NodeInfo> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(const hf3fs::meta::client::ServerSelectionStrategy::NodeInfo &node, FormatContext &ctx) const {
    return fmt::format_to(ctx.out(), "[{}]@{}.{}", node.address, node.nodeId, node.hostname);
  }
};

FMT_END_NAMESPACE

#pragma once

#include <cassert>
#include <fmt/core.h>
#include <folly/Executor.h>
#include <folly/Synchronized.h>
#include <folly/functional/Invoke.h>
#include <folly/logging/xlog.h>
#include <functional>
#include <memory>
#include <optional>
#include <string>
#include <string_view>
#include <tuple>
#include <utility>
#include <vector>

#include "client/mgmtd/IMgmtdClientForServer.h"
#include "common/app/ClientId.h"
#include "common/app/NodeId.h"
#include "common/kv/IKVEngine.h"
#include "common/kv/ITransaction.h"
#include "common/kv/KeyPrefix.h"
#include "common/serde/Serde.h"
#include "common/utils/BackgroundRunner.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/Coroutine.h"
#include "common/utils/CoroutinesPool.h"
#include "common/utils/Duration.h"
#include "common/utils/Result.h"
#include "common/utils/String.h"
#include "common/utils/UtcTime.h"
#include "common/utils/Uuid.h"
#include "fbs/meta/Common.h"
#include "fbs/meta/Service.h"
#include "meta/store/FileSession.h"
#include "meta/store/Inode.h"
#include "meta/store/Utils.h"

namespace hf3fs::meta::server {

class FileHelper;

class SessionManager {
 public:
  class Config : public ConfigBase<Config> {
    CONFIG_HOT_UPDATED_ITEM(enable, true);
    CONFIG_HOT_UPDATED_ITEM(scan_interval, 5_min);
    CONFIG_HOT_UPDATED_ITEM(scan_batch, 1024u);
    CONFIG_HOT_UPDATED_ITEM(sync_on_prune_session, false);
    CONFIG_HOT_UPDATED_ITEM(session_timeout, 5_min);
    CONFIG_OBJ(scan_workers, CoroutinesPoolBase::Config, [](auto &c) {
      c.set_coroutines_num(8);
      c.set_queue_size(128);
    });
    CONFIG_OBJ(close_workers, CoroutinesPoolBase::Config, [](auto &c) {
      c.set_coroutines_num(32);
      c.set_queue_size(1024);
    });
  };

  SessionManager(const Config &cfg,
                 flat::NodeId nodeId,
                 std::shared_ptr<kv::IKVEngine> kvEngine,
                 std::shared_ptr<client::ICommonMgmtdClient> mgmtd,
                 std::shared_ptr<FileHelper> fileHelper)
      : config_(cfg),
        nodeId_(nodeId),
        kvEngine_(kvEngine),
        mgmtd_(mgmtd),
        fileHelper_(fileHelper) {}
  ~SessionManager() { stopAndJoin(); }

  void start(CPUExecutorGroup &exec);
  void stopAndJoin();

  using CloseFunc = std::function<CoTryTask<void>(const meta::CloseReq &)>;
  void setCloseFunc(CloseFunc close) { close_ = close; }

  // for admin_cli
  CoTryTask<std::vector<FileSession>> listSessions();
  CoTryTask<std::vector<FileSession>> listSessions(InodeId inodeId);
  CoTryTask<size_t> pruneManually();

 private:
  struct PruneSessions {
    std::atomic<size_t> finished = 0;
    folly::Synchronized<std::map<Uuid, FileSession>> sessions;
  };

  class ScanTask {
   public:
    ScanTask(size_t shard, std::shared_ptr<PruneSessions> prune)
        : shard_(shard),
          prune_(prune) {}

    CoTryTask<size_t> run(SessionManager &manager);

   private:
    size_t shard_ = -1;
    std::shared_ptr<PruneSessions> prune_;
  };

  // try to close and sync
  class CloseTask {
   public:
    CloseTask(FileSession session)
        : session_(std::move(session)) {}
    CoTryTask<void> run(SessionManager &manager);

   private:
    FileSession session_;
  };

  CoTask<void> scanTask();
  CoTryTask<std::shared_ptr<PruneSessions>> loadPrune();

  const Config &config_;
  flat::NodeId nodeId_;
  std::shared_ptr<kv::IKVEngine> kvEngine_;
  std::shared_ptr<client::ICommonMgmtdClient> mgmtd_;
  std::shared_ptr<FileHelper> fileHelper_;
  std::unique_ptr<BackgroundRunner> scanRunner_;
  std::unique_ptr<CoroutinesPool<std::unique_ptr<ScanTask>>> scanWorkers_;
  std::unique_ptr<CoroutinesPool<std::unique_ptr<CloseTask>>> closeWorkers_;
  CloseFunc close_;
};

}  // namespace hf3fs::meta::server

FMT_BEGIN_NAMESPACE

template <>
struct formatter<hf3fs::meta::server::FileSession> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(const hf3fs::meta::server::FileSession &session, FormatContext &ctx) const {
    return fmt::format_to(ctx.out(),
                          "{{inodeId {}, client {}, session {}}}",
                          session.inodeId,
                          session.clientId,
                          session.sessionId);
  }
};

FMT_END_NAMESPACE

#pragma once

#include "common/serde/Serde.h"
#include "common/utils/String.h"
#include "common/utils/SysResource.h"
#include "common/utils/Uuid.h"

namespace hf3fs {

struct ClientId {
  SERDE_STRUCT_TYPED_FIELD(Uuid, uuid, hf3fs::Uuid{});
  SERDE_STRUCT_TYPED_FIELD(String, hostname, "<unknown host>");

 public:
  ClientId()
      : ClientId(Uuid::random()) {}

  explicit ClientId(const Uuid &uuid, std::string_view hostname = "")
      : uuid(uuid),
        hostname(hostname) {
    if (hostname.empty()) {
      auto hostnameRes = SysResource::hostname(/*physicalMachineName=*/true);
      if (hostnameRes) {
        hostname = *hostnameRes;
      }
    }
  }

  static ClientId max() { return ClientId{hf3fs::Uuid::max()}; }

  static ClientId zero() { return ClientId{hf3fs::Uuid::zero()}; }

  static ClientId random(std::string_view hostname = "") { return ClientId{Uuid::random(), hostname}; }

  bool operator==(const ClientId &other) const { return this->uuid == other.uuid; }

  bool operator<(const ClientId &other) const { return this->uuid < other.uuid; }
};
static_assert(serde::Serializable<ClientId>);

}  // namespace hf3fs

template <>
struct std::hash<hf3fs::ClientId> {
  size_t operator()(const hf3fs::ClientId &clientId) const { return std::hash<hf3fs::Uuid>()(clientId.uuid); }
};

FMT_BEGIN_NAMESPACE

template <>
struct formatter<hf3fs::ClientId> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(const hf3fs::ClientId &clientId, FormatContext &ctx) const {
    return fmt::format_to(ctx.out(), "ClientId({})@{}", clientId.uuid.toHexString(), clientId.hostname);
  }
};

FMT_END_NAMESPACE

#pragma once

#include "common/serde/Serde.h"
#include "common/utils/Duration.h"
#include "common/utils/StrongType.h"
#include "common/utils/Transform.h"
#include "common/utils/UtcTimeSerde.h"

namespace hf3fs::flat {
STRONG_TYPEDEF(uint32_t, Uid);
STRONG_TYPEDEF(uint32_t, Gid);
// todo: add a format func for permission
STRONG_TYPEDEF(uint32_t, Permission);

struct Token : public std::string {
  using std::string::string;
  using std::string::operator=;
  Token() = default;
  Token(const std::string &o)
      : std::string(o) {}
  Token(std::string &&o)
      : std::string(std::move(o)) {}
};

struct UserInfo {
  SERDE_STRUCT_FIELD(uid, Uid());
  SERDE_STRUCT_FIELD(gid, Gid());
  SERDE_STRUCT_FIELD(groups, std::vector<Gid>());
  SERDE_STRUCT_FIELD(token, Token{});

 public:
  UserInfo(Uid uid, Gid gid, std::string token);
  UserInfo(Uid uid = {}, Gid gid = {}, std::vector<Gid> groups = {}, std::string token = {});

  bool isRoot() const;
  bool inGroup(Gid group) const;
  bool operator==(const UserInfo &o) const;
  static bool compareUserAndToken(const UserInfo &a, const UserInfo &b);
};

struct TokenAttr {
  SERDE_STRUCT_FIELD(token, Token{});
  SERDE_STRUCT_FIELD(endTime, UtcTime());  // 0 means endless
 public:
  bool active(UtcTime now) const;
  bool isEndless() const;

  bool operator==(const TokenAttr &other) const;
};

struct UserAttr {
  SERDE_STRUCT_FIELD(name, std::string());
  SERDE_STRUCT_FIELD(token, Token{});  // the only one endless token
  SERDE_STRUCT_FIELD(gid, Gid());
  SERDE_STRUCT_FIELD(groups, std::vector<Gid>());
  SERDE_STRUCT_FIELD(root, false);
  SERDE_STRUCT_FIELD(tokens, std::vector<TokenAttr>());  // tokens with ttl
  SERDE_STRUCT_FIELD(uid, Uid());
  SERDE_STRUCT_FIELD(admin, false);

 public:
  Result<Void> validateToken(std::string_view token, UtcTime now) const;

  Result<Void> addNewToken(std::string_view token,
                           UtcTime now,
                           bool invalidateExisted = true,
                           size_t maxTokens = 5,
                           Duration tokenLifetimeExtendLength = Duration(std::chrono::days(7)));

  bool operator==(const UserAttr &other) const;
};

}  // namespace hf3fs::flat

FMT_BEGIN_NAMESPACE

template <>
struct formatter<hf3fs::flat::UserInfo> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(const hf3fs::flat::UserInfo &user, FormatContext &ctx) const {
    auto groups = transformTo<std::vector>(std::span{user.groups.begin(), user.groups.size()},
                                           [](hf3fs::flat::Gid gid) { return gid.toUnderType(); });
    return fmt::format_to(ctx.out(),
                          "(uid {}, gid {}, groups ({}))",
                          user.uid.toUnderType(),
                          user.gid.toUnderType(),
                          fmt::join(groups.begin(), groups.end(), ","));
  }
};

FMT_END_NAMESPACE

template <>
struct ::hf3fs::serde::SerdeMethod<::hf3fs::flat::Token> {
  static auto serdeTo(const ::hf3fs::flat::Token &o) { return std::string_view{o}; }
  static Result<::hf3fs::flat::Token> serdeFrom(std::string_view in) { return ::hf3fs::flat::Token{in}; }
  static constexpr std::string_view serdeToReadable(const ::hf3fs::flat::Token &) { return "SECRET TOKEN"; };
};

template <>
struct hf3fs::serde::SerdeMethod<hf3fs::flat::UserInfo> {
  static std::string serdeToReadable(hf3fs::flat::UserInfo user) { return fmt::format("{}", user); }
};

#pragma once

#include <folly/ThreadLocal.h>
#include <rocksdb/db.h>
#include <rocksdb/write_batch.h>

#include "kv/KVStore.h"

namespace hf3fs::kv {

class RocksDBStore : public KVStore {
 public:
  RocksDBStore(const Config &config)
      : config_(config) {}

  // get value corresponding to key.
  Result<std::string> get(std::string_view key) final;

  // get the first key which is greater than input key.
  Result<Void> iterateKeysWithPrefix(std::string_view prefix,
                                     uint32_t limit,
                                     IterateFunc func,
                                     std::optional<std::string> *nextValidKey = nullptr) final;

  // put a key-value pair.
  Result<Void> put(std::string_view key, std::string_view value, bool sync = false) final;

  // remove a key-value pair.
  Result<Void> remove(std::string_view key) final;

  // batch operations.
  class RocksDBBatchOperations : public BatchOperations {
   public:
    RocksDBBatchOperations(RocksDBStore &db)
        : db_(db) {}
    // put a key-value pair.
    void put(std::string_view key, std::string_view value) override;
    // remove a key.
    void remove(std::string_view key) override;
    // clear a batch operations.
    void clear() override { writeBatch_.Clear(); }
    // commit a batch of operations.
    Result<Void> commit() override;
    // destroy self.
    void destroy() override;

   private:
    RocksDBStore &db_;
    rocksdb_internal::WriteBatch writeBatch_;
  };
  BatchOptionsPtr createBatchOps() override;

  // iterator.
  class RocksDBIterator : public Iterator {
   public:
    RocksDBIterator(std::unique_ptr<rocksdb_internal::Iterator> it)
        : iterator_(std::move(it)) {}
    void seek(std::string_view key) override;
    void seekToFirst() override;
    void seekToLast() override;
    void next() override;
    Result<Void> status() const override;
    bool valid() const override;
    std::string_view key() const override;
    std::string_view value() const override;
    void destroy() override;

   private:
    std::unique_ptr<rocksdb_internal::Iterator> iterator_;
  };
  IteratorPtr createIterator() override;

  // create a RocksDB instance.
  static std::unique_ptr<RocksDBStore> create(const Config &config, const Options &options);

 protected:
  // initialize rocksdb.
  Result<Void> init(const Options &optionsIn);

 private:
  const Config &config_;
  std::unique_ptr<rocksdb_internal::DB> db_;
};

}  // namespace hf3fs::kv

#pragma once

#include <folly/IPAddressV4.h>

#include "common/net/Socket.h"
#include "common/utils/Address.h"
#include "common/utils/Duration.h"
#include "common/utils/FdWrapper.h"

namespace hf3fs::net {

class TcpSocket : public Socket {
 public:
  TcpSocket() = default;
  explicit TcpSocket(int fd)
      : fd_(fd) {}

  ~TcpSocket() override { close(); }

  // file descriptor monitored by epoll. [Socket]
  int fd() const final { return fd_; }

  // start the connection with timeout in the IO thread pool executor.
  CoTryTask<void> connect(Address addr, Duration timeout);

  // describe the address of the peer.
  std::string describe() final { return peerAddr_.str(); }

  // return the address of the peer.
  auto peerAddr() const { return peerAddr_; }

  folly::IPAddressV4 peerIP() final { return folly::IPAddressV4::fromLong(peerAddr_.ip); }

  // poll read and/or write events for this socket.
  Result<Events> poll(uint32_t events) final;

  // receive a piece of buffer. return the received size.
  Result<size_t> recv(folly::MutableByteRange buf) final;

  // send a batch of buffers. return the send size.
  Result<size_t> send(struct iovec iov[], uint32_t len) final;

  // flush the write buffers. ignore for TCP.
  Result<Void> flush() final { return Void{}; }

  // check the liveness of the socket.
  Result<Void> check() final { return Void{}; };

  // close this socket.
  void close() { fd_.close(); }

  // initialize this socket.
  Result<Void> init(bool isDomainSocket = false);

  // valid.
  bool valid() const { return fd_.valid(); }

  // ::poll with timeout.
  Result<Events> pollWithTimeout(short events, Duration timeout);

  // send a buffer in sync mode.
  Result<Void> sendSync(uint8_t *buff, uint32_t size, RelativeTime startTime, Duration timeout);

  // receive a piece of buffer in sync mode.
  Result<Void> recvSync(folly::MutableByteRange buf, RelativeTime startTime, Duration timeout);

 private:
  FdWrapper fd_;
  Address peerAddr_{0};
};

}  // namespace hf3fs::net

#pragma once

#include <folly/ThreadLocal.h>
#include <leveldb/cache.h>
#include <leveldb/db.h>
#include <leveldb/write_batch.h>

#include "LevelDBLogger.h"
#include "kv/KVStore.h"

namespace hf3fs::kv {

class LevelDBStore : public KVStore {
 public:
  LevelDBStore(const Config &config)
      : config_(config) {}

  // get value corresponding to key.
  Result<std::string> get(std::string_view key) final;

  // get the first key which is greater than input key.
  Result<Void> iterateKeysWithPrefix(std::string_view prefix,
                                     uint32_t limit,
                                     IterateFunc func,
                                     std::optional<std::string> *nextValidKey = nullptr) final;

  // put a key-value pair.
  Result<Void> put(std::string_view key, std::string_view value, bool sync = false) final;

  // remove a key-value pair.
  Result<Void> remove(std::string_view key) final;

  // batch operations.
  class LevelDBBatchOperations : public BatchOperations {
   public:
    LevelDBBatchOperations(LevelDBStore &db)
        : db_(db) {}
    // put a key-value pair.
    void put(std::string_view key, std::string_view value) override;
    // remove a key.
    void remove(std::string_view key) override;
    // clear a batch operations.
    void clear() override { writeBatch_.Clear(); }
    // commit a batch of operations.
    Result<Void> commit() override;
    // destroy self.
    void destroy() override;

   private:
    LevelDBStore &db_;
    leveldb::WriteBatch writeBatch_;
  };
  BatchOptionsPtr createBatchOps() override;

  // iterator.
  class LevelDBIterator : public Iterator {
   public:
    LevelDBIterator(std::unique_ptr<leveldb::Iterator> it)
        : iterator_(std::move(it)) {}
    void seek(std::string_view key) override;
    void seekToFirst() override;
    void seekToLast() override;
    void next() override;
    Result<Void> status() const override;
    bool valid() const override;
    std::string_view key() const override;
    std::string_view value() const override;
    void destroy() override;

   private:
    std::unique_ptr<leveldb::Iterator> iterator_;
  };
  IteratorPtr createIterator() override;

  // create a LevelDB instance.
  static std::unique_ptr<LevelDBStore> create(const Config &config, const Options &options);

 protected:
  // initialize leveldb.
  Result<Void> init(const Options &optionsIn);

 private:
  const Config &config_;
  std::unique_ptr<leveldb::DB> db_;
  std::unique_ptr<leveldb::Cache> cache_;
  std::unique_ptr<LevelDBLogger> logger_;
};

}  // namespace hf3fs::kv

#pragma once

#include <folly/hash/Hash.h>
#include <folly/sorted_vector_types.h>
#include <functional>
#include <string>
#include <utility>
#include <variant>

#include "common/serde/Serde.h"
#include "common/utils/UtcTime.h"

namespace hf3fs::monitor {

class TagSet {
 public:
  using Map = folly::sorted_vector_map<std::string, std::string>;
  using Iterator = Map::iterator;
  using ConstIterator = Map::const_iterator;

  TagSet() = default;
  TagSet(std::initializer_list<Map::value_type> init)
      : tag_set_(init) {}

  void addTag(std::string tagName, std::string tagValue) { tag_set_.emplace(tagName, tagValue); }

  static TagSet create(std::string tagName, std::string tagValue) {
    TagSet ts;
    ts.addTag(std::move(tagName), std::move(tagValue));
    return ts;
  }

  TagSet newTagSet(std::string tagName, std::string tagValue) const {
    TagSet newTagSet(*this);
    newTagSet.addTag(tagName, tagValue);
    return newTagSet;
  }

  std::map<String, String> asMap() const { return {begin(), end()}; }

  size_t size() const { return tag_set_.size(); }

  Iterator begin() { return tag_set_.begin(); }
  Iterator end() { return tag_set_.end(); }

  ConstIterator begin() const { return tag_set_.begin(); }
  ConstIterator end() const { return tag_set_.end(); }

  ConstIterator cbegin() const { return tag_set_.cbegin(); }
  ConstIterator cend() const { return tag_set_.cend(); }

  Iterator find(const std::string &key) { return tag_set_.find(key); }
  ConstIterator find(const std::string &key) const { return tag_set_.find(key); }

  bool contains(const std::string &key) const { return tag_set_.find(key) != tag_set_.end(); }
  std::string &operator[](const std::string &key) { return tag_set_[key]; }

  void traverseTags(std::function<void(const std::string &, const std::string &)> fn) const {
    for (auto &tag : tag_set_) {
      fn(tag.first, tag.second);
    }
  }

  friend bool operator==(const TagSet &a, const TagSet &b) { return a.tag_set_ == b.tag_set_; }

 private:
  SERDE_CLASS_FIELD(tag_set, Map{});
};

inline TagSet instanceTagSet(std::string instance) {
  constexpr std::string_view kMetricInstanceTag = "instance";
  return {{std::string(kMetricInstanceTag), instance}};
}

inline TagSet threadTagSet(const std::string_view &threadName) {
  constexpr std::string_view kMetricThreadTag = "thread";
  return {{std::string(kMetricThreadTag), std::string(threadName)}};
}

struct Distribution {
  constexpr static auto kTypeNameForSerde = "dist";

  SERDE_STRUCT_FIELD(cnt, double{});
  SERDE_STRUCT_FIELD(sum, double{});
  SERDE_STRUCT_FIELD(min, double{});
  SERDE_STRUCT_FIELD(max, double{});
  SERDE_STRUCT_FIELD(p50, double{});
  SERDE_STRUCT_FIELD(p90, double{});
  SERDE_STRUCT_FIELD(p95, double{});
  SERDE_STRUCT_FIELD(p99, double{});

 public:
  double mean() const { return cnt == 0.0 ? 0.0 : sum / cnt; }
};

struct Sample {
  SERDE_STRUCT_FIELD(name, std::string{});
  SERDE_STRUCT_FIELD(tags, TagSet{});
  SERDE_STRUCT_FIELD(timestamp, UtcTime{});
  SERDE_STRUCT_FIELD(value, (std::variant<int64_t, Distribution>{}));

 public:
  bool isNumber() const { return std::holds_alternative<int64_t>(value); }
  int64_t number() const { return std::get<int64_t>(value); }

  bool isDistribution() const { return std::holds_alternative<Distribution>(value); }
  Distribution &dist() { return std::get<Distribution>(value); }
  const Distribution &dist() const { return std::get<Distribution>(value); }
};

}  // namespace hf3fs::monitor

template <>
struct std::hash<hf3fs::monitor::TagSet> {
  std::size_t operator()(const hf3fs::monitor::TagSet &tag_set) const {
    return folly::hash::hash_range(tag_set.cbegin(), tag_set.cend());
  }
};

#pragma once

#include <chrono>
#include <cstdint>
#include <string>
#include <string_view>

#include "common/utils/Result.h"

namespace hf3fs {

class Duration : public std::chrono::nanoseconds {
 public:
  using is_serde_copyable = void;

  explicit constexpr Duration(std::chrono::nanoseconds duration = {})
      : std::chrono::nanoseconds(duration) {}

  auto asSec() const { return std::chrono::duration_cast<std::chrono::seconds>(*this); }
  auto asUs() const { return std::chrono::duration_cast<std::chrono::microseconds>(*this); }
  auto asMs() const { return std::chrono::duration_cast<std::chrono::milliseconds>(*this); }

  std::string toString() const;
  std::string serdeToReadable() const { return toString(); }

  auto operator+(Duration o) const { return Duration{std::chrono::nanoseconds(*this) + std::chrono::nanoseconds(o)}; }
  auto operator-(Duration o) const { return Duration{std::chrono::nanoseconds(*this) - std::chrono::nanoseconds(o)}; }
  auto operator*(auto times) const {
    return Duration{std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::nanoseconds(*this) * times)};
  }

  operator std::chrono::microseconds() const { return asUs(); }
  operator std::chrono::milliseconds() const { return asMs(); }

  constexpr auto operator<=>(const Duration &) const = default;

  static Result<Duration> from(std::string_view input);

  static Duration zero() { return Duration(std::chrono::nanoseconds::zero()); }
};

inline constexpr Duration operator""_ns(unsigned long long v) { return Duration(std::chrono::nanoseconds(v)); }
inline constexpr Duration operator""_us(unsigned long long v) { return Duration(std::chrono::microseconds(v)); }
inline constexpr Duration operator""_ms(unsigned long long v) { return Duration(std::chrono::milliseconds(v)); }
inline constexpr Duration operator""_s(unsigned long long v) { return Duration(std::chrono::seconds(v)); }
inline constexpr Duration operator""_min(unsigned long long v) { return Duration(std::chrono::minutes(v)); }
inline constexpr Duration operator""_h(unsigned long long v) { return Duration(std::chrono::hours(v)); }
inline constexpr Duration operator""_d(unsigned long long v) { return Duration(std::chrono::days(v)); }

class RelativeTime : public std::chrono::nanoseconds {
 public:
  static auto now() { return RelativeTime{std::chrono::steady_clock::now().time_since_epoch()}; }
  static auto zero() { return RelativeTime{std::chrono::nanoseconds::zero()}; }

  auto operator-(RelativeTime time) const {
    return Duration{std::chrono::nanoseconds(*this) - std::chrono::nanoseconds(time)};
  }

  auto operator+(Duration time) const {
    return RelativeTime{std::chrono::nanoseconds(*this) + std::chrono::nanoseconds(time)};
  }
  auto operator-(Duration time) const {
    return RelativeTime{std::chrono::nanoseconds(*this) - std::chrono::nanoseconds(time)};
  }
};

}  // namespace hf3fs

FMT_BEGIN_NAMESPACE

template <>
struct formatter<hf3fs::Duration> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(const hf3fs::Duration &duration, FormatContext &ctx) const {
    return formatter<std::string_view>::format(duration.toString(), ctx);
  }
};

FMT_END_NAMESPACE

#pragma once

#include <folly/Overload.h>
#include <folly/Utility.h>
#include <gtest/gtest_prod.h>
#include <optional>
#include <utility>
#include <variant>

#include "common/kv/ITransaction.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Duration.h"
#include "common/utils/Path.h"
#include "fbs/meta/Common.h"
#include "meta/components/AclCache.h"
#include "meta/store/DirEntry.h"
#include "meta/store/Inode.h"
#include "meta/store/Utils.h"

namespace hf3fs::meta::server {

/**
 * Path Resolution.
 *
 * Note: PathResolveOp always use snapshotLoad, so it won't add any key into read conflict set.
 * User should add keys into read conflict set manually if needed.
 */
class PathResolveOp : folly::NonCopyableNonMovable {
 public:
  struct ResolveResult {
    // for parent, we may already got it's Inode, or just dirEntry points to it, or just cached acl
    std::variant<std::pair<InodeId, Acl>, Inode, DirEntry> parent;
    std::optional<DirEntry> dirEntry;

    ResolveResult(std::variant<std::pair<InodeId, Acl>, Inode, DirEntry> parent, std::optional<DirEntry> dirEntry)
        : parent(std::move(parent)),
          dirEntry(std::move(dirEntry)) {}

    InodeId getParentId() const { return getInodeId(parent); }
    Acl getParentAcl() const { return getDirectoryAcl(parent); }
    CoTryTask<Inode> getParentInode(kv::IReadOnlyTransaction &txn) const {
      if (std::holds_alternative<Inode>(parent)) {
        co_return std::get<Inode>(parent);
      } else if (std::holds_alternative<DirEntry>(parent)) {
        co_return co_await std::get<DirEntry>(parent).snapshotLoadInode(txn);
      } else {
        auto parentId = std::get<std::pair<InodeId, Acl>>(parent).first;
        co_return (co_await Inode::snapshotLoad(txn, parentId)).then(checkMetaFound<Inode>);
      }
    }
  };

  struct ResolveRangeResult : ResolveResult {
    Path missing;
    ResolveRangeResult(ResolveResult result, Path missing)
        : ResolveResult(std::move(result)),
          missing(missing) {}
  };

  PathResolveOp(IReadOnlyTransaction &txn, AclCache &aclCache, const UserInfo &userInfo, Path *trace = nullptr)
      : PathResolveOp(txn, aclCache, userInfo, trace, 4, 8, 5_s) {}
  PathResolveOp(IReadOnlyTransaction &txn,
                AclCache &aclCache,
                const UserInfo &userInfo,
                Path *trace,
                size_t maxSymlinkCount,
                size_t maxSymlinkDepth,
                Duration aclCacheTime)
      : txn_(txn),
        user_(userInfo),
        aclCache_(aclCache),
        trace_(trace),
        depth_(0),
        symlinkCnt_(0),
        maxSymlinkCount_(maxSymlinkCount),
        maxSymlinkDepth_(maxSymlinkDepth),
        aclCacheTime_(aclCacheTime),
        pathComponents_(0) {}
  ~PathResolveOp();

  CoTryTask<Inode> inode(const PathAt &path, AtFlags flags, bool checkRefCnt);
  CoTryTask<DirEntry> dirEntry(const PathAt &path, AtFlags flags);

  CoTryTask<ResolveResult> path(const PathAt &path, AtFlags flags);
  CoTryTask<ResolveResult> byDirectoryInodeId(InodeId inodeId);
  CoTryTask<ResolveRangeResult> pathRange(const PathAt &path);

  CoTryTask<ResolveResult> symlink(DirEntry entry);

 private:
  template <typename>
  FRIEND_TEST(TestResolve, ResolveComponent);

  CoTryTask<DirEntry> dirEntry(InodeId parent, const Path &path, bool followLastSymlink);

  CoTryTask<ResolveResult> path(InodeId parent, const Path &path);
  CoTryTask<ResolveResult> path(InodeId parent, const Path &path, bool followLastSymlink);
  CoTryTask<ResolveResult> pathComponent(const std::variant<InodeId, DirEntry> &parent, const Path &name);
  CoTryTask<ResolveResult> pathRange(InodeId parent, Path::const_iterator &begin, const Path::const_iterator &end);

  IReadOnlyTransaction &txn_;
  const UserInfo &user_;
  AclCache &aclCache_;
  Path *trace_;

  size_t depth_;
  size_t symlinkCnt_;

  size_t maxSymlinkCount_;
  size_t maxSymlinkDepth_;
  Duration aclCacheTime_;

  size_t pathComponents_;
};

}  // namespace hf3fs::meta::server

#pragma once

#include <folly/executors/CPUThreadPoolExecutor.h>

#include "common/utils/ConfigBase.h"
#include "common/utils/ConstructLog.h"

namespace hf3fs {
class CPUExecutorGroup {
 public:
  enum class ExecutorStrategy {
    SHARED_QUEUE,  // fallback to CPUThreadPoolExecutor
    SHARED_NOTHING,
    WORK_STEALING,
    ROUND_ROBIN,
    GROUP_WAITING_4,
    GROUP_WAITING_8,
  };

  struct Config : public ConfigBase<Config> {
    CONFIG_ITEM(threadCount, 32);
    CONFIG_ITEM(strategy, ExecutorStrategy::ROUND_ROBIN);
  };

  CPUExecutorGroup(uint32_t threadCount,
                   std::string name,
                   ExecutorStrategy executorStrategy = ExecutorStrategy::SHARED_QUEUE);

  CPUExecutorGroup(std::string name, const Config &config)
      : CPUExecutorGroup(config.threadCount(), std::move(name), config.strategy()) {}

  ~CPUExecutorGroup();

  folly::CPUThreadPoolExecutor &get(size_t i) const { return *executors_[i]; };

  folly::CPUThreadPoolExecutor &randomPick() const;

  folly::CPUThreadPoolExecutor &pickNext() const {
    auto pos = next_.fetch_add(1, std::memory_order_acq_rel);
    return get(pos % size());
  }

  folly::CPUThreadPoolExecutor &pickNextFree() const;

  size_t size() const { return executors_.size(); }

  void join();

  auto &getAll() { return executors_; }
  auto &getAll() const { return executors_; }

  const std::string &getName() const { return name_; }
  folly::ThreadPoolExecutor::PoolStats getPoolStats() const;

 private:
  ConstructLog<"CPUExecutorGroup"> constructLog_;
  const std::string name_;
  std::vector<std::unique_ptr<folly::CPUThreadPoolExecutor>> executors_;
  mutable std::atomic<size_t> next_{0};
};
}  // namespace hf3fs

#pragma once

#include <folly/concurrency/ConcurrentHashMap.h>
#include <optional>
#include <vector>

#include "common/monitor/Recorder.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/FdWrapper.h"
#include "common/utils/Path.h"
#include "common/utils/Result.h"
#include "common/utils/RobinHood.h"
#include "storage/store/ChunkFileStore.h"
#include "storage/store/ChunkMetaStore.h"
#include "storage/store/ChunkMetadata.h"

namespace hf3fs::storage {

enum class PointQueryStrategy {
  NONE,
  CLASSIC,
  MODERN,
};

class ChunkStore {
 public:
  class Config : public ConfigBase<Config> {
    CONFIG_OBJ(kv_store, kv::KVStore::Config);
    CONFIG_OBJ(file_store, ChunkFileStore::Config);
    CONFIG_OBJ(meta_store, ChunkMetaStore::Config);
    CONFIG_ITEM(mutex_num, 257u, ConfigCheckers::isPositivePrime<uint32_t>);
    CONFIG_ITEM(kv_path, Path{});
    CONFIG_HOT_UPDATED_ITEM(migrate_kv_store, false);
    CONFIG_HOT_UPDATED_ITEM(force_persist, true);
    CONFIG_HOT_UPDATED_ITEM(point_query_strategy, PointQueryStrategy::NONE);
  };

  using Map = folly::ConcurrentHashMap<ChunkId, ChunkInfo>;

  ChunkStore(const Config &config, GlobalFileStore &globalFileStore)
      : config_(config),
        fileStore_(config_.file_store(), globalFileStore),
        metaStore_(config_.meta_store(), fileStore_) {}

  ChunkMetaStore &chunkMetaStore() { return metaStore_; }

  // create chunk store.
  Result<Void> create(const PhysicalConfig &config);

  // load chunk store.
  Result<Void> load(const PhysicalConfig &config);

  // add new chunk size.
  Result<Void> addChunkSize(const std::vector<Size> &sizeList);

  // migrate meta store.
  Result<Void> migrate(const PhysicalConfig &config) { return metaStore_.migrate(config_.kv_store(), config); }

  // get meta of a chunk file.
  Result<Map::ConstIterator> get(const ChunkId &chunkId);

  // create a new chunk file.
  Result<Void> createChunk(const ChunkId &chunkId,
                           uint32_t chunkSize,
                           ChunkInfo &chunkInfo,
                           folly::CPUThreadPoolExecutor &executor,
                           bool allowToAllocate);

  // set meta of a chunk file.
  Result<Void> set(const ChunkId &chunkId, const ChunkInfo &chunkInfo, bool persist = true);

  // remove a chunk file.
  Result<Void> remove(ChunkId chunkId, ChunkInfo &chunkInfo);

  // recycle a batch of chunks.
  Result<bool> punchHole() { return metaStore_.punchHole(); }

  // sync meta kv.
  Result<Void> sync() { return metaStore_.sync(); }

  // query chunks: the chunk ids in result are in reverse lexicographical order
  Result<std::vector<std::pair<ChunkId, ChunkMetadata>>> queryChunks(const ChunkIdRange &chunkIdRange);

  // list all chunk ids.
  Result<Void> getAllMetadata(ChunkMetaVector &metas);

  // get meta iterator.
  auto metaIterator() { return metaStore_.iterator(); }

  // get used size.
  uint64_t usedSize() const { return metaStore_.usedSize(); }

  // get reserved and unrecycled size.
  Result<Void> unusedSize(int64_t &reservedSize, int64_t &unrecycledSize) {
    return metaStore_.unusedSize(reservedSize, unrecycledSize);
  }

  // get all uncommitted chunk ids.
  const auto &uncommitted() { return metaStore_.uncommitted(); }

  // reset uncommitted chunk to committed state.
  Result<Void> resetUncommitted(ChainVer chainVer);

  // enable or disable emergency recycling.
  void setEmergencyRecycling(bool enable) { return metaStore_.setEmergencyRecycling(enable); }

 private:
  const Config &config_;
  ChunkFileStore fileStore_;
  ChunkMetaStore metaStore_;
  TargetId targetId_;
  monitor::TagSet tag_;
  static constexpr auto kShardsNum = 32u;
  std::array<Map, kShardsNum> maps_;
};

}  // namespace hf3fs::storage

#pragma once

#include <algorithm>
#include <cmath>
#include <exception>
#include <fmt/core.h>
#include <folly/CancellationToken.h>
#include <folly/Function.h>
#include <folly/MPMCQueue.h>
#include <folly/concurrency/UnboundedQueue.h>
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <folly/experimental/coro/BoundedQueue.h>
#include <folly/futures/Future.h>
#include <memory>
#include <optional>
#include <stdexcept>
#include <vector>

#include "common/kv/IKVEngine.h"
#include "common/kv/ITransaction.h"
#include "common/kv/KeyPrefix.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Duration.h"
#include "common/utils/Result.h"
#include "meta/store/DirEntry.h"
#include "meta/store/Inode.h"

namespace hf3fs::meta::server {

class MetaScan {
 public:
  struct Options {
    // scan options
    int threads = 4;
    int coroutines = 8;
    int items_per_getrange = -1;
    double backoff_min_wait = 0.1;   // 100ms
    double backoff_max_wait = 5;     // 5s
    double backoff_total_wait = 60;  // 60s
    // log level
    std::string logging;
    // create FDB client with given config path
    std::string fdb_cluster_file;
  };

  MetaScan(Options options,
           std::shared_ptr<kv::IKVEngine> kvEngine = {} /* create new fdb client if kvEngine is not set */);
  ~MetaScan();

  std::vector<Inode> getInodes();
  std::vector<DirEntry> getDirEntries();

  kv::IKVEngine &kvEngine() { return *kvEngine_; }

 private:
  struct KeyRange {
    std::string begin;
    std::string end;
    bool hasMore;

    KeyRange()
        : begin(),
          end(),
          hasMore(false) {}
    KeyRange(std::string begin, std::string end)
        : begin(std::move(begin)),
          end(std::move(end)),
          hasMore(true) {}

    static std::vector<KeyRange> split(std::string prefix);

    CoTryTask<kv::IReadOnlyTransaction::GetRangeResult> snapshotGetRange(kv::IReadOnlyTransaction &txn, int32_t limit);

    std::string describe() const {
      return fmt::format("[begin {:02x}, end {:02x}, hasMore {}]", fmt::join(begin, ""), fmt::join(end, ""), hasMore);
    }
  };

  template <typename T>
  struct BackgroundTask {
    folly::coro::BoundedQueue<std::vector<T>> queue;
    folly::SemiFuture<Result<Void>> future;
    folly::CancellationSource cancel;

    BackgroundTask(size_t cap)
        : queue(cap),
          future(folly::SemiFuture<Result<Void>>::makeEmpty()),
          cancel() {}
  };

  void createKVEngine();

  template <typename T>
  std::vector<T> waitResult(std::optional<BackgroundTask<T>> &task);

  CoTryTask<void> scanInode(BackgroundTask<Inode> &task);
  CoTryTask<void> scanDirEntry(BackgroundTask<DirEntry> &task);

  template <typename T>
  CoTryTask<void> scan(kv::KeyPrefix prefix, BackgroundTask<T> &task);

  template <typename T>
  CoTryTask<void> scanRange(KeyRange range, BackgroundTask<T> &task);

  std::mutex mutex_;
  Options options_;
  std::optional<std::jthread> fdbNetwork_;
  std::shared_ptr<kv::IKVEngine> kvEngine_;
  folly::CPUThreadPoolExecutor exec_;
  std::optional<BackgroundTask<Inode>> scanInodeTask_;
  std::optional<BackgroundTask<DirEntry>> scanDirEntryTask_;
};

}  // namespace hf3fs::meta::server

#pragma once

#include <algorithm>
#include <cstdint>
#include <folly/Utility.h>
#include <folly/dynamic.h>
#include <folly/hash/Checksum.h>
#include <folly/json.h>
#include <folly/logging/Logger.h>
#include <folly/logging/xlog.h>
#include <limits>
#include <string_view>
#include <utility>
#include <vector>

#include "common/serde/Serde.h"
#include "common/utils/MagicEnum.hpp"
#include "common/utils/Path.h"
#include "common/utils/UtcTime.h"
#include "fbs/meta/Common.h"
#include "fbs/meta/Schema.h"
#include "fbs/mgmtd/MgmtdTypes.h"

namespace hf3fs::meta::server {

struct Event {
  enum class Type { Create, Mkdir, HardLink, Remove, Truncate, OpenWrite, CloseWrite, Rename, Symlink, GC };

  Type type;
  folly::dynamic data;

  Event(Type type)
      : Event(type, folly::dynamic::object()) {
    addField("event", magic_enum::enum_name(type));
    addField("ts", UtcClock::now().toMicroseconds());
  }
  Event(Type type, folly::dynamic data)
      : type(type),
        data(std::move(data)) {}

  void log() const;
  void log(const folly::json::serialization_opts &opts) const;

  Event &addField(folly::dynamic key, folly::dynamic value) {
    data.insert(std::move(key), std::move(value));
    return *this;
  }
};

struct MetaEventTrace {
  SERDE_STRUCT_FIELD(eventType, Event::Type::Create);
  SERDE_STRUCT_FIELD(inodeId, InodeId());
  SERDE_STRUCT_FIELD(parentId, InodeId());
  SERDE_STRUCT_FIELD(entryName, std::string());
  SERDE_STRUCT_FIELD(dstParentId, InodeId());
  SERDE_STRUCT_FIELD(dstEntryName, std::string());
  SERDE_STRUCT_FIELD(ownerId, Uid(0));
  SERDE_STRUCT_FIELD(userId, Uid());
  SERDE_STRUCT_FIELD(client, ClientId{Uuid::zero()});
  SERDE_STRUCT_FIELD(tableId, flat::ChainTableId());
  SERDE_STRUCT_FIELD(inodeType, InodeType::File);
  SERDE_STRUCT_FIELD(nlink, uint16_t(0));
  SERDE_STRUCT_FIELD(length, uint64_t(0));
  SERDE_STRUCT_FIELD(truncateVer, uint64_t(0));
  SERDE_STRUCT_FIELD(dynStripe, uint32_t(0));
  SERDE_STRUCT_FIELD(oflags, OpenFlags());
  SERDE_STRUCT_FIELD(recursiveRemove, false);
  SERDE_STRUCT_FIELD(removedChunks, size_t(0));
  SERDE_STRUCT_FIELD(pruneSession, false);
  SERDE_STRUCT_FIELD(symLinkTarget, Path());
  SERDE_STRUCT_FIELD(origPath, Path());
};

}  // namespace hf3fs::meta::server

/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#pragma once

#include <folly/Memory.h>
#include <folly/SpinLock.h>
#include <folly/ThreadLocal.h>
#include <folly/concurrency/AtomicSharedPtr.h>
#include <folly/stats/TDigest.h>
#include <memory>

namespace hf3fs {

/*
 * Stat digests, such as TDigest, can be expensive to merge. It is faster to
 * buffer writes and merge them in larger chunks. DigestBuilder buffers writes
 * to improve performance.
 *
 * Values are stored in a cpu local buffer. Hot stats will merge the cpu local
 * buffer into a cpu-local digest when the buffer size is reached.
 *
 * All methods in this class are thread safe, but it probably doesn't make sense
 * for multiple threads to call build simultaneously. A typical usage is to
 * buffer writes for a period of time, and then have one thread call build to
 * merge the buffer into some other DigestT instance.
 */
class DigestBuilder {
 public:
  explicit DigestBuilder(size_t bufferSize, size_t digestSize);

  /*
   * Builds a TDigest from the buffer. All values used to build the TDigest are
   * removed from the buffer.
   */
  folly::TDigest build();

  /*
   * Adds a value to the buffer.
   */
  void append(double value);

 private:
  struct State {
    folly::SpinLock mutex;
    std::atomic<bool> collected{false};
    std::vector<double> buffer;
    std::unique_ptr<folly::TDigest> digest;
  };

  struct TlsBuffer {
    folly::atomic_shared_ptr<State> state{std::make_shared<State>()};

    TlsBuffer() noexcept = default;

    TlsBuffer(TlsBuffer &&other) = delete;
    TlsBuffer &operator=(TlsBuffer &&other) = delete;
  };

  struct TlsBufferTag {};

  folly::ThreadLocal<TlsBuffer, TlsBufferTag> tlsBuffers_;
  size_t bufferSize_;
  size_t digestSize_;
};

}  // namespace hf3fs

#pragma once

#include <folly/experimental/symbolizer/Symbolizer.h>
#include <folly/fibers/Semaphore.h>
#include <folly/logging/xlog.h>

namespace hf3fs {

/* Semaphore with changeable effective capacity and fixed max capacity */
class Semaphore : public folly::NonCopyableNonMovable {
 public:
  Semaphore(size_t usableTokens, size_t maxTokens = 4096)
      : semaphore_(maxTokens),
        maxTokens_(maxTokens),
        usableTokens_(std::min(usableTokens, maxTokens)) {
    XLOGF_IF(CRITICAL,
             usableTokens > maxTokens,
             "usableTokens {} > maxTokens {}, stack trace:\n{}",
             usableTokens,
             maxTokens,
             folly::symbolizer::getStackTraceStr());
    for (size_t i = usableTokens_; i < maxTokens_; i++) semaphore_.wait();
    XLOGF(INFO,
          "usableTokens: {}, maxTokens: {}, availableTokens: {}",
          usableTokens_,
          maxTokens_,
          semaphore_.getAvailableTokens());
  }

  size_t getMaxTokens() const { return maxTokens_; }

  size_t getUsableTokens() const {
    std::scoped_lock<std::mutex> lock(usableTokensMutex_);
    return usableTokens_;
  }

  void signal() { semaphore_.signal(); }

  void wait() { semaphore_.wait(); }

  bool try_wait() { return semaphore_.try_wait(); }

  folly::coro::Task<void> co_wait() { return semaphore_.co_wait(); }

  void changeUsableTokens(size_t newUsableTokens) {
    std::scoped_lock<std::mutex> lock(usableTokensMutex_);
    XLOGF(WARN, "Try to update usableTokens from {} to {}", usableTokens_, newUsableTokens);
    newUsableTokens = std::min(newUsableTokens, maxTokens_);
    for (size_t i = newUsableTokens; i < usableTokens_; i++) semaphore_.wait();
    for (size_t i = usableTokens_; i < newUsableTokens; i++) semaphore_.signal();
    XLOGF(WARN, "Updated usableTokens from {} to {}", usableTokens_, newUsableTokens);
    usableTokens_ = newUsableTokens;
  }

 private:
  folly::fibers::Semaphore semaphore_;
  const size_t maxTokens_;
  size_t usableTokens_;
  mutable std::mutex usableTokensMutex_;
};
}  // namespace hf3fs

#pragma once

#include <cassert>
#include <folly/Expected.h>
#include <folly/logging/xlog.h>

#include "common/utils/Status.h"

#define RETURN_ERROR(result) return hf3fs::makeError(std::move(result.error()))

#define MAKE_ERROR_F(code, ...) hf3fs::makeError((code), fmt::format(__VA_ARGS__))

#define CO_RETURN_ON_ERROR_WRAP(originResult, code, fmt_str, ...)                                                 \
  do {                                                                                \
    auto &&_sub_result = (originResult);                                                                          \
    if (UNLIKELY(_sub_result.hasError())) {                                                                       \
      co_return MAKE_ERROR_F(code, fmt_str ". origin error: {}" __VA_OPT__(, ) __VA_ARGS__, _sub_result.error()); \
    }                                                                                \
  } while (false)

#define CO_RETURN_ON_ERROR_MSG_WRAP(originResult, fmt_str, ...)                                     \
  do {                                                                                \
    auto &&_sub_result = (originResult);                                                            \
    if (UNLIKELY(_sub_result.hasError())) {                                                         \
      auto _code = _sub_result.error().code();                                                      \
      auto _msg = _sub_result.error().message();                                                    \
      co_return MAKE_ERROR_F(_code, fmt_str ". origin error: {}" __VA_OPT__(, ) __VA_ARGS__, _msg); \
    }                                                                                \
  } while (false)

#define RETURN_ON_ERROR_WRAP(originResult, code, fmt_str, ...)                                                 \
  do {                                                                                \
    auto &&_sub_result = (originResult);                                                                       \
    if (UNLIKELY(_sub_result.hasError())) {                                                                    \
      return MAKE_ERROR_F(code, fmt_str ". origin error: {}" __VA_OPT__(, ) __VA_ARGS__, _sub_result.error()); \
    }                                                                                \
  } while (false)

#define RETURN_ON_ERROR_MSG_WRAP(originResult, fmt_str, ...)                                     \
  do {                                                                                \
    auto &&_sub_result = (originResult);                                                         \
    if (UNLIKELY(_sub_result.hasError())) {                                                      \
      auto _code = _sub_result.error().code();                                                   \
      auto _msg = _sub_result.error().message();                                                 \
      return MAKE_ERROR_F(_code, fmt_str ". origin error: {}" __VA_OPT__(, ) __VA_ARGS__, _msg); \
    }                                                                                \
  } while (false)

#define RUNTIME_ASSERT_RESULT(result, fmt_str, ...)                                                         \
  do {                                                                                \
    auto &&_result = (result);                                                                              \
    XLOGF_IF(FATAL, _result.hasError(), fmt_str ". error: {}" __VA_OPT__(, ) __VA_ARGS__, _result.error()); \
  } while (false)

#define RETURN_ON_ERROR(result)         \
  do {                                  \
    auto &&_result = (result);          \
    if (UNLIKELY(_result.hasError())) { \
      RETURN_ERROR(_result);            \
    }                                   \
  } while (0)

#define RETURN_AND_LOG_ON_ERROR(result)         \
  do {                                          \
    auto &&_result = (result);                  \
    if (UNLIKELY(_result.hasError())) {         \
      XLOGF(ERR, "error: {}", _result.error()); \
      RETURN_ERROR(_result);                    \
    }                                           \
  } while (0)

#define CHECK_RESULT(name, expr)           \
  auto &&name##Result = (expr);            \
  if (UNLIKELY(name##Result.hasError())) { \
    RETURN_ERROR(name##Result);            \
  }                                        \
  auto &name = *name##Result

#define CO_RETURN_ERROR(result)                                                                      \
  do {                                                                                \
    assert(result.hasError());                                                                       \
    if (result.error().message().empty())                                                            \
      co_return hf3fs::makeError(result.error().code(),                                              \
                                 fmt::format("{}:{}, '{}' has error", __FILE__, __LINE__, #result)); \
    else                                                                                \
      co_return hf3fs::makeError(std::move(result.error()));                                         \
  } while (0)

#define CO_RETURN_ON_ERROR(result)      \
  do {                                  \
    auto &&_result = (result);          \
    if (UNLIKELY(_result.hasError())) { \
      CO_RETURN_ERROR(_result);         \
    }                                   \
  } while (0)

#define CO_RETURN_AND_LOG_ERROR(result)      \
  do {                                       \
    XLOGF(ERR, "error: {}", result.error()); \
    co_return result;                        \
  } while (0)

#define CO_RETURN_AND_LOG_ON_ERROR(result)      \
  do {                                          \
    auto &&_result = (result);                  \
    if (UNLIKELY(_result.hasError())) {         \
      XLOGF(ERR, "error: {}", _result.error()); \
      CO_RETURN_ERROR(_result);                 \
    }                                           \
  } while (0)

#define INLINE_TYPED_RUN(TYPE, code_block) [&]() -> TYPE code_block()
#define CO_INLINE_TYPED_RUN(TYPE, code_block) [&]() -> CoTask<TYPE> code_block()
#define INLINE_RUN(code_block) [&] code_block()

namespace hf3fs {
template <typename T>
using Result = folly::Expected<T, Status>;

template <typename T>
struct IsResult : std::false_type {};

template <typename T>
struct IsResult<Result<T>> : std::true_type {};

using Void = folly::Unit;

template <typename... Args>
[[nodiscard]] inline folly::Unexpected<Status> makeError(Args &&...args) {
  return folly::makeUnexpected(Status(std::forward<Args>(args)...));
}

template <typename T>
[[nodiscard]] inline status_code_t getStatusCode(const Result<T> &result) {
  return result.hasError() ? result.error().code() : StatusCode::kOK;
}

}  // namespace hf3fs

FMT_BEGIN_NAMESPACE

template <>
struct formatter<folly::Unit> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(const folly::Unit &, FormatContext &ctx) const {
    return fmt::format_to(ctx.out(), "Void{{}}");
  }
};

template <typename T>
struct formatter<hf3fs::Result<T>> : formatter<T> {
  template <typename FormatContext>
  auto format(const hf3fs::Result<T> &result, FormatContext &ctx) const {
    if (result.hasError()) return fmt::format_to(ctx.out(), "{}", result.error());
    return formatter<T>::format(result.value(), ctx);
  }
};

template <>
struct formatter<folly::Unexpected<hf3fs::Status>> : formatter<hf3fs::Status> {
  template <typename FormatContext>
  auto format(const folly::Unexpected<hf3fs::Status> &status, FormatContext &ctx) const {
    return formatter<hf3fs::Status>::format(status.error(), ctx);
  }
};

FMT_END_NAMESPACE

#pragma once

#include <fmt/format.h>

#include "Utils.h"
#include "common/app/ApplicationBase.h"
#include "common/utils/LogCommands.h"

DECLARE_string(cfg);
DECLARE_bool(dump_default_cfg);
DECLARE_bool(use_local_cfg);

namespace hf3fs {
template <typename Server>
class TwoPhaseApplication : public ApplicationBase {
  using Launcher = typename Server::Launcher;
  using ServerConfig = typename Server::Config;

 public:
  TwoPhaseApplication()
      : launcher_(std::make_unique<Launcher>()) {}

 private:
  struct Config : public ConfigBase<Config> {
    CONFIG_OBJ(common, typename Server::CommonConfig);
    CONFIG_OBJ(server, ServerConfig);
  };

  Result<Void> parseFlags(int *argc, char ***argv) final {
    RETURN_ON_ERROR(launcher_->parseFlags(argc, argv));

    static constexpr std::string_view dynamicConfigPrefix = "--config.";
    return ApplicationBase::parseFlags(dynamicConfigPrefix, argc, argv, configFlags_);
  }

  Result<Void> initApplication() final {
    if (FLAGS_dump_default_cfg) {
      fmt::print("{}\n", config_.toString());
      exit(0);
    }

    auto firstInitRes = launcher_->init();
    XLOGF_IF(FATAL, !firstInitRes, "Failed to init launcher: {}", firstInitRes.error());

    app_detail::loadAppInfo([this] { return launcher_->loadAppInfo(); }, appInfo_);
    app_detail::initConfig(config_, configFlags_, appInfo_, [this] { return launcher_->loadConfigTemplate(); });
    XLOGF(INFO, "Server config inited");

    app_detail::initCommonComponents(config_.common(), Server::kName, appInfo_.nodeId);

    onLogConfigUpdated_ = app_detail::makeLogConfigUpdateCallback(config_.common().log(), Server::kName);
    onMemConfigUpdated_ = app_detail::makeMemConfigUpdateCallback(config_.common().memory(), appInfo_.hostname);

    XLOGF(INFO, "Full Config:\n{}", config_.toString());
    app_detail::persistConfig(config_);

    XLOGF(INFO, "Start to init server");
    auto initRes = initServer();
    XLOGF_IF(FATAL, !initRes, "Init server failed: {}", initRes.error());
    XLOGF(INFO, "Init server finished");

    XLOGF(INFO, "Start to start server");
    auto startRes = startServer();
    XLOGF_IF(FATAL, !startRes, "Start server failed: {}", startRes.error());
    XLOGF(INFO, "Start server finished");

    launcher_.reset();

    return Void{};
  }

  void stop() final {
    XLOGF(INFO, "Stop TwoPhaseApplication...");
    if (launcher_) {
      launcher_.reset();
    }
    stopAndJoin(server_.get());
    server_.reset();
    XLOGF(INFO, "Stop TwoPhaseApplication finished");
  }

  config::IConfig *getConfig() final { return &config_; }

  const flat::AppInfo *info() const final { return &appInfo_; }

  bool configPushable() const final { return FLAGS_cfg.empty() && !FLAGS_use_local_cfg; }

  void onConfigUpdated() { app_detail::persistConfig(config_); }

 private:
  Result<Void> initServer() {
    server_ = std::make_unique<Server>(config_.server());
    RETURN_ON_ERROR(server_->setup());
    XLOGF(INFO, "{}", server_->describe());
    return Void{};
  }

  Result<Void> startServer() {
    auto startResult = launcher_->startServer(*server_, appInfo_);
    XLOGF_IF(FATAL, !startResult, "Start server failed: {}", startResult.error());
    appInfo_ = server_->appInfo();
    return Void{};
  }

  ConfigFlags configFlags_;

  Config config_;
  flat::AppInfo appInfo_;
  std::unique_ptr<Launcher> launcher_;
  std::unique_ptr<Server> server_;
  std::unique_ptr<ConfigCallbackGuard> onLogConfigUpdated_;
  std::unique_ptr<ConfigCallbackGuard> onMemConfigUpdated_;
};
}  // namespace hf3fs

#pragma once

#include <span>
#include <vector>

#include "StorageClient.h"
#include "StorageMessenger.h"
#include "UpdateChannelAllocator.h"
#include "common/net/Client.h"
#include "common/utils/Address.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Result.h"
#include "fbs/storage/Common.h"

namespace hf3fs::storage::client {

class StorageClientBlobImpl : public StorageClient {
 public:
  StorageClientBlobImpl(const ClientId &clientId, const Config &config, hf3fs::client::ICommonMgmtdClient &mgmtdClient);

  ~StorageClientBlobImpl() override;

  Result<Void> start() override;

  void stop() override;

  hf3fs::client::ICommonMgmtdClient &getMgmtdClient() override;

  CoTryTask<void> batchRead(std::span<ReadIO> readIOs,
                            const flat::UserInfo &userInfo,
                            const ReadOptions &options = ReadOptions(),
                            std::vector<ReadIO *> *failedIOs = nullptr) override;

  CoTryTask<void> batchWrite(std::span<WriteIO> writeIOs,
                             const flat::UserInfo &userInfo,
                             const WriteOptions &options = WriteOptions(),
                             std::vector<WriteIO *> *failedIOs = nullptr) override;

  CoTryTask<void> read(ReadIO &readIO,
                       const flat::UserInfo &userInfo,
                       const ReadOptions &options = ReadOptions()) override;

  CoTryTask<void> write(WriteIO &writeIO,
                        const flat::UserInfo &userInfo,
                        const WriteOptions &options = WriteOptions()) override;

  CoTryTask<void> queryLastChunk(std::span<QueryLastChunkOp> ops,
                                 const flat::UserInfo &userInfo,
                                 const ReadOptions &options = ReadOptions(),
                                 std::vector<QueryLastChunkOp *> *failedOps = nullptr) override;

  CoTryTask<void> removeChunks(std::span<RemoveChunksOp> ops,
                               const flat::UserInfo &userInfo,
                               const WriteOptions &options = WriteOptions(),
                               std::vector<RemoveChunksOp *> *failedOps = nullptr) override;

  CoTryTask<void> truncateChunks(std::span<TruncateChunkOp> ops,
                                 const flat::UserInfo &userInfo,
                                 const WriteOptions &options = WriteOptions(),
                                 std::vector<TruncateChunkOp *> *failedOps = nullptr) override;

  CoTryTask<void> listChunks(std::span<ListChunksOp> ops,
                             const flat::UserInfo &userInfo,
                             const ReadOptions &options = ReadOptions(),
                             std::vector<ListChunksOp *> *failedOps = nullptr) override;

  CoTryTask<void> batchCopy(std::span<CopyIO> copyIOs,
                            const flat::UserInfo &userInfo,
                            const WriteOptions &options = WriteOptions(),
                            std::vector<CopyIO *> *failedIOs = nullptr) override;

  CoTryTask<SpaceInfoRsp> querySpaceInfo(NodeId nodeId) override;

  CoTryTask<CreateTargetRsp> createTarget(NodeId nodeId, const CreateTargetReq &req) override;

  CoTryTask<std::vector<Result<QueryChunkRsp>>> queryChunk(const QueryChunkReq &req) override;

  CoTryTask<ChunkMetaVector> getAllChunkMetadata(const ChainId &chainId, const TargetId &targetId) override;
};

}  // namespace hf3fs::storage::client

#pragma once

#include <folly/executors/CPUThreadPoolExecutor.h>

#include "chunk_engine/src/cxx.rs.h"
#include "common/utils/CPUExecutorGroup.h"
#include "common/utils/CoLockManager.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/RobinHood.h"
#include "fbs/mgmtd/HeartbeatInfo.h"
#include "fbs/storage/Common.h"
#include "storage/service/TargetMap.h"
#include "storage/store/StorageTarget.h"

namespace hf3fs::test {
struct StorageTargetsHelper;
}

namespace hf3fs::storage {

class StorageTargets {
 public:
  class Config : public ConfigBase<Config> {
    CONFIG_ITEM(target_paths, std::vector<Path>{}, [](auto &vec) { return !vec.empty(); });
    CONFIG_ITEM(target_num_per_path, 0u);
    CONFIG_HOT_UPDATED_ITEM(collect_all_fds, true);
    CONFIG_HOT_UPDATED_ITEM(space_info_cache_timeout, 5_s);
    CONFIG_HOT_UPDATED_ITEM(allow_disk_without_uuid, false);
    CONFIG_HOT_UPDATED_ITEM(create_engine_path, true);
    CONFIG_OBJ(storage_target, StorageTarget::Config);
  };

  class CreateConfig : public ConfigBase<CreateConfig> {
    CONFIG_ITEM(target_ids, std::vector<flat::TargetId::UnderlyingType>{});
    CONFIG_ITEM(physical_file_count, 256u);
    CONFIG_ITEM(allow_disk_without_uuid, false);
    CONFIG_ITEM(allow_existing_targets, false);
    CONFIG_ITEM(chunk_size_list, (std::vector<Size>{512_KB, 1_MB, 2_MB, 4_MB, 16_MB, 64_MB}));
    CONFIG_ITEM(only_chunk_engine, false);
  };

  StorageTargets(const Config &config, AtomicallyTargetMap &targetMap)
      : config_(config),
        targetMap_(targetMap) {}
  ~StorageTargets();

  Result<Void> init(CPUExecutorGroup &executor);

  // create a batch of storage targets.
  Result<Void> create(const CreateConfig &createConfig);

  // create new storage target.
  Result<Void> create(const CreateTargetReq &req);

  // open a batch of storage targets.
  Result<Void> load(CPUExecutorGroup &executor);

  // load a target.
  Result<Void> loadTarget(const Path &targetPath);

  // get fd list.
  auto &fds() const { return fds_; }

  // get space info.
  Result<std::vector<SpaceInfo>> spaceInfos(bool force);

  // get target paths.
  auto &targetPaths() const { return targetPaths_; }

  // get manufacturers.
  auto &manufacturers() const { return manufacturers_; }

  // global file store.
  auto &globalFileStore() { return globalFileStore_; }

  // chunk engines.
  auto &engines() const { return engines_; }

  // remove target.
  Result<Void> removeChunkEngineTarget(ChainId chainId, uint32_t diskIndex) {
    auto &engine = *engines_[diskIndex];
    return ChunkEngine::removeAllChunks(engine, chainId);
  }

 private:
  friend struct test::StorageTargetsHelper;
  ConstructLog<"storage::StorageTargets"> constructLog_;
  const Config &config_;
  AtomicallyTargetMap &targetMap_;
  GlobalFileStore globalFileStore_;

  std::vector<Path> targetPaths_;
  std::vector<std::string> manufacturers_;
  std::map<Path, uint32_t> pathToDiskIndex_;
  std::vector<rust::Box<chunk_engine::Engine>> engines_;

  CoLockManager<> targetLocks_;
  RelativeTime spaceInfoUpdatedTime_;
  std::vector<SpaceInfo> cachedSpaceInfos_;

  std::vector<int> fds_;
};

}  // namespace hf3fs::storage

#pragma once

#include <folly/ThreadLocal.h>
#include <mutex>

#include "common/utils/ConfigBase.h"
#include "common/utils/FdWrapper.h"
#include "common/utils/Path.h"
#include "common/utils/Result.h"
#include "common/utils/RobinHood.h"
#include "common/utils/Shards.h"
#include "storage/store/ChunkFileView.h"
#include "storage/store/ChunkMetadata.h"
#include "storage/store/GlobalFileStore.h"
#include "storage/store/PhysicalConfig.h"

namespace hf3fs::storage {

class ChunkFileStore {
 public:
  class Config : public ConfigBase<Config> {
    CONFIG_ITEM(preopen_chunk_size_list, std::set<Size>{});
  };

  ChunkFileStore(const Config &config, GlobalFileStore &globalFileStore)
      : config_(config),
        globalFileStore_(globalFileStore) {}

  // create file store.
  Result<Void> create(const PhysicalConfig &config);

  // load file store.
  Result<Void> load(const PhysicalConfig &config);

  // add new chunk size.
  Result<Void> addChunkSize(const std::vector<Size> &sizeList);

  // get a chunk file. [thread-safe]
  Result<ChunkFileView> open(ChunkFileId fileId);

  // recycle a chunk. [thread-safe]
  Result<Void> punchHole(ChunkFileId fileId, size_t offset);

  // allocate space. [thread-safe]
  Result<Void> allocate(ChunkFileId fileId, size_t offset, size_t size);

 protected:
  // open inner file. [thread-safe]
  Result<FileDescriptor *> openInnerFile(ChunkFileId fileId, bool createFile = false);

  // create inner file.
  Result<Void> createInnerFile(Size chunkSize);

 private:
  const Config &config_;
  GlobalFileStore &globalFileStore_;

  Path path_;
  uint32_t physicalFileCount_{};

  constexpr static auto kShardsNum = 64u;
  folly::ThreadLocal<robin_hood::unordered_map<ChunkFileId, FileDescriptor *>> tlsCache_;
};

}  // namespace hf3fs::storage

/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#pragma once

#include <folly/Range.h>
#include <folly/logging/LogWriter.h>

#include "IWritableFile.h"

namespace hf3fs::logging {

/**
 * A LogWriter implementation that immediately writes to a file descriptor when
 * it is invoked.
 *
 * The downside of this class is that logging I/O occurs directly in your
 * normal program threads, so that logging I/O may block or slow down normal
 * processing.
 *
 * However, one benefit of this class is that log messages are written out
 * immediately, so if your program crashes, all log messages generated before
 * the crash will have already been written, and no messages will be lost.
 */
class ImmediateFileWriter : public folly::LogWriter {
 public:
  /**
   * Construct an ImmediateFileWriter that writes to the specified File object.
   */
  explicit ImmediateFileWriter(std::shared_ptr<IWritableFile> file);

  using LogWriter::writeMessage;
  void writeMessage(folly::StringPiece buffer, uint32_t flags = 0) override;
  void flush() override;

  /**
   * Returns true if the output steam is a tty.
   */
  bool ttyOutput() const override { return file_->ttyOutput(); }

 private:
  ImmediateFileWriter(ImmediateFileWriter const &) = delete;
  ImmediateFileWriter &operator=(ImmediateFileWriter const &) = delete;

  std::shared_ptr<IWritableFile> file_;
};
}  // namespace hf3fs::logging

#pragma once

#include <folly/Expected.h>
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <folly/executors/ThreadPoolExecutor.h>
#include <folly/logging/xlog.h>
#include <memory>
#include <optional>
#include <unistd.h>

#include "common/utils/ConfigBase.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Result.h"
#include "mgmtd/service/MgmtdConfig.h"
#include "mgmtd/service/MgmtdOperator.h"
#include "mgmtd/service/MgmtdService.h"

namespace hf3fs::mgmtd {

class MockMgmtd {
 public:
  struct Config : ConfigBase<Config> {
    CONFIG_OBJ(mgmtd, MgmtdConfig);
    CONFIG_ITEM(node_id, 1u);
    CONFIG_ITEM(name, "mock-mgmtd");
    CONFIG_ITEM(cluster_id, "mock-cluster");
  };

  static CoTryTask<std::unique_ptr<MockMgmtd>> create(Config config,
                                                      std::shared_ptr<kv::IKVEngine> kvEngine,
                                                      CPUExecutorGroup *exec) {
    flat::ServiceGroupInfo groupInfo;
    groupInfo.services.emplace(mgmtd::MgmtdService::kServiceName);
    groupInfo.endpoints.push_back(net::Address::from("TCP://127.0.0.1:8000").value());

    flat::AppInfo info;
    info.nodeId = flat::NodeId(config.node_id());
    info.pid = static_cast<uint32_t>(getpid());
    info.serviceGroups = {groupInfo};
    info.clusterId = config.cluster_id();

    auto env = std::make_shared<core::ServerEnv>();
    env->setAppInfo(info);
    env->setKvEngine(kvEngine);
    env->setBackgroundExecutor(exec);

    auto mgmtd = std::make_unique<MockMgmtd>();
    mgmtd->config_ = config;
    mgmtd->mgmtdOperator_ = std::make_unique<mgmtd::MgmtdOperator>(std::move(env), mgmtd->config_.mgmtd());

    mgmtd->mgmtdOperator_->start();
    co_return mgmtd;
  }

  void stop() { mgmtdOperator_.reset(); }

  std::unique_ptr<mgmtd::MgmtdService> getService() { return std::make_unique<mgmtd::MgmtdService>(*mgmtdOperator_); }

  MgmtdOperator &getOperator() { return *mgmtdOperator_; }

 private:
  Config config_;
  std::unique_ptr<mgmtd::MgmtdOperator> mgmtdOperator_;
};

}  // namespace hf3fs::mgmtd

#pragma once

#include "common/serde/Serde.h"
#include "common/utils/Path.h"
#include "common/utils/Size.h"
#include "kv/KVStore.h"

namespace hf3fs::storage {

// Physical configuration of the storage target. Store in `target.toml`.
static inline constexpr auto kPhysicalConfigFileName = "target.toml";

class PhysicalConfig {
  SERDE_STRUCT_FIELD(path, Path{});
  SERDE_STRUCT_FIELD(target_id, uint64_t{});
  SERDE_STRUCT_FIELD(block_device_uuid, std::string{});
  SERDE_STRUCT_FIELD(allow_disk_without_uuid, false);
  SERDE_STRUCT_FIELD(allow_existing_targets, false);

  SERDE_STRUCT_FIELD(physical_file_count, 256u);
  SERDE_STRUCT_FIELD(chunk_size_list, (std::vector<Size>{512_KB, 1_MB, 2_MB, 4_MB, 16_MB, 64_MB}));
  SERDE_STRUCT_FIELD(chain_id, uint32_t{});
  SERDE_STRUCT_FIELD(kv_store_type, kv::KVStore::Type::LevelDB);
  SERDE_STRUCT_FIELD(has_sentinel, false);
  SERDE_STRUCT_FIELD(kv_store_name, std::string{"meta"});
  SERDE_STRUCT_FIELD(kv_path, std::optional<Path>{});
  SERDE_STRUCT_FIELD(only_chunk_engine, false);

 public:
  Path kvPath() const {
    if (kv_path.has_value()) {
      return *kv_path / fmt::format("{}_{}", kv_store_name, target_id);
    }
    return path / kv_store_name;
  }
};

}  // namespace hf3fs::storage

#pragma once

#include <condition_variable>
#include <folly/Utility.h>
#include <optional>

#include "common/net/IOWorker.h"
#include "common/net/Listener.h"
#include "common/net/Processor.h"
#include "common/net/ThreadPoolGroup.h"
#include "common/serde/ClientContext.h"
#include "common/serde/Services.h"
#include "common/utils/Address.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/Duration.h"
#include "common/utils/Result.h"

namespace hf3fs::net {

class ServiceGroup : public folly::MoveOnly {
 public:
  class Config : public ConfigBase<Config> {
    CONFIG_ITEM(services, std::set<std::string>{});
    CONFIG_ITEM(use_independent_thread_pool, false);
    CONFIG_ITEM(network_type, Address::RDMA);
    CONFIG_HOT_UPDATED_ITEM(check_connections_interval, 60_s);
    CONFIG_HOT_UPDATED_ITEM(connection_expiration_time, 1_d);
    CONFIG_OBJ(io_worker, IOWorker::Config);
    CONFIG_OBJ(processor, Processor::Config);
    CONFIG_OBJ(listener, Listener::Config);
  };

  explicit ServiceGroup(const Config &config, ThreadPoolGroup &tpg);

  ~ServiceGroup();

  // add normal service.
  template <class Service>
  Result<Void> addSerdeService(std::unique_ptr<Service> &&obj, std::optional<Address::Type> type = {}) {
    return serdeServices_.addService(std::move(obj), type.value_or(config_.network_type()) == Address::RDMA);
  }

  // setup this group.
  Result<Void> setup();

  // start this group.
  Result<Void> start();

  // stop and wait.
  void stopAndJoin();

  // set processor frozen.
  void setFrozen(bool frozen = true) { processor_.setFrozen(frozen, config_.network_type() == Address::RDMA); }

  // get listening address list.
  auto &addressList() const { return listener_.addressList(); }

  // get service name list.
  auto &serviceNameList() const { return config_.services(); }

  // get io worker.
  auto &ioWorker() { return ioWorker_; }

  // set coroutines pool getter for processor.
  void setCoroutinesPoolGetter(auto &&g) { processor_.setCoroutinesPoolGetter(std::forward<decltype(g)>(g)); }

 protected:
  CoTask<void> checkConnectionsRegularly();

 private:
  ConstructLog<"net::ServiceGroup"> constructLog_;
  const Config &config_;
  ThreadPoolGroup &tpg_;
  serde::Services serdeServices_;
  Processor processor_;
  IOWorker ioWorker_;
  Listener listener_;

  CancellationSource cancel_;
  folly::SemiFuture<Void> future_;
};

}  // namespace hf3fs::net

#pragma once

#include <atomic>
#include <folly/Random.h>
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <memory>
#include <vector>

#include "common/utils/BoundedQueue.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/Result.h"
#include "storage/aio/AioStatus.h"
#include "storage/aio/BatchReadJob.h"
#include "storage/store/StorageTargets.h"

namespace hf3fs::storage {

class AioReadWorker {
 public:
  enum class IoEngine {
    libaio,
    io_uring,
    random,
  };

  class Config : public ConfigBase<Config> {
    CONFIG_ITEM(num_threads, 32ul);
    CONFIG_ITEM(queue_size, 4096u);
    CONFIG_ITEM(max_events, 512u);
    CONFIG_ITEM(enable_io_uring, true);
    CONFIG_HOT_UPDATED_ITEM(min_complete, 128u);
    CONFIG_HOT_UPDATED_ITEM(wait_all_inflight, false);      // deprecated.
    CONFIG_HOT_UPDATED_ITEM(inflight_control_offset, 128);  // deprecated.
    CONFIG_HOT_UPDATED_ITEM(ioengine, IoEngine::libaio);

   public:
    inline bool useIoUring() const {
      if (!enable_io_uring()) {
        return false;
      }
      switch (ioengine()) {
        case IoEngine::io_uring:
          return true;
        case IoEngine::libaio:
          return false;
        case IoEngine::random:
          return folly::Random::rand32() & 1;
      }
    }
  };

  AioReadWorker(const Config &config)
      : config_(config),
        queue_(config.queue_size()),
        executors_(std::make_pair(config_.num_threads(), config_.num_threads()),
                   std::make_shared<folly::NamedThreadFactory>("AioRead")) {}
  ~AioReadWorker();

  CoTask<void> enqueue(AioReadJobIterator job) { co_await queue_.co_enqueue(job); }

  Result<Void> start(const std::vector<int> &fds, const std::vector<struct iovec> &iovecs);

  Result<Void> stopAndJoin();

 protected:
  Result<Void> run(AioStatus &aioStatus, IoUringStatus &ioUringStatus);

 private:
  ConstructLog<"storage::AioReadWorker"> constructLog_;
  const Config &config_;
  BoundedQueue<AioReadJobIterator> queue_;

  folly::CPUThreadPoolExecutor executors_;
  std::atomic<uint32_t> initialized_{};
  folly::Synchronized<Result<Void>, std::mutex> initResult_{Void{}};
};

}  // namespace hf3fs::storage

#pragma once

#include <folly/concurrency/AtomicSharedPtr.h>
#include <optional>

#include "common/serde/ClientContext.h"
#include "common/serde/MessagePacket.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Result.h"
#include "fbs/meta/Service.h"
#include "fbs/mgmtd/ChainRef.h"
#include "stubs/MetaService/IMetaServiceStub.h"
#include "stubs/MetaService/MetaServiceStub.h"
#include "stubs/common/Stub.h"

namespace hf3fs::meta {

using flat::ChainTableId;
using flat::Uid;
using flat::UserInfo;

class DummyMetaServiceStub : public IMetaServiceStub {
 public:
  ~DummyMetaServiceStub() override = default;

#define NOT_IMPLEMENTED_FUNC(NAME, REQ, RESP)                                                          \
  CoTryTask<RESP> NAME(const REQ &req, const net::UserRequestOptions &, serde::Timestamp *) override { \
    co_return co_await NAME(req);                                                                      \
  }                                                                                \
  virtual CoTryTask<RESP> NAME(const REQ &) { co_return makeError(StatusCode::kNotImplemented); }

  NOT_IMPLEMENTED_FUNC(statFs, StatFsReq, StatFsRsp);
  NOT_IMPLEMENTED_FUNC(stat, StatReq, StatRsp);
  NOT_IMPLEMENTED_FUNC(create, CreateReq, CreateRsp);
  NOT_IMPLEMENTED_FUNC(mkdirs, MkdirsReq, MkdirsRsp);
  NOT_IMPLEMENTED_FUNC(symlink, SymlinkReq, SymlinkRsp);
  NOT_IMPLEMENTED_FUNC(hardLink, HardLinkReq, HardLinkRsp);
  NOT_IMPLEMENTED_FUNC(remove, RemoveReq, RemoveRsp);
  NOT_IMPLEMENTED_FUNC(open, OpenReq, OpenRsp);
  NOT_IMPLEMENTED_FUNC(sync, SyncReq, SyncRsp);
  NOT_IMPLEMENTED_FUNC(close, CloseReq, CloseRsp);
  NOT_IMPLEMENTED_FUNC(rename, RenameReq, RenameRsp);
  NOT_IMPLEMENTED_FUNC(list, ListReq, ListRsp);
  NOT_IMPLEMENTED_FUNC(truncate, TruncateReq, TruncateRsp);
  NOT_IMPLEMENTED_FUNC(getRealPath, GetRealPathReq, GetRealPathRsp);
  NOT_IMPLEMENTED_FUNC(setAttr, SetAttrReq, SetAttrRsp);
  NOT_IMPLEMENTED_FUNC(pruneSession, PruneSessionReq, PruneSessionRsp);
  NOT_IMPLEMENTED_FUNC(dropUserCache, DropUserCacheReq, DropUserCacheRsp);
  NOT_IMPLEMENTED_FUNC(lockDirectory, LockDirectoryReq, LockDirectoryRsp);
  NOT_IMPLEMENTED_FUNC(testRpc, TestRpcReq, TestRpcRsp);
  NOT_IMPLEMENTED_FUNC(batchStat, BatchStatReq, BatchStatRsp);
  NOT_IMPLEMENTED_FUNC(batchStatByPath, BatchStatByPathReq, BatchStatByPathRsp);

  virtual CoTryTask<AuthRsp> authenticate(const AuthReq &req) { co_return AuthRsp{req.user}; }
  CoTryTask<AuthRsp> authenticate(const AuthReq &req, const net::UserRequestOptions &, serde::Timestamp *) override {
    co_return co_await authenticate(req);
  }

#undef NOT_IMPLEMENTED_FUNC
};

struct MockMetaStubHolder {
  void setStub(std::unique_ptr<DummyMetaServiceStub> st) { stub = std::move(st); }

  folly::atomic_shared_ptr<IMetaServiceStub> stub;
};

class DummyMetaServiceStubWithInode : public DummyMetaServiceStub {
 public:
  DummyMetaServiceStubWithInode(std::variant<File, Directory, Symlink> data)
      : inode_({InodeId(0x10de1d), {data, Acl{Uid(0), Gid(0), Permission(0777)}}}) {}

  CoTryTask<CreateRsp> create(const CreateReq &) override { co_return CreateRsp(inode_, false); }
  CoTryTask<OpenRsp> open(const OpenReq &) override { co_return OpenRsp(inode_, false); }
  CoTryTask<StatRsp> stat(const StatReq &) override { co_return StatRsp(inode_); }

 protected:
  Inode inode_;
};
class DummyMetaServiceStubWithDir : public DummyMetaServiceStubWithInode {
 public:
  DummyMetaServiceStubWithDir()
      : DummyMetaServiceStubWithInode(Directory{InodeId(0x10de1dff), Layout::newEmpty(ChainTableId(1), 512 << 10, 8)}) {
  }
};
class DummyMetaServiceStubWithFile : public DummyMetaServiceStubWithInode {
 public:
  DummyMetaServiceStubWithFile()
      : DummyMetaServiceStubWithInode(File(Layout::newEmpty(ChainTableId(1), 512 << 10, 8))) {}
};
class DummyMetaServiceStubWithSymlink : public DummyMetaServiceStubWithInode {
 public:
  DummyMetaServiceStubWithSymlink()
      : DummyMetaServiceStubWithInode(Symlink{"/b"}) {}
};
}  // namespace hf3fs::meta

template <>
struct ::hf3fs::stubs::StubMockContext<hf3fs::meta::IMetaServiceStub> {
  std::shared_ptr<meta::MockMetaStubHolder> stub;
};

namespace hf3fs::meta {

template <>
class MetaServiceStub<hf3fs::stubs::StubMockContext<IMetaServiceStub>> : public IMetaServiceStub {
 public:
  using ContextType = hf3fs::stubs::StubMockContext<IMetaServiceStub>;

  MetaServiceStub(ContextType ctx)
      : stub_(std::move(ctx.stub)) {}
  ~MetaServiceStub() override = default;

#define FORWARD_RPC_FUNC(NAME, REQ, RESP)                                                                           \
  CoTryTask<RESP> NAME(const REQ &req, const net::UserRequestOptions &opts, serde::Timestamp *timestamp) override { \
    co_return co_await stub_->stub.load()->NAME(req, opts, timestamp);                                              \
  }

  FORWARD_RPC_FUNC(statFs, StatFsReq, StatFsRsp);
  FORWARD_RPC_FUNC(stat, StatReq, StatRsp);
  FORWARD_RPC_FUNC(create, CreateReq, CreateRsp);
  FORWARD_RPC_FUNC(mkdirs, MkdirsReq, MkdirsRsp);
  FORWARD_RPC_FUNC(symlink, SymlinkReq, SymlinkRsp);
  FORWARD_RPC_FUNC(hardLink, HardLinkReq, HardLinkRsp);
  FORWARD_RPC_FUNC(remove, RemoveReq, RemoveRsp);
  FORWARD_RPC_FUNC(open, OpenReq, OpenRsp);
  FORWARD_RPC_FUNC(sync, SyncReq, SyncRsp);
  FORWARD_RPC_FUNC(close, CloseReq, CloseRsp);
  FORWARD_RPC_FUNC(rename, RenameReq, RenameRsp);
  FORWARD_RPC_FUNC(list, ListReq, ListRsp);
  FORWARD_RPC_FUNC(truncate, TruncateReq, TruncateRsp);
  FORWARD_RPC_FUNC(getRealPath, GetRealPathReq, GetRealPathRsp);
  FORWARD_RPC_FUNC(setAttr, SetAttrReq, SetAttrRsp);
  FORWARD_RPC_FUNC(pruneSession, PruneSessionReq, PruneSessionRsp);
  FORWARD_RPC_FUNC(dropUserCache, DropUserCacheReq, DropUserCacheRsp);
  FORWARD_RPC_FUNC(authenticate, AuthReq, AuthRsp);
  FORWARD_RPC_FUNC(lockDirectory, LockDirectoryReq, LockDirectoryRsp);
  FORWARD_RPC_FUNC(testRpc, TestRpcReq, TestRpcRsp);
  FORWARD_RPC_FUNC(batchStat, BatchStatReq, BatchStatRsp);
  FORWARD_RPC_FUNC(batchStatByPath, BatchStatByPathReq, BatchStatByPathRsp);

#undef FORWARD_RPC_FUNC

 private:
  std::shared_ptr<MockMetaStubHolder> stub_;
};

}  // namespace hf3fs::meta

#pragma once

#include <folly/executors/CPUThreadPoolExecutor.h>

#include "common/utils/BoundedQueue.h"
#include "storage/store/StorageTargets.h"
#include "storage/update/UpdateJob.h"

namespace hf3fs::storage {

class UpdateWorker {
 public:
  class Config : public ConfigBase<Config> {
    CONFIG_ITEM(queue_size, 4096u);
    CONFIG_ITEM(num_threads, 32ul);
    CONFIG_ITEM(bg_num_threads, 8ul);
  };

  UpdateWorker(const Config &config)
      : config_(config),
        executors_(std::make_pair(config_.num_threads(), config_.num_threads()),
                   std::make_shared<folly::NamedThreadFactory>("Update")),
        bgExecutors_(std::make_pair(config_.bg_num_threads(), config_.bg_num_threads()),
                     std::make_shared<folly::NamedThreadFactory>("Recycle")) {}
  ~UpdateWorker() { stopAndJoin(); }

  Result<Void> start(uint32_t numberOfDisks);
  void stopAndJoin();

  CoTask<void> enqueue(UpdateJob *job) {
    assert(job->target()->diskIndex() < queueVec_.size());
    co_await queueVec_[job->target()->diskIndex()]->co_enqueue(job);
  }

 protected:
  using Queue = BoundedQueue<UpdateJob *>;
  void run(Queue &queue);

 private:
  ConstructLog<"storage::UpdateWorker"> constructLog_;
  const Config &config_;
  std::vector<std::unique_ptr<Queue>> queueVec_;
  folly::CPUThreadPoolExecutor executors_;
  folly::CPUThreadPoolExecutor bgExecutors_;
  std::atomic_flag stopped_;
};

}  // namespace hf3fs::storage

#pragma once

#include <atomic>
#include <condition_variable>
#include <folly/ProducerConsumerQueue.h>
#include <mutex>
#include <shared_mutex>

#include "common/monitor/ClickHouseClient.h"
#include "common/monitor/LogReporter.h"
#include "common/monitor/MonitorCollectorClient.h"
#include "common/monitor/Recorder.h"
#include "common/monitor/Sample.h"
#include "common/utils/Address.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/Coroutine.h"
#include "common/utils/ObjectPool.h"

namespace hf3fs::monitor {

class MonitorInstance;
class Collector {
 public:
  Collector();
  ~Collector();

  void add(const std::string &name, Recorder &var);
  void del(const std::string &name, Recorder &var);
  void collectAll(size_t bucketIndex, std::vector<Sample> &samples, bool cleanInactive);

 private:
  void resize(size_t n);

  struct Impl;
  std::unique_ptr<Impl> impl;

  friend class MonitorInstance;
};

class Monitor {
 public:
  class ReporterConfig : public ConfigBase<ReporterConfig> {
    CONFIG_VARIANT_TYPE("clickhouse");
    CONFIG_OBJ(clickhouse, ClickHouseClient::Config);
    CONFIG_OBJ(log, LogReporter::Config);
    CONFIG_OBJ(monitor_collector, MonitorCollectorClient::Config);
  };

  class Config : public ConfigBase<Config> {
    CONFIG_OBJ_ARRAY(reporters, ReporterConfig, 4, [](auto &) { return 0; });
    CONFIG_ITEM(num_collectors, 1);
    CONFIG_HOT_UPDATED_ITEM(collect_period, 1_s);
  };

  static Result<Void> start(const Config &config, String hostnameExtra = {});
  static void stop();

  static std::unique_ptr<MonitorInstance> createMonitorInstance();
  static MonitorInstance &getDefaultInstance();
};

class MonitorInstance {
 public:
  ~MonitorInstance();
  Result<Void> start(const Monitor::Config &config, String hostnameExtra = {});
  void stop();

  size_t getCollectorCount() const { return collectorContexts_.size(); }

  Collector &getCollector() {
    static Collector collector;
    return collector;
  }

 private:
  static constexpr size_t kMaxNumSampleBatches = 60;
  using SampleBatch = std::vector<Sample>;
  using SampleBatchPool = ObjectPool<SampleBatch, kMaxNumSampleBatches, kMaxNumSampleBatches>;

  struct CollectorContext {
    folly::ProducerConsumerQueue<SampleBatchPool::Ptr> samplesQueue_ =
        folly::ProducerConsumerQueue<SampleBatchPool::Ptr>(kMaxNumSampleBatches);

    size_t bucketIndex = 0;
    std::mutex mutex_;
    std::condition_variable collectorCond_;
    std::condition_variable reporterCond_;
    std::jthread collectThread_;
    std::jthread reportThread_;
  };

  void periodicallyCollect(CollectorContext &context, const Monitor::Config &config);
  void reportSamples(CollectorContext &context, std::vector<std::unique_ptr<Reporter>> reporters);

  std::atomic<bool> stop_ = true;

  std::vector<std::unique_ptr<CollectorContext>> collectorContexts_;
};

}  // namespace hf3fs::monitor

#pragma once

#include <boost/core/ignore_unused.hpp>
#include <fcntl.h>
#include <folly/Likely.h>
#include <folly/lang/Bits.h>
#include <gtest/gtest_prod.h>
#include <memory>
#include <optional>
#include <queue>
#include <string_view>
#include <utility>
#include <variant>
#include <vector>

#include "client/mgmtd/ICommonMgmtdClient.h"
#include "client/storage/StorageClient.h"
#include "common/kv/IKVEngine.h"
#include "common/kv/ITransaction.h"
#include "common/kv/WithTransaction.h"
#include "common/monitor/Recorder.h"
#include "common/monitor/Sample.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Path.h"
#include "common/utils/Result.h"
#include "common/utils/Status.h"
#include "common/utils/UtcTime.h"
#include "fbs/meta/Service.h"
#include "meta/base/Config.h"
#include "meta/components/AclCache.h"
#include "meta/components/ChainAllocator.h"
#include "meta/components/FileHelper.h"
#include "meta/components/GcManager.h"
#include "meta/components/InodeIdAllocator.h"
#include "meta/components/SessionManager.h"
#include "meta/store/DirEntry.h"
#include "meta/store/Inode.h"
#include "meta/store/PathResolve.h"
#include "meta/store/Utils.h"

namespace hf3fs::meta::server {
using hf3fs::kv::IReadOnlyTransaction;
using hf3fs::kv::IReadWriteTransaction;

template <typename Rsp>
class IOperation {
 public:
  using RspT = Rsp;

  virtual ~IOperation() = default;

  virtual bool isReadOnly() = 0;
  virtual bool retryMaybeCommitted() { return true; }

  virtual bool needIdempotent(Uuid &clientId, Uuid &requestId) const {
    boost::ignore_unused(clientId, requestId);
    return false;
  }

  virtual std::string_view name() const { return "other"; }
  virtual flat::Uid user() const { return flat::Uid(-1); }

  virtual CoTryTask<Rsp> run(IReadWriteTransaction &) = 0;

  virtual void retry(const Status &) = 0;
  virtual void finish(const Result<Rsp> &) = 0;

  CoTryTask<Rsp> operator()(IReadWriteTransaction &txn) { co_return co_await run(txn); }
};

class MetaStore {
 public:
  MetaStore(const Config &config,
            analytics::StructuredTraceLog<MetaEventTrace> &metaEventTraceLog,
            std::shared_ptr<Distributor> distributor,
            std::shared_ptr<InodeIdAllocator> inodeAlloc,
            std::shared_ptr<ChainAllocator> chainAlloc,
            std::shared_ptr<FileHelper> fileHelper,
            std::shared_ptr<SessionManager> sessionManager,
            std::shared_ptr<GcManager> gcManager)
      : config_(config),
        metaEventTraceLog_(metaEventTraceLog),
        distributor_(distributor),
        inodeAlloc_(inodeAlloc),
        chainAlloc_(chainAlloc),
        fileHelper_(fileHelper),
        sessionManager_(sessionManager),
        gcManager_(gcManager),
        aclCache_(2 << 20 /* 2m acl */) {}

  auto &getEventTraceLog() { return metaEventTraceLog_; }

  template <typename Rsp>
  using Op = IOperation<Rsp>;

  template <typename Rsp>
  using OpPtr = std::unique_ptr<IOperation<Rsp>>;

  static OpPtr<Void> initFileSystem(ChainAllocator &chainAlloc, Layout rootLayout);

  OpPtr<Void> initFs(Layout rootLayout) { return MetaStore::initFileSystem(*chainAlloc_, rootLayout); }

  OpPtr<StatFsRsp> statFs(const StatFsReq &req);

  OpPtr<StatRsp> stat(const StatReq &req);

  OpPtr<BatchStatRsp> batchStat(const BatchStatReq &req);

  OpPtr<BatchStatByPathRsp> batchStatByPath(const BatchStatByPathReq &req);

  OpPtr<GetRealPathRsp> getRealPath(const GetRealPathReq &req);

  OpPtr<OpenRsp> open(OpenReq &req);

  OpPtr<CreateRsp> tryOpen(CreateReq &req);

  OpPtr<MkdirsRsp> mkdirs(const MkdirsReq &req);

  OpPtr<SymlinkRsp> symlink(const SymlinkReq &req);

  OpPtr<RemoveRsp> remove(const RemoveReq &req);

  OpPtr<RenameRsp> rename(const RenameReq &req);

  OpPtr<ListRsp> list(const ListReq &req);

  OpPtr<SyncRsp> sync(const SyncReq &req);

  OpPtr<HardLinkRsp> hardLink(const HardLinkReq &req);

  OpPtr<SetAttrRsp> setAttr(const SetAttrReq &req);

  OpPtr<PruneSessionRsp> pruneSession(const PruneSessionReq &req);

  OpPtr<TestRpcRsp> testRpc(const TestRpcReq &req);

  OpPtr<LockDirectoryRsp> lockDirectory(const LockDirectoryReq &req);

 private:
  template <typename>
  FRIEND_TEST(TestRemove, GC);

  template <typename Rsp>
  friend class Operation;

  const Config &config_;
  analytics::StructuredTraceLog<MetaEventTrace> &metaEventTraceLog_;
  std::shared_ptr<Distributor> distributor_;
  std::shared_ptr<InodeIdAllocator> inodeAlloc_;
  std::shared_ptr<ChainAllocator> chainAlloc_;
  std::shared_ptr<FileHelper> fileHelper_;
  std::shared_ptr<SessionManager> sessionManager_;
  std::shared_ptr<GcManager> gcManager_;
  AclCache aclCache_;
};

}  // namespace hf3fs::meta::server

#pragma once

#include <chrono>
#include <folly/Range.h>
#include <memory>
#include <sys/uio.h>

#include "common/net/Network.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Result.h"

namespace hf3fs::net {

class Socket {
 public:
  virtual ~Socket() = default;

  // describe this socket.
  virtual std::string describe() = 0;
  virtual folly::IPAddressV4 peerIP() = 0;

  // file descriptor monitored by epoll.
  virtual int fd() const = 0;

  using Events = uint32_t;
  constexpr static auto kEventReadableFlag = (1u << 0);
  constexpr static auto kEventWritableFlag = (1u << 1);
  // poll read and/or write events for this socket.
  virtual Result<Events> poll(uint32_t events) = 0;

  // receive a piece of buffer. return the received size.
  virtual Result<size_t> recv(folly::MutableByteRange buf) = 0;
  // send a batch of buffers. return the send size.
  virtual Result<size_t> send(struct iovec iov[], uint32_t len) = 0;
  // flush the write buffers.
  virtual Result<Void> flush() = 0;
  // check the liveness of the socket.
  virtual Result<Void> check() = 0;
};
using SocketPtr = std::shared_ptr<Socket>;

}  // namespace hf3fs::net

#pragma once

#include <atomic>
#include <folly/io/coro/ServerSocket.h>
#include <thread>

#include "common/net/IOWorker.h"
#include "common/net/Network.h"
#include "common/net/Transport.h"
#include "common/net/ib/IBConnectService.h"
#include "common/net/ib/IBSocket.h"
#include "common/utils/ConfigBase.h"

namespace hf3fs::net {

class ServiceGroup;

class Listener {
 public:
  class Config : public ConfigBase<Config> {
    CONFIG_ITEM(listen_port, uint16_t{0});
    CONFIG_ITEM(reuse_port, false);
    CONFIG_ITEM(listen_queue_depth, 4096u);
    CONFIG_ITEM(filter_list, std::set<std::string>{});  // use all available network cards if filter is empty.
    CONFIG_ITEM(rdma_listen_ethernet, true);            // support setup RDMA connection with Ethernet by default.
    CONFIG_ITEM(domain_socket_index, 1u);
    CONFIG_HOT_UPDATED_ITEM(rdma_accept_timeout, 15_s);
  };

  Listener(const Config &config,
           const IBSocket::Config &ibconfig,
           IOWorker &ioWorker,
           folly::IOThreadPoolExecutor &connThreadPool,
           Address::Type networkType);
  ~Listener() { stopAndJoin(); }

  // setup listener.
  Result<Void> setup();

  // start listening.
  Result<Void> start(ServiceGroup &group);

  // stop listening.
  void stopAndJoin();

  // get listening address list.
  const std::vector<Address> &addressList() const { return addressList_; }

 protected:
  // do listen with calcellation support.
  CoTask<void> listen(folly::coro::ServerSocket socket);

  // accept a TCP connection.
  CoTask<void> acceptTCP(std::unique_ptr<folly::coro::Transport> tr);

  // accept a RDMA connection.
  void acceptRDMA(std::unique_ptr<IBSocket> socket);
  CoTask<void> checkRDMA(std::weak_ptr<Transport> weak);

  // release a TCP connection.
  CoTask<void> release(folly::coro::ServerSocket /* socket */);

 private:
  const Config &config_;
  const IBSocket::Config &ibconfig_;
  IOWorker &ioWorker_;
  folly::IOThreadPoolExecutor &connThreadPool_;
  Address::Type networkType_;

  CancellationSource cancel_;
  std::vector<Address> addressList_;
  std::vector<folly::coro::ServerSocket> serverSockets_;
  std::atomic<size_t> running_{0};
};

}  // namespace hf3fs::net

#pragma once

#include <atomic>
#include <condition_variable>
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <mutex>

#include "client/storage/UpdateChannelAllocator.h"
#include "common/serde/Serde.h"
#include "common/utils/ConcurrencyLimiter.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/CoroutinesPool.h"
#include "common/utils/Duration.h"
#include "common/utils/Shards.h"
#include "fbs/storage/Common.h"
#include "storage/service/TargetMap.h"

namespace hf3fs::storage {

struct Components;

class ResyncWorker {
 public:
  enum FullSyncLevel {
    NONE,
    HEAVY,  // sync all.
  };
  struct Config : ConfigBase<Config> {
    CONFIG_ITEM(num_threads, 16ul);
    CONFIG_ITEM(num_channels, 1024u);
    CONFIG_HOT_UPDATED_ITEM(batch_size, 16u);
    CONFIG_HOT_UPDATED_ITEM(sync_start_timeout, 10_s);
    CONFIG_HOT_UPDATED_ITEM(full_sync_chains, std::set<uint32_t>{});  // full sync all chains if it is empty.
    CONFIG_HOT_UPDATED_ITEM(full_sync_level, FullSyncLevel::NONE);
    CONFIG_OBJ(pool, CoroutinesPoolBase::Config);
    CONFIG_OBJ(batch_concurrency_limiter, ConcurrencyLimiterConfig, [](auto &c) { c.set_max_concurrency(64); });
  };
  ResyncWorker(const Config &config, Components &components);

  // start resync worker.
  Result<Void> start();

  // stop resync worker. End all sync tasks immediately.
  Result<Void> stopAndJoin();

 protected:
  void loop();

  // handle sync job.
  CoTryTask<void> handleSync(VersionedChainId vChainId);

  // forward sync request.
  CoTryTask<void> forward(const TargetPtr &target,
                          const monitor::TagSet &tag,
                          const ClientId &clientId,
                          ChunkId chunkId,
                          UpdateType updateType,
                          uint32_t chunkSize);

 private:
  ConstructLog<"storage::ResyncWorker"> constructLog_;
  const Config &config_;
  Components &components_;
  folly::CPUThreadPoolExecutor executors_;
  CoroutinesPool<VersionedChainId> pool_;
  client::UpdateChannelAllocator updateChannelAllocator_;
  ConcurrencyLimiter<uint32_t> batchConcurrencyLimiter_;

  std::mutex mutex_;
  std::condition_variable cond_;
  std::atomic<bool> stopping_ = false;
  std::atomic<bool> started_ = false;
  std::atomic<bool> stopped_ = false;

  struct SyncingStatus {
    SERDE_STRUCT_FIELD(isSyncing, false);
    SERDE_STRUCT_FIELD(lastSyncingTime, RelativeTime{});
  };
  using SyncingChainIds = robin_hood::unordered_map<ChainId, SyncingStatus>;
  Shards<SyncingChainIds, 32> shards_;
  std::atomic_uint64_t requestId_;
};

}  // namespace hf3fs::storage

#pragma once

#include <variant>

#include "LocalTargetInfo.h"
#include "MgmtdTypes.h"
#include "common/app/AppInfo.h"
#include "common/app/ConfigStatus.h"

namespace hf3fs::flat {
class MetaHeartbeatInfo : public serde::SerdeHelper<MetaHeartbeatInfo> {
 public:
  static constexpr auto kTypeCode = NodeType::META;

  SERDE_STRUCT_FIELD(dummy, Void{});
};

class StorageHeartbeatInfo : public serde::SerdeHelper<StorageHeartbeatInfo> {
 public:
  static constexpr auto kTypeCode = NodeType::STORAGE;

  SERDE_STRUCT_FIELD(targets, std::vector<LocalTargetInfo>{});
};

class MgmtdHeartbeatInfo : public serde::SerdeHelper<MgmtdHeartbeatInfo> {
 public:
  static constexpr auto kTypeCode = NodeType::MGMTD;

  SERDE_STRUCT_FIELD(dummy, Void{});
};

class HeartbeatInfo : public serde::SerdeHelper<HeartbeatInfo> {
 public:
  HeartbeatInfo() = default;

  explicit HeartbeatInfo(FbsAppInfo appInfo)
      : app(std::move(appInfo)) {}

  HeartbeatInfo(FbsAppInfo appInfo, MetaHeartbeatInfo info)
      : HeartbeatInfo(std::move(appInfo)) {
    set(std::move(info));
  }

  HeartbeatInfo(FbsAppInfo appInfo, StorageHeartbeatInfo info)
      : HeartbeatInfo(std::move(appInfo)) {
    set(std::move(info));
  }

  HeartbeatInfo(FbsAppInfo appInfo, MgmtdHeartbeatInfo info)
      : HeartbeatInfo(std::move(appInfo)) {
    set(std::move(info));
  }

  NodeType type() const {
    return std::visit(
        [](auto &&v) {
          using T = std::decay_t<decltype(v)>;
          return T::kTypeCode;
        },
        info_);
  }

  MetaHeartbeatInfo &asMeta() { return std::get<MetaHeartbeatInfo>(info_); }

  const MetaHeartbeatInfo &asMeta() const { return std::get<MetaHeartbeatInfo>(info_); }

  StorageHeartbeatInfo &asStorage() { return std::get<StorageHeartbeatInfo>(info_); }

  const StorageHeartbeatInfo &asStorage() const { return std::get<StorageHeartbeatInfo>(info_); }

  MgmtdHeartbeatInfo &asMgmtd() { return std::get<MgmtdHeartbeatInfo>(info_); }

  const MgmtdHeartbeatInfo &asMgmtd() const { return std::get<MgmtdHeartbeatInfo>(info_); }

  void set(MetaHeartbeatInfo info) { info_ = std::move(info); }

  void set(StorageHeartbeatInfo info) { info_ = std::move(info); }

  void set(MgmtdHeartbeatInfo info) { info_ = std::move(info); }

  using Payload = std::variant<MetaHeartbeatInfo, StorageHeartbeatInfo, MgmtdHeartbeatInfo>;
  void set(Payload payload) { info_ = std::move(payload); }

  SERDE_CLASS_FIELD(info, Payload{});

  SERDE_STRUCT_FIELD(app, FbsAppInfo{});
  SERDE_STRUCT_FIELD(hbVersion,
                     HeartbeatVersion(1));  // HeartbeatVersion should be larger than server's so use 1 as default value
  SERDE_STRUCT_FIELD(
      configVersion,
      ConfigVersion(0));  // ConfigVersion should be smaller than or equal to server's so use 0 as default value
  SERDE_STRUCT_FIELD(configStatus, ConfigStatus::NORMAL);
};
}  // namespace hf3fs::flat

#pragma once

#include <folly/executors/CPUThreadPoolExecutor.h>
#include <folly/executors/IOThreadPoolExecutor.h>
#include <folly/executors/TimekeeperScheduledExecutor.h>
#include <folly/experimental/coro/WithCancellation.h>
#include <folly/experimental/symbolizer/Symbolizer.h>
#include <folly/synchronization/Baton.h>

#include "common/utils/CPUExecutorGroup.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/Coroutine.h"
#include "common/utils/ExecutorStatsReporter.h"

namespace hf3fs::net {

class ThreadPoolGroup {
 public:
  struct Config : public ConfigBase<Config> {
    CONFIG_ITEM(num_proc_threads, 2ul);
    CONFIG_ITEM(num_io_threads, 2ul);
    CONFIG_ITEM(num_bg_threads, 2ul);
    CONFIG_ITEM(num_connect_threads, 2ul);
    CONFIG_ITEM(collect_stats, false);
    CONFIG_ITEM(enable_work_stealing, false);  // deprecated
    CONFIG_ITEM(proc_thread_pool_stratetry, CPUExecutorGroup::ExecutorStrategy::SHARED_QUEUE);
    CONFIG_ITEM(io_thread_pool_stratetry, CPUExecutorGroup::ExecutorStrategy::SHARED_QUEUE);
    CONFIG_ITEM(bg_thread_pool_stratetry, CPUExecutorGroup::ExecutorStrategy::SHARED_QUEUE);
  };

  ThreadPoolGroup(const std::string &name, const Config &config)
      : fullname_(fmt::format("{}@{}", name, fmt::ptr(&config))),
        startStacktrace_(folly::symbolizer::getStackTraceStr()),
        procThreadPool_(config.num_proc_threads(), name + "Proc", config.proc_thread_pool_stratetry()),
        ioThreadPool_(config.num_io_threads(), name + "IO", config.io_thread_pool_stratetry()),
        bgThreadPool_(config.num_bg_threads(), name + "BG", config.bg_thread_pool_stratetry()),
        connThreadPool_(config.num_connect_threads(),
                        config.num_connect_threads(),
                        std::make_shared<folly::NamedThreadFactory>(name + "Conn")),
        procThreadPoolReporter_(procThreadPool_),
        ioThreadPoolReporter_(ioThreadPool_),
        bgThreadPoolReporter_(bgThreadPool_),
        connThreadPoolReporter_(connThreadPool_),
        stopping_(false) {
    if (config.collect_stats()) {
      future_ = co_withCancellation(cancel_.getToken(), collectStatsPeriodically())
                    .scheduleOn(&bgThreadPool_.randomPick())
                    .start();
    }
  }

  ~ThreadPoolGroup() {
    if (!stopping_) {
      XLOGF(WARN, "Thread pool group '{}' not stopped before destructed, created by {}", fullname_, startStacktrace_);
      stopAndJoin();
    }
  }

  auto &procThreadPool() { return procThreadPool_; }
  auto &ioThreadPool() { return ioThreadPool_; }
  auto &bgThreadPool() { return bgThreadPool_; }
  auto &connThreadPool() { return connThreadPool_; }

  void stopAndJoin();

 private:
  CoTask<void> collectStatsPeriodically();

 private:
  ConstructLog<"net::ThreadPoolGroup"> constructLog_;
  std::string fullname_;
  std::string startStacktrace_;
  CPUExecutorGroup procThreadPool_;
  CPUExecutorGroup ioThreadPool_;
  CPUExecutorGroup bgThreadPool_;
  folly::IOThreadPoolExecutor connThreadPool_;
  ExecutorStatsReporter<CPUExecutorGroup> procThreadPoolReporter_;
  ExecutorStatsReporter<CPUExecutorGroup> ioThreadPoolReporter_;
  ExecutorStatsReporter<CPUExecutorGroup> bgThreadPoolReporter_;
  ExecutorStatsReporter<folly::IOThreadPoolExecutor> connThreadPoolReporter_;
  folly::CancellationSource cancel_;
  folly::SemiFuture<Void> future_;
  std::atomic_bool stopping_;
};

}  // namespace hf3fs::net

#pragma once

#include <boost/algorithm/string.hpp>
#include <span>

#include "MgmtdTypes.h"
#include "common/app/AppInfo.h"
#include "common/app/ConfigStatus.h"
#include "common/serde/SerdeHelper.h"
#include "common/utils/Selector.h"
#include "common/utils/UtcTimeSerde.h"

namespace hf3fs::flat {

struct NodeInfo : public serde::SerdeHelper<NodeInfo> {
  std::vector<net::Address> extractAddresses(const String &serviceName,
                                             std::optional<net::Address::Type> addressType = std::nullopt) const;

  std::map<String, std::vector<net::Address>> getAllServices() const;

  bool operator==(const NodeInfo &other) const { return serde::equals(*this, other); }

  SERDE_STRUCT_FIELD(app, FbsAppInfo{});
  SERDE_STRUCT_FIELD(type, NodeType(NodeType::MGMTD));
  SERDE_STRUCT_FIELD(status, NodeStatus(NodeStatus::HEARTBEAT_CONNECTING));
  SERDE_STRUCT_FIELD(lastHeartbeatTs, UtcTime{});
  SERDE_STRUCT_FIELD(tags, std::vector<TagPair>{});
  SERDE_STRUCT_FIELD(configVersion, ConfigVersion(0));
  SERDE_STRUCT_FIELD(configStatus, ConfigStatus::NORMAL);
};

inline auto selectNodeByType(flat::NodeType type) {
  auto p = [type](const flat::NodeInfo &node) { return node.type == type; };
  return makeSelector<flat::NodeInfo>(std::move(p));
}

inline auto selectActiveNode() {
  auto p = [](const flat::NodeInfo &node) {
    return node.status == flat::NodeStatus::HEARTBEAT_CONNECTED || node.status == flat::NodeStatus::PRIMARY_MGMTD;
  };
  return makeSelector<flat::NodeInfo>(std::move(p));
}

inline auto selectNodeById(flat::NodeId id) {
  auto p = [id](const flat::NodeInfo &node) { return node.app.nodeId == id; };
  return makeSelector<flat::NodeInfo>(std::move(p));
}

inline auto selectNodeByStatus(flat::NodeStatus status) {
  auto p = [status](const flat::NodeInfo &node) { return node.status == status; };
  return makeSelector<flat::NodeInfo>(std::move(p));
}

inline auto selectNodeByTrafficZone(std::string_view zone) {
  std::vector<std::string_view> zones;
  if (!zone.empty()) {
    std::vector<std::string_view> tmp;
    boost::split(tmp, zone, boost::is_any_of(", "));
    std::copy_if(tmp.begin(), tmp.end(), std::back_inserter(zones), [](std::string_view s) { return !s.empty(); });
  }
  auto p = [zones = std::move(zones)](const flat::NodeInfo &node) {
    if (zones.empty()) return true;  // select any node if zone is empty
    for (const auto &tp : node.tags) {
      if (tp.key == kTrafficZoneTagKey && std::find(zones.begin(), zones.end(), tp.value) != zones.end()) return true;
    }
    return false;
  };
  return makeSelector<flat::NodeInfo>(std::move(p));
}

}  // namespace hf3fs::flat

#pragma once

#include <chrono>
#include <fmt/core.h>
#include <folly/SharedMutex.h>
#include <folly/Synchronized.h>
#include <folly/experimental/coro/BlockingWait.h>
#include <folly/experimental/coro/Mutex.h>
#include <folly/experimental/coro/SharedMutex.h>
#include <memory>
#include <optional>
#include <utility>

#include "client/mgmtd/ICommonMgmtdClient.h"
#include "client/storage/StorageClient.h"
#include "common/utils/BackgroundRunner.h"
#include "common/utils/CPUExecutorGroup.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Duration.h"
#include "common/utils/UtcTime.h"
#include "fbs/meta/Common.h"
#include "fbs/meta/Service.h"
#include "meta/base/Config.h"
#include "meta/store/Inode.h"

namespace hf3fs::meta::server {

using FsStatus = StatFsRsp;

class FileHelper {
 public:
  using RetryOptions = storage::client::RetryOptions;

  FileHelper(const Config &config,
             std::shared_ptr<client::ICommonMgmtdClient> mgmtdClient,
             std::shared_ptr<storage::client::StorageClient> storageClient)
      : config_(config),
        mgmtdClient_(std::move(mgmtdClient)),
        storageClient_(std::move(storageClient)) {}

  ~FileHelper() { stopAndJoin(); }

  void start(CPUExecutorGroup &exec);
  void stopAndJoin();

  CoTryTask<uint64_t> queryLength(const UserInfo &userInfo, const Inode &inode, bool *hasHole = nullptr);

  CoTryTask<size_t> remove(const UserInfo &userInfo,
                           const Inode &inode,
                           RetryOptions retry,
                           uint32_t removeChunksBatchSize);

  CoTryTask<FsStatus> statFs(const UserInfo &userInfo, std::chrono::milliseconds cacheDuration);

  std::optional<FsStatus> cachedFsStatus() const { return cachedFsStatus_.rlock()->status_; }

 private:
  CoTryTask<void> updateStatFs();

  const Config &config_;
  std::shared_ptr<client::ICommonMgmtdClient> mgmtdClient_;
  std::shared_ptr<storage::client::StorageClient> storageClient_;

  struct CachedFsStatus {
    RelativeTime update_;
    std::optional<FsStatus> status_;
  };

  std::unique_ptr<BackgroundRunner> bgRunner_;
  folly::Synchronized<CachedFsStatus> cachedFsStatus_;
};

}  // namespace hf3fs::meta::server

#ifdef ENABLE_FUSE_APPLICATION

#include "FuseApplication.h"

int main(int argc, char *argv[]) {
  gflags::AllowCommandLineReparsing();
  using namespace hf3fs;
  return fuse::FuseApplication().run(argc, argv);
}
#else
#include <folly/ScopeGuard.h>

#include "FuseConfig.h"
#include "FuseMainLoop.h"
#include "FuseOps.h"
#include "common/logging/LogInit.h"

using namespace hf3fs;
using namespace hf3fs::fuse;

DECLARE_string(cfg);
DECLARE_bool(use_local_cfg);

auto withRetry(auto &&f, std::string_view desc) {
  using RetType = decltype(f());
  auto retryInterval = std::chrono::milliseconds(10);
  constexpr auto maxRetryInterval = std::chrono::milliseconds(1000);
  std::optional<RetType> res;
  for (int i = 0; i < 20; ++i) {
    res = f();
    if (*res) break;
    XLOGF(CRITICAL, "{} failed: {}\nretryCount: {}", desc, res->error(), i);
    std::this_thread::sleep_for(retryInterval);
    retryInterval = std::min(2 * retryInterval, maxRetryInterval);
  }
  return *res;
}

int main(int argc, char *argv[]) {
  gflags::AllowCommandLineReparsing();
  FuseConfig hf3fsConfig;
  hf3fsConfig.init(&argc, &argv);

  auto ibResult = net::IBManager::start(hf3fsConfig.ib_devices());
  XLOGF_IF(FATAL, !ibResult, "Failed to start IBManager: {}", ibResult.error());
  SCOPE_EXIT { hf3fs::net::IBManager::stop(); };

  auto logConfigStr = logging::generateLogConfig(hf3fsConfig.log(), String("hf3fs_fuse"));
  XLOGF(INFO, "LogConfig: {}", logConfigStr);
  logging::initOrDie(logConfigStr);
  XLOGF(INFO, "{}", VersionInfo::full());

  auto monitorResult = monitor::Monitor::start(hf3fsConfig.monitor());
  XLOGF_IF(FATAL, !monitorResult, "Parse config file from flags failed: {}", monitorResult.error());

  auto physicalHostnameRes = SysResource::hostname(/*physicalMachineName=*/true);
  XLOGF_IF(FATAL, !physicalHostnameRes, "Get physical hostname failed: {}", physicalHostnameRes.error());

  auto containerHostnameRes = SysResource::hostname(/*physicalMachineName=*/false);
  XLOGF_IF(FATAL, !containerHostnameRes, "Get container hostname failed: {}", containerHostnameRes.error());

  auto clientId = ClientId::random(*physicalHostnameRes);

  flat::AppInfo appInfo;
  appInfo.clusterId = hf3fsConfig.cluster_id();
  appInfo.hostname = *physicalHostnameRes;
  appInfo.pid = SysResource::pid();
  appInfo.releaseVersion = flat::ReleaseVersion::fromVersionInfo();

  auto &d = getFuseClientsInstance();
  if (auto res = d.init(appInfo, hf3fsConfig.mountpoint(), hf3fsConfig.token_file(), hf3fsConfig); !res) {
    XLOGF(FATAL, "Init fuse clients failed: {}", res.error());
  }
  SCOPE_EXIT { d.stop(); };

  return fuseMainLoop(argv[0],
                      hf3fsConfig.allow_other(),
                      hf3fsConfig.mountpoint(),
                      hf3fsConfig.io_bufs().max_buf_size(),
                      hf3fsConfig.cluster_id());
}

#endif

#pragma once

#include <atomic>
#include <folly/experimental/coro/Baton.h>

#include "common/utils/Coroutine.h"

namespace hf3fs {
template <typename BatonType = folly::coro::Baton>
class CountDownLatch {
 public:
  explicit CountDownLatch(int64_t count = 0)
      : count_(count) {}

  void countDown(int64_t n = 1) {
    auto remaining = count_.fetch_sub(n, std::memory_order_acq_rel);
    assert(remaining >= n);
    if (remaining == n) {
      baton_.post();
    }
  }

  int64_t increase(int64_t n = 1) { return count_.fetch_add(n, std::memory_order_acq_rel); }

  CoTask<void> wait() {
    if (count_.load(std::memory_order_acquire) > 0) co_await baton_;
  }

  bool tryWait() { return count_.load(std::memory_order_acquire) == 0; }

  void reset() { baton_.reset(); }

 private:
  std::atomic<int64_t> count_;
  BatonType baton_;
};
}  // namespace hf3fs

#pragma once

#include <string>
#include <variant>

#include "common/serde/ClientContext.h"
#include "common/serde/ClientMockContext.h"
#include "fbs/mgmtd/MgmtdTypes.h"

namespace hf3fs::stubs {
using flat::NodeId;

template <typename IStub>
struct StubMockContext {};

template <template <typename Ctx> typename StubImpl, bool SupportsStubStub = false>
class SerdeStub {
 public:
  using RealStub = StubImpl<serde::ClientContext>;
  using MockStub = StubImpl<serde::ClientMockContext>;
  using IStub = typename RealStub::InterfaceType;
  using MockContext = std::conditional_t<SupportsStubStub, stubs::StubMockContext<IStub>, int>;
  using StubStub = std::conditional_t<SupportsStubStub, StubImpl<MockContext>, void>;

  template <typename Ctx>
  SerdeStub(Ctx ctx, NodeId node = NodeId(0), std::string_view host = "mock")
      : nodeId(node),
        hostname(host),
        stub_{std::in_place_type<StubImpl<Ctx>>, std::move(ctx)} {}

  IStub *operator->() { return get(); }

  IStub *get() {
    return std::visit([](auto &v) -> IStub * { return &v; }, stub_);
  }

 public:
  NodeId nodeId{0};
  std::string hostname;

 private:
  std::conditional_t<SupportsStubStub, std::variant<RealStub, MockStub, StubStub>, std::variant<RealStub, MockStub>>
      stub_;
};

}  // namespace hf3fs::stubs

#pragma once

#include <memory>

#include "client/mgmtd/MgmtdClientForServer.h"
#include "client/storage/StorageClient.h"
#include "common/logging/LogConfig.h"
#include "common/net/Client.h"
#include "common/net/Server.h"
#include "common/utils/BackgroundRunner.h"
#include "common/utils/ConfigBase.h"
#include "core/app/ServerAppConfig.h"
#include "core/app/ServerLauncher.h"
#include "core/app/ServerLauncherConfig.h"
#include "core/app/ServerMgmtdClientFetcher.h"
#include "fdb/HybridKvEngineConfig.h"
#include "meta/base/Config.h"
#include "meta/service/MetaOperator.h"

namespace hf3fs::meta::server {

class MetaServer : public net::Server {
 public:
  static constexpr auto kName = "Meta";
  static constexpr auto kNodeType = flat::NodeType::META;

  struct CommonConfig : public ApplicationBase::Config {
    CommonConfig() {
      using logging::LogConfig;
      log().set_categories({LogConfig::makeRootCategoryConfig(), LogConfig::makeEventCategoryConfig()});
      log().set_handlers({LogConfig::makeNormalHandlerConfig(),
                          LogConfig::makeErrHandlerConfig(),
                          LogConfig::makeFatalHandlerConfig(),
                          LogConfig::makeEventHandlerConfig()});
    }
  };

  using AppConfig = core::ServerAppConfig;
  struct LauncherConfig : public core::ServerLauncherConfig {
    LauncherConfig() { mgmtd_client() = hf3fs::client::MgmtdClientForServer::Config{}; }
  };
  using RemoteConfigFetcher = core::launcher::ServerMgmtdClientFetcher;
  using Launcher = core::ServerLauncher<MetaServer>;

  struct Config : public ConfigBase<Config> {
    CONFIG_ITEM(use_memkv, false);  // deprecated

    CONFIG_OBJ(base, net::Server::Config, [](net::Server::Config &c) {
      c.set_groups_length(2);
      c.groups(0).listener().set_listen_port(8000);
      c.groups(0).set_services({"MetaSerde"});

      c.groups(1).set_network_type(net::Address::TCP);
      c.groups(1).listener().set_listen_port(9000);
      c.groups(1).set_use_independent_thread_pool(true);
      c.groups(1).set_services({"Core"});
    });
    CONFIG_OBJ(fdb, kv::fdb::FDBConfig);  // deprecated
    CONFIG_OBJ(meta, meta::server::Config);
    CONFIG_OBJ(background_client, net::Client::Config);
    CONFIG_OBJ(mgmtd_client, ::hf3fs::client::MgmtdClientForServer::Config);
    CONFIG_OBJ(storage_client, storage::client::StorageClient::Config, [](storage::client::StorageClient::Config &cfg) {
      cfg.retry().set_init_wait_time(2_s);
      cfg.retry().set_max_wait_time(5_s);
      cfg.retry().set_max_retry_time(5_s);
      cfg.retry().set_max_failures_before_failover(1);
    });
    CONFIG_OBJ(kv_engine, kv::HybridKvEngineConfig);
  };

  MetaServer(const Config &config);
  ~MetaServer() override;

  using net::Server::start;
  Result<Void> start(const flat::AppInfo &info, std::shared_ptr<kv::IKVEngine> kvEngine);
  Result<Void> start(const flat::AppInfo &info,
                     std::unique_ptr<net::Client> client,
                     std::shared_ptr<::hf3fs::client::MgmtdClient> mgmtdClient);

  // set up meta server.
  Result<Void> beforeStart() final;

  // tear down meta server.
  Result<Void> beforeStop() final;
  Result<Void> afterStop() final;

 private:
  const Config &config_;

  std::shared_ptr<kv::IKVEngine> kvEngine_;
  std::unique_ptr<net::Client> backgroundClient_;
  std::shared_ptr<::hf3fs::client::MgmtdClientForServer> mgmtdClient_;
  std::unique_ptr<MetaOperator> metaOperator_;
};

}  // namespace hf3fs::meta::server

#pragma once

#include "LauncherUtils.h"
#include "common/app/ApplicationBase.h"
#include "common/utils/ConstructLog.h"
#include "fbs/mgmtd/ConfigInfo.h"

DECLARE_string(app_cfg);
DECLARE_bool(dump_default_app_cfg);

DECLARE_string(launcher_cfg);
DECLARE_bool(dump_default_launcher_cfg);

namespace hf3fs::core {
class ServerLauncherBase {
 public:
  Result<Void> parseFlags(int *argc, char ***argv);

 protected:
  ApplicationBase::ConfigFlags appConfigFlags_;
  ApplicationBase::ConfigFlags launcherConfigFlags_;
};

template <typename Server>
class ServerLauncher : public ServerLauncherBase {
 public:
  using AppConfig = typename Server::AppConfig;
  using LauncherConfig = typename Server::LauncherConfig;
  using RemoteConfigFetcher = typename Server::RemoteConfigFetcher;
  static constexpr auto kNodeType = Server::kNodeType;

  ServerLauncher() = default;

  Result<Void> init() {
    appConfig_.init(FLAGS_app_cfg, FLAGS_dump_default_app_cfg, appConfigFlags_);
    launcherConfig_.init(FLAGS_launcher_cfg, FLAGS_dump_default_launcher_cfg, launcherConfigFlags_);

    XLOGF(INFO, "Full AppConfig:\n{}", appConfig_.toString());
    XLOGF(INFO, "Full LauncherConfig:\n{}", launcherConfig_.toString());

    auto ibResult = net::IBManager::start(launcherConfig_.ib_devices());
    XLOGF_IF(FATAL, !ibResult, "Failed to start IBManager: {}", ibResult.error());
    XLOGF(INFO, "IBDevice inited");

    fetcher_ = std::make_unique<RemoteConfigFetcher>(launcherConfig_);
    return Void{};
  }

  Result<std::pair<String, String>> loadConfigTemplate() {
    auto res = fetcher_->loadConfigTemplate(kNodeType);
    RETURN_ON_ERROR(res);
    return std::make_pair(res->content, res->genUpdateDesc());
  }

  Result<flat::AppInfo> loadAppInfo() {
    auto appInfo = launcher::buildBasicAppInfo(appConfig_.getNodeId(), launcherConfig_.cluster_id());
    RETURN_ON_ERROR(fetcher_->completeAppInfo(appInfo));
    return appInfo;
  }

  Result<Void> startServer(Server &server, const flat::AppInfo &appInfo) {
    if constexpr (requires { fetcher_->startServer(server, appInfo); }) {
      return fetcher_->startServer(server, appInfo);
    } else {
      return server.start(appInfo);
    }
  }

  const auto &appConfig() const { return appConfig_; }

  const auto &launcherConfig() const { return launcherConfig_; }

 private:
  AppConfig appConfig_;
  LauncherConfig launcherConfig_;

  std::unique_ptr<RemoteConfigFetcher> fetcher_;
};
}  // namespace hf3fs::core

#pragma once

#include <cstdint>
#include <optional>
#include <string_view>

#include "common/kv/ITransaction.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Status.h"
#include "fdb/FDB.h"

namespace hf3fs::kv {

class FDBTransaction : public IReadWriteTransaction {
 public:
  FDBTransaction(fdb::Transaction &&tr)
      : tr_(std::move(tr)),
        errcode_(0) {}

  // Read operations
  CoTryTask<std::optional<String>> snapshotGet(std::string_view key) override;
  CoTryTask<GetRangeResult> snapshotGetRange(const KeySelector &begin, const KeySelector &end, int32_t limit) override;
  CoTryTask<std::optional<String>> get(std::string_view key) override;
  CoTryTask<GetRangeResult> getRange(const KeySelector &begin, const KeySelector &end, int32_t limit) override;

  CoTryTask<void> cancel() override;

  CoTryTask<void> addReadConflict(std::string_view key) override;
  CoTryTask<void> addReadConflictRange(std::string_view begin, std::string_view end) override;

  // Write operations
  CoTryTask<void> set(std::string_view key, std::string_view value) override;
  CoTryTask<void> clear(std::string_view key) override;

  CoTryTask<void> setVersionstampedKey(std::string_view key, uint32_t offset, std::string_view value) override;
  CoTryTask<void> setVersionstampedValue(std::string_view key, std::string_view value, uint32_t offset) override;

  CoTryTask<void> clearRange(std::string_view begin, std::string_view end);  // only used in test.

  CoTryTask<void> commit() override;
  void reset() override;

  Result<Void> setOption(FDBTransactionOption option, std::string_view value = {});
  CoTask<bool> onError(fdb_error_t errcode);

  CoTryTask<int64_t> getReadVersion();
  int64_t getCommittedVersion() override;
  void setReadVersion(int64_t version) override;

  fdb_error_t errcode() const { return errcode_; }

 private:
  friend class TestFDBTransaction;
  static Status testFDBError(int errCode, bool commit);

  fdb::Transaction tr_;
  std::atomic<fdb_error_t> errcode_;
};

}  // namespace hf3fs::kv

#pragma once

#include <folly/concurrency/AtomicSharedPtr.h>
#include <mutex>
#include <set>
#include <vector>

namespace hf3fs {
struct AvailSlots {
  AvailSlots(int c)
      : cap(c) {}
  std::optional<int> alloc() {
    std::lock_guard lock(mutex);
    if (!free.empty()) {
      auto idx = *free.begin();
      free.erase(free.begin());
      return idx;
    }

    auto idx = nextAvail.load();
    if (idx < cap) {
      ++nextAvail;
      return idx;
    } else {
      return std::nullopt;
    }
  }
  void dealloc(int idx) {
    if (idx < 0 || idx >= cap) {
      return;
    }

    std::lock_guard lock(mutex);
    if (idx == nextAvail - 1) {
      while (free.find(--nextAvail) != free.end()) {
        // move back next avail as much as possible
        ;
      }
    } else {
      free.insert(idx);
    }
  }

  int cap;
  mutable std::mutex mutex;
  std::atomic<int> nextAvail{0};
  std::set<int> free;
};

template <typename T>
struct AtomicSharedPtrTable {
  AtomicSharedPtrTable(int cap)
      : slots(cap),
        table(cap) {}

  std::optional<int> alloc() { return slots.alloc(); }
  void dealloc(int idx) { slots.dealloc(idx); }
  void remove(int idx) {
    if (idx < 0 || idx >= (int)table.size()) {
      return;
    }

    auto &ap = table[idx];
    if (!ap.load()) {
      return;
    }

    ap.store(std::shared_ptr<T>());
    dealloc(idx);
  }

  AvailSlots slots;
  std::vector<folly::atomic_shared_ptr<T>> table;
};
};  // namespace hf3fs

#pragma once

#include "stubs/common/IStubFactory.h"
#include "stubs/core/ICoreServiceStub.h"

namespace hf3fs::client {
class CoreClient {
 public:
  using CoreStub = core::ICoreServiceStub;
  using CoreStubFactory = stubs::IStubFactory<CoreStub>;

  explicit CoreClient(std::unique_ptr<CoreStubFactory> stubFactory)
      : stubFactory_(std::move(stubFactory)) {}
  ~CoreClient() = default;

#define DEFINE_SERDE_SERVICE_METHOD_FULL(svc, name, Name, id, reqtype, rsptype)                                      \
  CoTryTask<core::rsptype> name(net::Address addr, const core::reqtype &req) {                                       \
    auto stub = stubFactory_->create(addr);                                                                          \
    co_return co_await stub->name(req);                                                                              \
  }                                                                                \
                                                                                \
  CoTryTask<core::rsptype> name(const std::vector<net::Address> &addrs, const core::reqtype &req) {                  \
    for (auto addr : addrs) {                                                                                \
      auto stub = stubFactory_->create(addr);                                                                        \
      auto res = co_await stub->name(req);                                                                           \
      if (res || StatusCode::typeOf(res.error().code()) != StatusCodeType::RPC) co_return res;                       \
    }                                                                                \
    co_return makeError(RPCCode::kConnectFailed,                                                                     \
                        fmt::format("Try to call CoreService::{} failed, addrs: {}", #name, fmt::join(addrs, ","))); \
  }

#include "fbs/core/service/CoreServiceDef.h"

 private:
  std::unique_ptr<CoreStubFactory> stubFactory_;
};  // namespace hf3fs::client
}  // namespace hf3fs::client

#pragma once

#include <boost/core/ignore_unused.hpp>
#include <mutex>
#include <stack>

#include "fbs/storage/Common.h"

namespace hf3fs::storage::client {

/*
  Allocate a channel id, which is used to de-duplicate updates at storage service,
  for example, write, truncate, remove etc.
*/
class UpdateChannelAllocator {
 public:
  static constexpr size_t kChannelIdNumBits = sizeof(ChannelId::UnderlyingType) * 8ULL;
  static constexpr size_t kMaxNumChannels = (1ULL << kChannelIdNumBits) - 1ULL;

 public:
  UpdateChannelAllocator(size_t numChannels);

  ~UpdateChannelAllocator();

  bool allocate(UpdateChannel &channel, uint32_t slots = 1);

  void release(UpdateChannel &channel);

 private:
  size_t numChannels_;
  std::mutex availableChannelMutex_;
  std::stack<ChannelId> availableChannelIds_;
  std::atomic_uint64_t nextSeqNum_ = 1;
};

}  // namespace hf3fs::storage::client

#pragma once

#include <folly/Random.h>
#include <mutex>
#include <vector>

#include "common/utils/StatusCodeDetails.h"
#include "fbs/storage/Common.h"

namespace hf3fs::storage::client {

/* Slim routing info */

struct SlimTargetInfo {
  TargetId targetId;
  NodeId nodeId;

  bool operator==(const SlimTargetInfo &other) const = default;
};

struct SlimChainInfo {
  ChainId chainId;
  ChainVer version;
  flat::RoutingInfoVersion routingInfoVer;
  size_t totalNumTargets;  // total number of targets including non-serving targets
  std::vector<SlimTargetInfo> servingTargets;
};

enum TargetSelectionMode {
  Default = 0,
  LoadBalance,
  RoundRobin,
  RandomTarget,
  TailTarget,
  HeadTarget,
  ManualMode,
  EndOfMode
};

class TargetSelectionOptions : public hf3fs::ConfigBase<TargetSelectionOptions> {
  // if mode = Tail/Head, but tail/head is not in the specified traffic zone, the read could fail
  // if mode = LB/RR/Random, only storage targets hosted in the specified traffic zone are selected
  CONFIG_HOT_UPDATED_ITEM(mode, TargetSelectionMode::Default);
  CONFIG_HOT_UPDATED_ITEM(targetIndex, 0U);  // the target chosen by user in ManualMode
  CONFIG_HOT_UPDATED_ITEM(trafficZone, "");
};

// An instance of each TargetSelectionStrategy implementation is created for each batch read.
class TargetSelectionStrategy {
 public:
  TargetSelectionStrategy(const TargetSelectionOptions &options)
      : options_(options) {}

  virtual ~TargetSelectionStrategy() = default;

  // selectTarget() is called for each read IO in the batch.
  virtual Result<SlimTargetInfo> selectTarget(const SlimChainInfo &chain) = 0;

  virtual bool selectAnyTarget() { return false; }

  // reset the internal states
  virtual void reset() {}

  static std::unique_ptr<TargetSelectionStrategy> create(const TargetSelectionOptions &options);

 protected:
  static constexpr size_t kNodeIdKeyedMapExpectedNumElements = 1000;
  TargetSelectionOptions options_;
};

}  // namespace hf3fs::storage::client

#pragma once

#include "common/net/Transport.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Duration.h"
#include "common/utils/LockManager.h"
#include "common/utils/RobinHood.h"
#include "common/utils/Shards.h"
#include "common/utils/Size.h"
#include "fbs/storage/Common.h"
#include "storage/service/TargetMap.h"

namespace hf3fs::storage {

struct Components;
class StorageOperator;

class ReliableUpdate {
 public:
  struct Config : ConfigBase<Config> {
    CONFIG_HOT_UPDATED_ITEM(clean_up_expired_clients, false);
    CONFIG_HOT_UPDATED_ITEM(expired_clients_timeout, 1_h);
  };
  ReliableUpdate(const Config &config, Components &components)
      : config_(config),
        components_(components) {}

  CoTask<IOResult> update(ServiceRequestContext &requestCtx,
                          UpdateReq &req,
                          net::IBSocket *ibSocket,
                          TargetPtr &target);

  Result<Void> cleanUpExpiredClients(const robin_hood::unordered_set<std::string> &activeClients);

  void beforeStop() { stopped_ = true; }

 private:
  ConstructLog<"storage::ReliableUpdate"> constructLog_;
  const Config &config_;
  Components &components_;
  std::atomic<bool> stopped_ = false;
  folly::coro::Mutex mutex_;

  struct ReqResult {
    SERDE_STRUCT_FIELD(channelSeqnum, ChannelSeqNum{0});
    SERDE_STRUCT_FIELD(requestId, RequestId{0});
    SERDE_STRUCT_FIELD(updateResult, IOResult{});
    SERDE_STRUCT_FIELD(succUpdateVer, ChunkVer{});
    SERDE_STRUCT_FIELD(generationId, uint32_t{});
  };

  struct ClientStatus {
    std::unordered_map<std::pair<ChainId, ChannelId>, ReqResult> channelMap;
    UtcTime lastUsedTime;
  };
  using ClientMap = std::unordered_map<ClientId, std::shared_ptr<ClientStatus>>;
  Shards<ClientMap, 1024> shards_;
};

}  // namespace hf3fs::storage

#pragma once

#include <variant>

#include "common/utils/Nameof.hpp"
#include "common/utils/TypeTraits.h"

namespace hf3fs::serde {

template <class T>
inline constexpr std::string_view type_name_v = [] {
  if constexpr (requires { std::string_view{T::kTypeNameForSerde}; }) {
    return std::string_view{T::kTypeNameForSerde};
  } else {
    return nameof::nameof_short_type<T>();
  }
}();

template <class T>
inline constexpr auto variant_type_names_v = std::array<std::string_view, 0>{};
template <class... Ts>
inline constexpr auto variant_type_names_v<std::variant<Ts...>> = std::array{type_name_v<Ts>...};

template <class T>
inline uint8_t variantTypeNameToIndex(std::string_view in) {
  uint8_t i = 0;
  for (auto &name : variant_type_names_v<T>) {
    if (name == in) {
      break;
    }
    ++i;
  }
  return i;
}

}  // namespace hf3fs::serde

#pragma once

#include <span>
#include <vector>

#include "common/net/Client.h"
#include "common/serde/MessagePacket.h"
#include "common/utils/Address.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Result.h"
#include "fbs/storage/Common.h"

namespace hf3fs::storage::client {

class StorageMessenger {
 public:
  StorageMessenger(const hf3fs::net::Client::Config &config);

  Result<Void> start(const std::string &name = "StorageMsgr") { return client_.start(name); }

  void stopAndJoin() { client_.stopAndJoin(); }

  CoTryTask<BatchReadRsp> batchRead(const hf3fs::net::Address &address,
                                    const BatchReadReq &request,
                                    const net::UserRequestOptions *options = nullptr,
                                    serde::Timestamp *timestamp = nullptr);

  CoTryTask<WriteRsp> write(const hf3fs::net::Address &address,
                            const WriteReq &request,
                            const net::UserRequestOptions *options = nullptr,
                            serde::Timestamp *timestamp = nullptr);

  CoTryTask<UpdateRsp> update(const hf3fs::net::Address &address,
                              const UpdateReq &request,
                              const net::UserRequestOptions *options = nullptr,
                              serde::Timestamp *timestamp = nullptr);

  CoTryTask<QueryLastChunkRsp> queryLastChunk(const hf3fs::net::Address &address,
                                              const QueryLastChunkReq &request,
                                              const net::UserRequestOptions *options = nullptr,
                                              serde::Timestamp *timestamp = nullptr);

  CoTryTask<RemoveChunksRsp> removeChunks(const hf3fs::net::Address &address,
                                          const RemoveChunksReq &request,
                                          const net::UserRequestOptions *options = nullptr,
                                          serde::Timestamp *timestamp = nullptr);

  CoTryTask<TruncateChunksRsp> truncateChunks(const hf3fs::net::Address &address,
                                              const TruncateChunksReq &request,
                                              const net::UserRequestOptions *options = nullptr,
                                              serde::Timestamp *timestamp = nullptr);

  CoTryTask<SpaceInfoRsp> querySpaceInfo(const hf3fs::net::Address &address,
                                         const SpaceInfoReq &request,
                                         const net::UserRequestOptions *options = nullptr,
                                         serde::Timestamp *timestamp = nullptr);

  CoTryTask<TargetSyncInfo> syncStart(const hf3fs::net::Address &address,
                                      const SyncStartReq &request,
                                      const net::UserRequestOptions *options = nullptr,
                                      serde::Timestamp *timestamp = nullptr);

  CoTryTask<SyncDoneRsp> syncDone(const hf3fs::net::Address &address,
                                  const SyncDoneReq &request,
                                  const net::UserRequestOptions *options = nullptr,
                                  serde::Timestamp *timestamp = nullptr);

  CoTryTask<CreateTargetRsp> createTarget(const hf3fs::net::Address &address,
                                          const CreateTargetReq &request,
                                          const net::UserRequestOptions *options = nullptr,
                                          serde::Timestamp *timestamp = nullptr);

  CoTryTask<OfflineTargetRsp> offlineTarget(const hf3fs::net::Address &address,
                                            const OfflineTargetReq &request,
                                            const net::UserRequestOptions *options = nullptr,
                                            serde::Timestamp *timestamp = nullptr);

  CoTryTask<RemoveTargetRsp> removeTarget(const hf3fs::net::Address &address,
                                          const RemoveTargetReq &request,
                                          const net::UserRequestOptions *options = nullptr,
                                          serde::Timestamp *timestamp = nullptr);

  CoTryTask<QueryChunkRsp> queryChunk(const hf3fs::net::Address &address,
                                      const QueryChunkReq &request,
                                      const net::UserRequestOptions *options = nullptr,
                                      serde::Timestamp *timestamp = nullptr);

  CoTryTask<GetAllChunkMetadataRsp> getAllChunkMetadata(const hf3fs::net::Address &address,
                                                        const GetAllChunkMetadataReq &request,
                                                        const net::UserRequestOptions *options = nullptr,
                                                        serde::Timestamp *timestamp = nullptr);

 private:
  hf3fs::net::Client client_;
};

}  // namespace hf3fs::storage::client

#pragma once

#include <atomic>
#include <bit>
#include <cstddef>
#include <cstdint>
#include <folly/Hash.h>
#include <folly/lang/Bits.h>
#include <string>

#include "common/serde/BigEndian.h"
#include "common/serde/Serde.h"
#include "common/utils/Int128.h"
#include "common/utils/Result.h"
#include "common/utils/Size.h"
#include "common/utils/StrongType.h"
#include "fbs/storage/Common.h"
#include "storage/store/ChunkFileView.h"

namespace hf3fs::storage {

inline constexpr auto kMaxChunkSize = 64_MB;

struct ChunkInfo {
  ChunkMetadata meta;
  ChunkFileView view;
};

struct ChunkPosition {
  SERDE_STRUCT_FIELD(fileIdx, uint32_t{});
  SERDE_STRUCT_FIELD(offset, serde::BigEndian<std::size_t>{});
};

void reportFatalEvent();

}  // namespace hf3fs::storage

template <>
struct ::std::hash<hf3fs::storage::ChunkFileId> {
  size_t operator()(hf3fs::storage::ChunkFileId id) const {
    return folly::hash::twang_mix64(reinterpret_cast<uint64_t &>(id));
  }
};

#pragma once

#include <atomic>
#include <common/utils/RobinHood.h>
#include <folly/concurrency/AtomicSharedPtr.h>
#include <memory>

#include "client/mgmtd/RoutingInfo.h"
#include "common/serde/Serde.h"
#include "common/utils/ConstructLog.h"
#include "fbs/mgmtd/MgmtdTypes.h"
#include "fbs/mgmtd/NodeInfo.h"
#include "fbs/mgmtd/TargetInfo.h"
#include "fbs/storage/Common.h"
#include "storage/store/StorageTarget.h"

namespace hf3fs::test {
struct TargetMapHelper;
}

namespace hf3fs::storage {

class TargetMap {
 public:
  // [observers] clone current map.
  std::shared_ptr<TargetMap> clone() const { return std::make_shared<TargetMap>(*this); }

  // [observers] get target id by chain id.
  Result<TargetId> getTargetId(ChainId chainId) const;

  // [observers] get target by target id.
  Result<const Target *> getTarget(TargetId targetId) const;

  // [observers] get target by versioned chain id.
  Result<const Target *> getByChainId(VersionedChainId vChainId, bool allowOutdatedChainVer) const;

  // [observers]
  auto &getTargets() const { return targets_; }

  // [observers]
  auto &syncingChains() const { return syncingChains_; }

  // [modifiers] add a new target.
  Result<Void> addStorageTarget(const std::shared_ptr<StorageTarget> &storageTarget);

  // [modifiers] get target by target id.
  Result<Target *> getMutableTarget(TargetId targetId);

  // [modifiers] sync receive is started.
  Result<Void> syncReceiveDone(VersionedChainId vChainId);

  // [modifiers] update by routing info.
  Result<Void> updateRouting(std::shared_ptr<hf3fs::client::RoutingInfo> r, bool log = true);

  // [modifiers] set target as offline.
  Result<Void> removeTarget(TargetId targetId);

  // [modifiers] set target as offline.
  Result<Void> offlineTarget(TargetId targetId);

  // [modifiers] set targets in path as offline.
  Result<Void> offlineTargets(const Path &path);

  // [modifiers] reject create chunk for targets in path.
  Result<Void> updateDiskState(const Path &path, bool lowSpace, bool rejectCreateChunk);

  // update local state.
  static hf3fs::flat::LocalTargetState updateLocalState(TargetId targetId,
                                                        hf3fs::flat::LocalTargetState localState,
                                                        hf3fs::flat::PublicTargetState publicState);

 private:
  friend struct test::TargetMapHelper;
  robin_hood::unordered_map<TargetId, Target> targets_;
  flat::RoutingInfoVersion routingInfoVersion_;
  robin_hood::unordered_map<ChainId, TargetId> chainToTarget_;
  std::vector<VersionedChainId> syncingChains_;
};

class AtomicallyTargetMap {
 public:
  // [observers] get a snapshot of target map.
  auto snapshot() const { return targetMap_.load(); }

  // [observers] get target by chain id.
  Result<std::shared_ptr<const Target>> getByChainId(VersionedChainId vChainId,
                                                     bool allowOutdatedChainVer = false) const;

  // [observers] get target by its id.
  Result<std::shared_ptr<const Target>> getByTargetId(TargetId targetId) const;

  // [modifiers] set update callback.
  void setUpdateCallback(auto &&func) {
    auto lock = std::unique_lock(mutex_);
    updateCallback_ = std::forward<decltype(func)>(func);
  }

  // [modifiers] add a target.
  Result<Void> addStorageTarget(std::shared_ptr<StorageTarget> storageTarget);

  // [modifiers] sync receive is done.
  Result<Void> syncReceiveDone(VersionedChainId vChainId);

  // [modifiers] update by routing info.
  Result<Void> updateRouting(std::shared_ptr<hf3fs::client::RoutingInfo> r);

  // [modifiers] set target as offline.
  Result<Void> removeTarget(TargetId targetId);

  // [modifiers] set target as offline.
  Result<Void> offlineTarget(TargetId targetId);

  // [modifiers] set targets in path as offline.
  Result<Void> offlineTargets(const Path &path);

  // [modifiers] reject create chunk for targets in path.
  Result<Void> updateDiskState(const Path &path, bool lowSpace, bool rejectCreateChunk);

  // [modifiers] update target used size.
  void updateTargetUsedSize();

  // [modifiers] release target map.
  auto release() { return targetMap_.exchange(nullptr); }

 protected:
  // [modifiers] update target map atomically.
  Result<Void> updateTargetMap(auto &&updateFunc);

 private:
  friend struct test::TargetMapHelper;
  ConstructLog<"storage::AtomicallyTargetMap"> constructLog_;
  std::mutex mutex_;  // for update operation.
  std::function<void(const TargetMap &)> updateCallback_ = [](auto) {};
  folly::atomic_shared_ptr<const TargetMap> targetMap_{std::make_shared<const TargetMap>()};
};

}  // namespace hf3fs::storage

#pragma once

#include "LocalTargetInfoWithNodeId.h"
#include "fbs/mgmtd/ChainTargetInfo.h"
#include "fbs/mgmtd/TargetInfo.h"

namespace hf3fs::mgmtd {
struct ChainTargetInfoEx : public flat::ChainTargetInfo {
  ChainTargetInfoEx() = default;
  ChainTargetInfoEx(const flat::ChainTargetInfo &cti, flat::LocalTargetState ls)
      : flat::ChainTargetInfo(cti),
        localState(ls) {}
  explicit ChainTargetInfoEx(const flat::TargetInfo &ti) {
    targetId = ti.targetId;
    publicState = ti.publicState;
    localState = ti.localState;
  }

  bool operator==(const ChainTargetInfo &other) const { return static_cast<const ChainTargetInfo &>(*this) == other; }
  bool operator==(const ChainTargetInfoEx &other) const {
    return *this == static_cast<const ChainTargetInfo &>(other) && localState == other.localState;
  }

  static ChainTargetInfoEx fromTargetInfo(const flat::TargetInfo &ti) { return ChainTargetInfoEx(ti); }

  static flat::TargetInfo toTargetInfo(const ChainTargetInfoEx &cti) {
    flat::TargetInfo ti;
    ti.targetId = cti.targetId;
    ti.publicState = cti.publicState;
    ti.localState = cti.localState;
    return ti;
  }

  flat::LocalTargetState localState{flat::LocalTargetState::INVALID};
};

// separate this function just for testing friendly
std::vector<ChainTargetInfoEx> generateNewChain(const std::vector<ChainTargetInfoEx> &oldTargets);

std::vector<ChainTargetInfoEx> rotateAsPreferredOrder(const std::vector<ChainTargetInfoEx> &oldTargets,
                                                      const std::vector<flat::TargetId> &preferredOrder);

// If the head of oldTargets is LASTSRV, move it to the tail and let the next target be the new LASTSRV.
// It's used when a LASTSRV target could not recover in a short time. Admin could let the next target be the new LASTSRV
// to resume the service. NOTE: it's on risk of losing some data forever.
std::vector<ChainTargetInfoEx> rotateLastSrv(const std::vector<ChainTargetInfoEx> &oldTargets);

std::vector<flat::ChainTargetInfo> shutdownChain(const std::vector<flat::ChainTargetInfo> &oldTargets);
}  // namespace hf3fs::mgmtd

#pragma once

#include <memory>

#include "client/mgmtd/MgmtdClientForServer.h"
#include "client/storage/StorageClient.h"
#include "common/logging/LogConfig.h"
#include "common/net/Client.h"
#include "common/net/Server.h"
#include "common/utils/BackgroundRunner.h"
#include "common/utils/ConfigBase.h"
#include "core/app/ServerAppConfig.h"
#include "core/app/ServerLauncher.h"
#include "core/app/ServerLauncherConfig.h"
#include "core/app/ServerMgmtdClientFetcher.h"

namespace hf3fs::simple_example::server {

class SimpleExampleServer : public net::Server {
 public:
  static constexpr auto kName = "SimpleExample";
  static constexpr auto kNodeType = flat::NodeType::CLIENT;

  struct CommonConfig : public ApplicationBase::Config {
    CommonConfig() {
      using logging::LogConfig;
      log().set_categories({LogConfig::makeRootCategoryConfig(), LogConfig::makeEventCategoryConfig()});
      log().set_handlers({LogConfig::makeNormalHandlerConfig(),
                          LogConfig::makeErrHandlerConfig(),
                          LogConfig::makeFatalHandlerConfig(),
                          LogConfig::makeEventHandlerConfig()});
    }
  };

  using AppConfig = core::ServerAppConfig;
  struct LauncherConfig : public core::ServerLauncherConfig {
    LauncherConfig() { mgmtd_client() = hf3fs::client::MgmtdClientForServer::Config{}; }
  };
  using RemoteConfigFetcher = core::launcher::ServerMgmtdClientFetcher;
  using Launcher = core::ServerLauncher<SimpleExampleServer>;

  struct Config : public ConfigBase<Config> {
    CONFIG_OBJ(base, net::Server::Config, [](net::Server::Config &c) {
      c.set_groups_length(2);
      c.groups(0).listener().set_listen_port(8000);
      c.groups(0).set_services({"SimpleExampleSerde"});

      c.groups(1).set_network_type(net::Address::TCP);
      c.groups(1).listener().set_listen_port(9000);
      c.groups(1).set_use_independent_thread_pool(true);
      c.groups(1).set_services({"Core"});
    });
    CONFIG_OBJ(background_client, net::Client::Config);
    CONFIG_OBJ(mgmtd_client, ::hf3fs::client::MgmtdClientForServer::Config);
    CONFIG_OBJ(storage_client, storage::client::StorageClient::Config);
  };

  SimpleExampleServer(const Config &config);
  ~SimpleExampleServer() override;

  Result<Void> beforeStart() final;

  Result<Void> beforeStop() final;

 private:
  const Config &config_;

  std::unique_ptr<net::Client> backgroundClient_;
  std::shared_ptr<::hf3fs::client::MgmtdClientForServer> mgmtdClient_;
};

}  // namespace hf3fs::simple_example::server

#pragma once

#include <folly/Utility.h>
#include <folly/logging/xlog.h>
#include <memory.h>
#include <memory>
#include <type_traits>
#include <utility>

#include "common/kv/ITransaction.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Result.h"

namespace hf3fs::kv {

template <typename RetryStrategy>
class WithTransaction : public folly::MoveOnly {
 public:
  WithTransaction(RetryStrategy strategy)
      : strategy_(std::move(strategy)) {}

  template <typename Handler>
  std::invoke_result_t<Handler, IReadOnlyTransaction &> run(std::unique_ptr<IReadOnlyTransaction> txn,
                                                            Handler &&handler) {
    co_return co_await run(*txn, std::forward<Handler>(handler));
  }

  template <typename Handler>
  std::invoke_result_t<Handler, IReadWriteTransaction &> run(std::unique_ptr<IReadWriteTransaction> txn,
                                                             Handler &&handler) {
    co_return co_await run(*txn, std::forward<Handler>(handler));
  }

  template <typename Handler>
  std::invoke_result_t<Handler, IReadOnlyTransaction &> run(IReadOnlyTransaction &txn, Handler &&handler) {
    auto result = strategy_.init(&txn);
    CO_RETURN_ON_ERROR(result);

    while (true) {
      auto result = co_await handler(txn);
      if (!result.hasError()) {
        co_return std::move(result);
      }
      auto retryResult = co_await strategy_.onError(&txn, std::move(result.error()));
      CO_RETURN_ON_ERROR(retryResult);
    }
  }

  template <typename Handler>
  std::invoke_result_t<Handler, IReadWriteTransaction &> run(IReadWriteTransaction &txn, Handler &&handler) {
    auto result = strategy_.init(&txn);
    CO_RETURN_ON_ERROR(result);

    while (true) {
      auto result = co_await runAndCommit(txn, std::forward<Handler>(handler));
      if (!result.hasError()) {
        co_return std::move(result);
      }
      auto retryResult = co_await strategy_.onError(&txn, std::move(result.error()));
      CO_RETURN_ON_ERROR(retryResult);
    }
  }

  RetryStrategy &getStrategy() { return strategy_; }

 private:
  template <typename Handler>
  std::invoke_result_t<Handler, IReadWriteTransaction &> runAndCommit(IReadWriteTransaction &txn, Handler &&handler) {
    auto result = co_await handler(txn);
    if (!result.hasError()) {
      auto commitResult = co_await txn.commit();
      CO_RETURN_ON_ERROR(commitResult);
    }
    co_return std::move(result);
  }

  RetryStrategy strategy_;
};

}  // namespace hf3fs::kv

#pragma once

#include <cstdint>
#include <string_view>

namespace hf3fs::tracing {
inline constexpr uint64_t kEventPrefixShift = 32;
inline constexpr uint64_t kPairEventShift = kEventPrefixShift - 2;
inline constexpr uint64_t kEventValueMask = (static_cast<uint64_t>(1) << kPairEventShift) - 1;

inline constexpr uint64_t kCommonPrefix = 0;
inline constexpr uint64_t kRpcPrefix = 1;
inline constexpr uint64_t kMetaPrefix = 2;
inline constexpr uint64_t kFdbPrefix = 3;

inline constexpr uint64_t makeValue(uint64_t category, uint64_t value) {
  auto c = (category << kEventPrefixShift);
  auto v = (value & kEventValueMask);
  return c | v;
}

inline constexpr uint64_t makePairValue(uint64_t category, bool first, uint64_t value) {
  auto c = (category << kEventPrefixShift);
  auto p = ((first + 1ULL) << kPairEventShift);
  auto v = (value & kEventValueMask);
  return c | p | v;
}

inline constexpr uint64_t getBeginEvent(uint64_t event) { return event | (2ULL << kPairEventShift); }
inline constexpr uint64_t getEndEvent(uint64_t event) { return event | (1ULL << kPairEventShift); }

#define TRACING_EVENT_NAME(category, name) k##category##name
#define TRACING_BEGIN_EVENT_NAME(category, name) k##category##Begin##name
#define TRACING_END_EVENT_NAME(category, name) k##category##End##name
#define TRACING_EVENT_PREFIX(category) k##category##Prefix

#define EVENT(category, name, value) \
  inline constexpr uint64_t TRACING_EVENT_NAME(category, name) = makeValue(TRACING_EVENT_PREFIX(category), value);

#define PAIR_EVENT(category, name, value)                              \
  EVENT(category, name, value);                                        \
  inline constexpr uint64_t TRACING_BEGIN_EVENT_NAME(category, name) = \
      makePairValue(TRACING_EVENT_PREFIX(category), true, value);      \
  inline constexpr uint64_t TRACING_END_EVENT_NAME(category, name) =   \
      makePairValue(TRACING_EVENT_PREFIX(category), false, value);

#include "TracingEventDetails.h"

#undef EVENT
#undef PAIR_EVENT

std::string_view toString(uint64_t event);
}  // namespace hf3fs::tracing

#pragma once

#include <folly/logging/xlog.h>
#include <optional>
#include <string>
#include <string_view>
#include <variant>
#include <vector>

#include "common/app/ClientId.h"
#include "common/kv/ITransaction.h"
#include "common/serde/Serde.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Result.h"
#include "common/utils/UtcTime.h"
#include "common/utils/Uuid.h"
#include "fbs/meta/Common.h"
#include "fbs/meta/Schema.h"

namespace hf3fs::meta::server {

using kv::IReadOnlyTransaction;
using kv::IReadWriteTransaction;

struct FileSession {
  SERDE_STRUCT_FIELD(inodeId, InodeId());
  SERDE_STRUCT_FIELD(clientId, ClientId::zero());
  SERDE_STRUCT_FIELD(sessionId, Uuid::zero());
  SERDE_STRUCT_FIELD(timestamp, UtcTime());
  SERDE_STRUCT_FIELD(payload, std::string());  // for placeholder

 public:
  static std::string prefix(InodeId inodeId);
  static std::string packKey(InodeId inodeId, Uuid session);
  static Result<std::pair<InodeId, Uuid>> unpackByInodeKey(std::string_view key);

  static FileSession create(InodeId inodeId, SessionInfo session, UtcTime timestamp = UtcClock::now()) {
    return {inodeId, session.client, session.session, timestamp};
  }
  static FileSession create(InodeId inodeId, ClientId clientId, Uuid sessionId, UtcTime timestamp = UtcClock::now()) {
    return {inodeId, clientId, sessionId, timestamp};
  }

  static Result<FileSession> unpack(std::string_view key, std::string_view value);

  static CoTryTask<std::optional<FileSession>> load(IReadOnlyTransaction &txn, InodeId inodeId, Uuid session);
  static CoTryTask<std::vector<FileSession>> list(IReadOnlyTransaction &txn,
                                                  InodeId inodeId,
                                                  bool snapshot,
                                                  size_t limit = 0);

  static CoTryTask<std::optional<FileSession>> snapshotCheckExists(IReadOnlyTransaction &txn, const InodeId inodeId);
  static CoTryTask<std::optional<FileSession>> checkExists(IReadWriteTransaction &txn, const InodeId inodeId);
  static CoTryTask<void> removeAll(IReadWriteTransaction &txn, InodeId inodeId);

  static constexpr size_t kShard = 256;
  static_assert(kShard == (1 << 8));
  static CoTryTask<std::vector<FileSession>> scan(IReadOnlyTransaction &txn,
                                                  size_t shard,
                                                  std::optional<FileSession> prev);

  CoTryTask<void> store(IReadWriteTransaction &txn) const;
  CoTryTask<void> remove(IReadWriteTransaction &txn) const;

  // prune, store FileSessions need to be pruned under special InodeId(-1)
  static FileSession createPrune(ClientId clientId, Uuid sessionId) {
    return FileSession::create(InodeId(-1), clientId, sessionId);
  }

  static CoTryTask<std::vector<FileSession>> listPrune(IReadOnlyTransaction &txn, size_t limit) {
    co_return co_await FileSession::list(txn, InodeId(-1), true, limit);
  }

  // static CoTryTask<std::optional<FileSession>> load(IReadOnlyTransaction &txn, ClientId clientId, Uuid session);
  // static CoTryTask<std::vector<FileSession>> list(IReadOnlyTransaction &txn, Uuid clientId, bool snapshot);
  // static std::string prefix(Uuid clientId);
  // static std::string packKey(Uuid clientId, Uuid session);
  // static Result<std::pair<Uuid, Uuid>> unpackByClientKey(std::string_view key);
};

}  // namespace hf3fs::meta::server

#pragma once
#include <cerrno>
#include <cstddef>
#include <cstdio>
#include <cstdlib>

#include "MemoryAllocatorInterface.h"

namespace hf3fs::memory {

#ifdef OVERRIDE_CXX_NEW_DELETE

void *allocate(size_t size);

void deallocate(void *mem);

void *memalign(size_t alignment, size_t size);

void logstatus(char *buf, size_t size);

bool profiling(bool active, const char *prefix);

void shutdown();

#else

inline void *allocate(size_t size) { return std::malloc(size); }

inline void deallocate(void *mem) { return std::free(mem); }

// A copy of folly::aligned_malloc() from third_party/folly/folly/Memory.h
inline void *memalign(size_t alignment, size_t size) {
  // use posix_memalign, but mimic the behaviour of memalign
  void *ptr = nullptr;
  int rc = posix_memalign(&ptr, alignment, size);
  return rc == 0 ? (errno = 0, ptr) : (errno = rc, nullptr);
}

inline void logstatus(char *buf, size_t size) { std::snprintf(buf, size, "C++ new/delete not overridden"); }

inline bool profiling(bool /*active*/, const char * /*prefix*/) { return false; }

inline void shutdown() {}

#endif

}  // namespace hf3fs::memory

#pragma once

#include <dirent.h>
#include <memory>
#include <optional>
#include <string>
#include <sys/stat.h>
#include <sys/vfs.h>

#include "hf3fs_expected.h"

#define HF3FS_SUPER_MAGIC 0x8f3f5fff  // hf3fs fff

namespace hf3fs::lib {

template <typename T>
using Result = nonstd::expected<T, std::pair<int, std::string>>;

struct Empty {};

using NoResult = Result<Empty>;

struct DIR;
struct dirent {
  unsigned char d_type;
  std::string d_name;
};

struct stat : public ::stat {
  std::optional<std::string> st_ltarg;  // symlink target
};

struct ioseg {
  int fd = -1;
  off_t off = -1;
};
struct iovec_handle_t;
struct iovec {
  // base/len can be different than returned by iovalloc, so long as the whole buffer is within the range
  void *iov_base;
  size_t iov_len;

  // returned by iovalloc
  std::shared_ptr<iovec_handle_t> iov_handle;

  void *user_data = nullptr;
};

class IClient {
 public:
  static Result<std::shared_ptr<IClient>> newClient(std::string_view mountName, std::string_view token);
  static Result<std::shared_ptr<IClient>> newSuperClient(std::string_view mountName, std::string_view token);
  IClient() = default;
  virtual ~IClient() = default;

 public:  // meta ops
  virtual NoResult statfs(std::string_view path, struct statfs *buf) = 0;
  virtual NoResult fstatfs(int fd, struct statfs *buf) = 0;

  virtual Result<std::shared_ptr<DIR>> opendir(std::string_view name, bool ignoreCache = false) = 0;
  virtual Result<std::shared_ptr<DIR>> opendirat(int dirfd, std::string_view name, bool ignoreCache = false) = 0;
  virtual Result<std::shared_ptr<DIR>> fdopendir(int fd) = 0;
  virtual NoResult rewinddir(const std::shared_ptr<DIR> &dirp) = 0;
  virtual Result<std::optional<dirent>> readdir(const std::shared_ptr<DIR> &dirp) = 0;
  virtual Result<int> dirfd(const std::shared_ptr<DIR> &dirp) = 0;

  virtual NoResult mkdir(std::string_view pathname, mode_t mode = 0777, bool recursive = false) = 0;
  virtual NoResult mkdirat(int dirfd, std::string_view pathname, mode_t mode = 0777, bool recursive = false) = 0;

  virtual NoResult rmdir(std::string_view pathname, bool recursive = false) = 0;
  virtual NoResult rmdirat(int dirfd, std::string_view pathname, bool recursive = false) = 0;
  virtual NoResult unlink(std::string_view pathname) = 0;
  virtual NoResult unlinkat(int dirfd, std::string_view pathname, int flags, bool recursive = false) = 0;
  virtual NoResult remove(std::string_view pathname,
                          std::optional<bool> isDir = std::nullopt,
                          bool recursive = false) = 0;
  virtual NoResult removeat(int dirfd,
                            std::string_view pathname,
                            std::optional<bool> isDir = std::nullopt,
                            bool recursive = false) = 0;

  virtual NoResult rename(std::string_view oldpath, std::string_view newpath) = 0;
  virtual NoResult renameat(int olddirfd, std::string_view oldpath, int newdirfd, std::string_view newpath) = 0;

  virtual NoResult stat(std::string_view pathname, struct stat *statbuf, bool ignoreCache = false) = 0;
  virtual NoResult fstat(int fd, struct stat *statbuf) = 0;
  virtual NoResult lstat(std::string_view pathname, struct stat *statbuf, bool ignoreCache = false) = 0;
  virtual NoResult fstatat(int dirfd, std::string_view pathname, struct stat *statbuf, int flags) = 0;

  virtual Result<std::string> readlink(std::string_view pathname, bool ignoreCache = false) = 0;
  virtual Result<std::string> readlinkat(int dirfd, std::string_view pathname, bool ignoreCache = false) = 0;
  virtual Result<std::string> realpath(std::string_view pathname, bool absolute = true) = 0;
  virtual Result<std::string> realpathat(int dirfd, std::string_view pathname, bool absolute = true) = 0;

  virtual Result<int> dup(int oldfd) = 0;
  virtual Result<int> dup2(int oldfd, int newfd) = 0;
  virtual Result<int> dup3(int oldfd, int newfd, int flags) = 0;

  // NOTE: we don't use umask to modify mode when open()/creat() new files
  virtual Result<int> creat(std::string_view pathname,
                            mode_t mode = 0666,
                            bool excl = false,
                            bool ignoreCache = false) = 0;
  virtual Result<int> creatat(int dirfd,
                              std::string_view pathname,
                              mode_t mode = 0666,
                              bool excl = false,
                              bool ignoreCache = false) = 0;
  virtual NoResult symlink(std::string_view target, std::string_view linkpath) = 0;
  virtual NoResult symlinkat(std::string_view target, int newdirfd, std::string_view linkpath) = 0;

  virtual NoResult link(std::string_view oldpath, std::string_view newpath) = 0;
  virtual NoResult linkat(int olddirfd,
                          std::string_view oldpath,
                          int newdirfd,
                          std::string_view newpath,
                          int flags) = 0;

  // we use O_NONBLOCK flag to indicate we want to ignore the inode cache
  // if you want to read file immediately after operating on it, use this flag
  virtual Result<int> open(std::string_view pathname, int flags, mode_t mode = 0666) = 0;
  virtual Result<int> openat(int dirfd, std::string_view pathname, int flags, mode_t mode = 0666) = 0;
  virtual NoResult close(int fd) = 0;

  virtual NoResult utimes(std::string_view filename, const struct timeval times[2]) = 0;
  virtual NoResult futimens(int fd, const struct timespec times[2]) = 0;
  virtual NoResult utimensat(int dirfd, std::string_view pathname, const struct timespec times[2], int flags) = 0;

  virtual NoResult chmod(std::string_view pathname, mode_t mode) = 0;
  // virtual NoResult fchmod(int fd, mode_t mode) = 0;
  virtual NoResult fchmodat(int dirfd, std::string_view pathname, mode_t mode, int flags) = 0;

  virtual NoResult chown(std::string_view pathname, uid_t owner, gid_t group) = 0;
  // virtual NoResult fchown(int fd, uid_t owner, gid_t group) = 0;
  virtual NoResult lchown(std::string_view pathname, uid_t owner, gid_t group) = 0;
  virtual NoResult fchownat(int dirfd, std::string_view pathname, uid_t owner, gid_t group, int flags) = 0;

  virtual NoResult chdir(std::string_view path, bool ignoreCache = false) = 0;
  virtual NoResult fchdir(int fd) = 0;
  virtual Result<std::string> getcwd() = 0;

 public:  // io ops
  virtual Result<struct iovec> iovalloc(size_t bytes, int numa = -1, bool global = false, size_t blockSize = 0) = 0;
  virtual NoResult iovfree(const std::shared_ptr<iovec_handle_t> &iovh) = 0;

  // has to use iovec returned by iovalloc, zero copy
  virtual NoResult preadv(int iovcnt, const struct iovec *iov, const struct ioseg *segv, ssize_t *resv) = 0;
  virtual NoResult pwritev(int iovcnt, const struct iovec *iov, const struct ioseg *segv, ssize_t *resv) = 0;

  // can use buffer allocated by caller, not zero copy, offset is maintained by client
  virtual Result<size_t> read(int fd, void *buf, size_t count, size_t readahead = 0) = 0;
  virtual Result<size_t> write(int fd, const void *buf, size_t count, bool flush = false) = 0;

  // may not be very accurate if seek from end,
  // since other clients may be writing and moving the eof when we're seeking
  virtual Result<off_t> lseek(int fd, off_t offset, int whence, size_t readahead = 0) = 0;

  virtual NoResult fdatasync(int fd) = 0;
  virtual NoResult fsync(int fd) = 0;
  virtual NoResult ftruncate(int fd, off_t length) = 0;

 public:  // advanced ops
  // caller can distribute the serialized sharedFileHandles to clients on other machines
  // and call openWithFileHandles() there to get fds without server communication
  // only fds opened for read/path/dirfd can be used this way, the root dir cannot though
  // used to reduce calls to the meta server, thus improve perf
  // how to distribute the bytes is not in consideration of the hf3fs client
  virtual Result<std::vector<uint8_t>> sharedFileHandles(const std::vector<int> &fds) = 0;
  // can return fewer than fhs, if too many files are opened
  virtual Result<std::vector<int>> openWithFileHandles(const std::vector<uint8_t> &fhs) = 0;
  virtual Result<std::string> sharedIovecHandle(const std::shared_ptr<iovec_handle_t> &iovh) = 0;
  virtual Result<struct iovec> openIovecHandle(const std::string &iovh, bool acrossAgent = false) = 0;
};
}  // namespace hf3fs::lib

#pragma once

#include "Serde.h"

namespace hf3fs::serde {
template <SerdeType T>
bool equals(const T &a, const T &b) {
  auto handler = [&](auto field) {
    using Field = std::remove_cvref_t<decltype(field)>;
    const auto &l = a.*(Field::getter);
    const auto &r = b.*(Field::getter);
    if constexpr (SerdeType<std::decay_t<decltype(l)>> && !requires { l == r; }) {
      return serde::equals(l, r);
    } else {
      return l == r;
    }
  };
  return refl::Helper::iterate<T>(std::move(handler));
}

template <SerdeType T, typename Ordering = std::weak_ordering>
auto compare(const T &a, const T &b) {
  auto cmp = Ordering::equivalent;
  auto handler = [&](auto field) {
    using Field = std::remove_cvref_t<decltype(field)>;
    auto res = a.*(Field::getter) <=> b.*(Field::getter);
    if (res == 0) {
      return true;
    } else {
      cmp = res < 0 ? Ordering::less : Ordering::greater;
      return false;
    }
  };
  refl::Helper::iterate<T>(std::move(handler));
  return cmp;
}
}  // namespace hf3fs::serde

#pragma once

#include "ServiceInfo.h"
#include "common/utils/Selector.h"
#include "common/utils/UtcTime.h"
#include "fbs/mgmtd/RoutingInfo.h"

namespace hf3fs::client {

class RoutingInfo {
 public:
  RoutingInfo() = default;
  RoutingInfo(std::shared_ptr<flat::RoutingInfo> info, SteadyTime refreshTime);

  const auto &raw() const { return info_; }

  SteadyTime lastRefreshTime() const { return lastRefreshTime_; }

  std::optional<flat::ChainInfo> getChain(flat::ChainId id) const;
  std::optional<flat::ChainInfo> getChain(flat::ChainRef ref) const;
  std::optional<flat::TargetInfo> getTarget(flat::TargetId id) const;
  std::optional<flat::NodeInfo> getNode(flat::NodeId id) const;

  template <UnaryPredicate<flat::NodeInfo> NodeSelector>
  std::vector<flat::NodeInfo> getNodeBy(const NodeSelector &selector) const {
    std::vector<flat::NodeInfo> res;
    if (info_) {
      for ([[maybe_unused]] const auto &[id, info] : info_->nodes) {
        if (selector(info)) res.push_back(info);
      }
    }
    return res;
  }

  template <UnaryPredicate<flat::NodeInfo> NodeSelector, UnaryPredicate<ServiceInfo> ServiceSelector>
  std::vector<ServiceInfo> getServiceBy(const NodeSelector &nodeSelector,
                                        const ServiceSelector &serviceSelector) const {
    std::vector<ServiceInfo> res;
    if (info_) {
      for ([[maybe_unused]] const auto &[id, node] : info_->nodes) {
        if (nodeSelector(node)) {
          auto smap = node.getAllServices();
          for (auto &[si, addrVec] : smap) {
            ServiceInfo info;
            info.name = si;
            info.nodeId = node.app.nodeId;
            info.nodeStatus = node.status;
            info.endpoints = std::move(addrVec);

            if (serviceSelector(info)) {
              res.push_back(std::move(info));
            }
          }
        }
      }
    }
    return res;
  }

 private:
  std::shared_ptr<flat::RoutingInfo> info_;
  SteadyTime lastRefreshTime_;
};

void logUnavailableChains(std::shared_ptr<RoutingInfo> routingInfo);
}  // namespace hf3fs::client

#pragma once

#include <fmt/format.h>
#include <folly/logging/xlog.h>

#include "Coroutine.h"

namespace hf3fs::_detail {
template <typename T>
struct CastResult {
  using type = Result<T>;
};

template <typename T>
struct CastResult<Result<T>> {
  using type = Result<T>;
};

template <>
struct CastResult<void> {
  using type = Result<Void>;
};

template <typename T>
struct CastResult<CoTask<T>> {
  using type = CoTask<typename CastResult<T>::type>;
};

template <typename F>
requires(!IsCoTaskV<F>) typename CastResult<std::invoke_result_t<F>>::type invoke(F &&f) {
  using RT = std::decay_t<decltype(f())>;
  if constexpr (std::is_void_v<RT>) {
    f();
    return Result<Void>(Void{});
  } else {
    return f();
  }
}

template <typename F>
requires(IsCoTaskV<F>) typename CastResult<std::invoke_result_t<F>>::type invoke(F &&f) {
  using RT = std::invoke_result_t<F>;
  if constexpr (std::is_same_v<RT, CoTask<void>>) {
    co_await f();
    co_return Void{};
  } else {
    co_return co_await f();
  }
}
}  // namespace hf3fs::_detail

#define LOG_COMMAND(NORMAL_LEVEL, desc, command)                   \
  do {                                                             \
    auto &&_desc = (desc);                                         \
    XLOGF(NORMAL_LEVEL, "{} ...", _desc);                          \
    auto _res = hf3fs::_detail::invoke([&] { return command; });   \
    using RT = std::decay_t<decltype(_res)>;                       \
    static_assert(std::is_same_v<RT, hf3fs::Result<hf3fs::Void>>); \
    if (UNLIKELY(_res.hasError())) {                               \
      XLOGF(ERR, "{} failed: {}", _desc, _res.error());            \
    } else {                                                       \
      XLOGF(NORMAL_LEVEL, "{} finished", _desc);                   \
    }                                                              \
  } while (false)

#define RETURN_ON_ERROR_LOG_WRAPPED(NORMAL_LEVEL, desc, command)   \
  do {                                                             \
    auto &&_desc = (desc);                                         \
    XLOGF(NORMAL_LEVEL, "{} ...", _desc);                          \
    auto _res = hf3fs::_detail::invoke([&] { return command; });   \
    using RT = std::decay_t<decltype(_res)>;                       \
    static_assert(std::is_same_v<RT, hf3fs::Result<hf3fs::Void>>); \
    if (UNLIKELY(_res.hasError())) {                               \
      XLOGF(ERR, "{} failed: {}", _desc, _res.error());            \
      RETURN_ERROR(_res);                                          \
    } else {                                                       \
      XLOGF(NORMAL_LEVEL, "{} finished", _desc);                   \
    }                                                              \
  } while (false)

#define CO_RETURN_ON_ERROR_LOG_WRAPPED(NORMAL_LEVEL, desc, command)                   \
  do {                                                                                \
    auto &&_desc = (desc);                                                            \
    XLOGF(NORMAL_LEVEL, "{} ...", _desc);                                             \
    auto _res = co_await hf3fs::_detail::invoke([&] { co_return co_await command; }); \
    using RT = std::decay_t<decltype(_res)>;                                          \
    static_assert(std::is_same_v<RT, hf3fs::Result<hf3fs::Void>>);                    \
    if (UNLIKELY(_res.hasError())) {                                                  \
      XLOGF(ERR, "{} failed: {}", _desc, _res.error());                               \
      CO_RETURN_ERROR(_res);                                                          \
    } else {                                                                          \
      XLOGF(NORMAL_LEVEL, "{} finished", _desc);                                      \
    }                                                                                \
  } while (false)

#define LOG_RESULT(NORMAL_LEVEL, command, ...)                                \
  do {                                                                        \
    auto &&_result = (command);                                               \
    if (_result.hasError())                                                   \
      XLOGF(ERR, "{} failed: {}", fmt::format(__VA_ARGS__), _result.error()); \
    else                                                                      \
      XLOGF(NORMAL_LEVEL, "{} finished", fmt::format(__VA_ARGS__));           \
  } while (false)

#define CO_RETURN_AND_LOG_RESULT(NORMAL_LEVEL, command, ...) \
  do {                                                       \
    auto &&_result2 = (command);                             \
    LOG_RESULT(NORMAL_LEVEL, _result2, __VA_ARGS__);         \
    co_return _result2;                                      \
  } while (false)

#define RETURN_AND_LOG_RESULT(NORMAL_LEVEL, command, ...) \
  do {                                                    \
    auto &&_result2 = (command);                          \
    LOG_RESULT(NORMAL_LEVEL, _result2, __VA_ARGS__);      \
    return _result2;                                      \
  } while (false)

#pragma once

#include <fmt/core.h>
#include <functional>

#include "MgmtdTypes.h"
#include "common/utils/StrongType.h"

namespace hf3fs::flat {

class ChainRef {
 public:
  ChainRef() {}
  ChainRef(ChainTableId tableId, ChainTableVersion tableVer, uint64_t index)
      : chainTableId(tableId),
        chainTableVersion(tableVer),
        chainIndex(index) {}

  std::tuple<ChainTableId, ChainTableVersion, uint64_t> decode() const {
    return std::make_tuple(chainTableId, chainTableVersion, chainIndex);
  }

  ChainTableId tableId() const { return chainTableId; }

  ChainTableVersion tableVersion() const { return chainTableVersion; }

  uint64_t index() const { return chainIndex; }

  auto operator<=>(const ChainRef &) const = default;

 private:
  ChainTableId chainTableId{0};
  ChainTableVersion chainTableVersion{0};
  uint64_t chainIndex{0};
};
}  // namespace hf3fs::flat

FMT_BEGIN_NAMESPACE

template <>
struct formatter<hf3fs::flat::ChainRef> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(const hf3fs::flat::ChainRef &ref, FormatContext &ctx) const {
    auto [id, v, i] = ref.decode();
    return fmt::format_to(ctx.out(), "ChainRef({}@{}-{})", id.toUnderType(), v.toUnderType(), i);
  }
};

FMT_END_NAMESPACE

#pragma once

#include "client/storage/StorageMessenger.h"
#include "common/net/Client.h"
#include "common/net/ib/RDMABuf.h"
#include "common/utils/ConfigBase.h"
#include "fbs/storage/Common.h"
#include "storage/update/UpdateJob.h"

namespace hf3fs::storage {

struct Components;
struct Target;

class ReliableForwarding {
 public:
  struct Config : ConfigBase<Config> {
    CONFIG_HOT_UPDATED_ITEM(retry_first_wait, 100_ms);
    CONFIG_HOT_UPDATED_ITEM(retry_max_wait, 1000_ms);
    CONFIG_HOT_UPDATED_ITEM(retry_total_time, 60_s);
    CONFIG_HOT_UPDATED_ITEM(max_inline_forward_bytes, Size{});
  };

  ReliableForwarding(const Config &config, Components &components)
      : config_(config),
        components_(components) {}

  Result<Void> init();

  void beforeStop() { stopped_ = true; }

  Result<Void> stopAndJoin();

  CoTask<IOResult> forwardWithRetry(ServiceRequestContext &requestCtx,
                                    const UpdateReq &req,
                                    const net::RDMARemoteBuf &rdmabuf,
                                    const ChunkEngineUpdateJob &chunkEngineJob,
                                    TargetPtr &target,
                                    CommitIO &commitIO,
                                    bool allowOutdatedChainVer = true);

  CoTask<IOResult> forward(const UpdateReq &req,
                           uint32_t retryCount,
                           const net::RDMARemoteBuf &rdmabuf,
                           const ChunkEngineUpdateJob &chunkEngineJob,
                           TargetPtr &target,
                           CommitIO &commitIO,
                           std::chrono::milliseconds timeout);

  CoTask<IOResult> doForward(const UpdateReq &req,
                             const net::RDMARemoteBuf &rdmabuf,
                             const ChunkEngineUpdateJob &chunkEngineJob,
                             uint32_t retryCount,
                             const Target &target,
                             bool &isSyncing,
                             std::chrono::milliseconds timeout);

 private:
  ConstructLog<"storage::ReliableForwarding"> constructLog_;
  const Config &config_;
  Components &components_;
  std::atomic<bool> stopped_ = false;
};

}  // namespace hf3fs::storage

#pragma once

#include <atomic>

namespace hf3fs {

// Wrapper of atomic value, support copy.
template <class T>
class AtomicValue {
 public:
  AtomicValue() = default;
  explicit AtomicValue(T value)
      : value_(value) {}
  AtomicValue(const AtomicValue &o)
      : value_(o.value()) {}

  T value(std::memory_order order = std::memory_order_seq_cst) const { return value_.load(order); }
  void setValue(T value, std::memory_order order = std::memory_order_seq_cst) { value_.store(value, order); }

  T load(std::memory_order order = std::memory_order_seq_cst) const { return value_.load(order); }
  void store(T value, std::memory_order order = std::memory_order_seq_cst) { value_.store(value, order); }

  operator T() const { return value(); }
  AtomicValue &operator=(T value) {
    setValue(value);
    return *this;
  }
  AtomicValue &operator=(const AtomicValue &o) {
    setValue(o.value());
    return *this;
  }

 private:
  std::atomic<T> value_;
};

}  // namespace hf3fs

#pragma once

#include "WithTimestamp.h"
#include "common/utils/SimpleRingBuffer.h"
#include "fbs/mgmtd/NodeInfo.h"
#include "fbs/mgmtd/NodeStatusChange.h"

namespace hf3fs::mgmtd {
class NodeInfoWrapper : public WithTimestamp<flat::NodeInfo> {
 public:
  using Base = WithTimestamp<flat::NodeInfo>;
  using Base::Base;

  void updateHeartbeatVersion(flat::HeartbeatVersion version) { version_ = version; }
  flat::HeartbeatVersion lastHbVersion() const { return version_; }

  void recordStatusChange(flat::NodeStatus status, UtcTime now) {
    if (statusChanges_.full()) statusChanges_.pop();
    statusChanges_.push(flat::NodeStatusChange::create(status, now));
  }

  std::vector<flat::NodeStatusChange> getAllStatusChanges() {
    return std::vector<flat::NodeStatusChange>(statusChanges_.begin(), statusChanges_.end());
  }

 private:
  flat::HeartbeatVersion version_{0};  // used for rejecting stale heartbeat requests
  static constexpr size_t kStatusChangeCount = 100;

  SimpleRingBuffer<flat::NodeStatusChange> statusChanges_{kStatusChangeCount};
};
}  // namespace hf3fs::mgmtd

#pragma once

#include <folly/logging/LogCategoryConfig.h>
#include <folly/logging/LogLevel.h>
#include <vector>

#include "common/utils/ConfigBase.h"
#include "common/utils/Size.h"
#include "common/utils/String.h"

namespace hf3fs::logging {
struct LogLevel {
  LogLevel() = default;
  LogLevel(folly::LogLevel l)
      : level(l) {}
  LogLevel(std::string_view s) { level = folly::stringToLogLevel(folly::StringPiece(s)); }

  String toString() const { return folly::logLevelToString(level); }
  operator folly::LogLevel() const { return level; }

  folly::LogLevel level = folly::LogLevel::INFO;
};

struct LogCategoryConfig : public ConfigBase<LogCategoryConfig> {
  CONFIG_ITEM(categories, std::vector<String>{"."});
  CONFIG_ITEM(level, LogLevel(folly::LogLevel::INFO));
  CONFIG_ITEM(inherit, true);
  CONFIG_ITEM(propagate, LogLevel(folly::LogLevel::MIN_LEVEL));
  CONFIG_ITEM(handlers, std::vector<String>());

 public:
  bool operator==(const LogCategoryConfig &o) const {
    return categories() == o.categories() && level() == o.level() && inherit() == o.inherit() &&
           propagate() == o.propagate() && handlers() == o.handlers();
  }

  bool operator!=(const LogCategoryConfig &o) const { return !(*this == o); }
};

struct LogHandlerConfig : public ConfigBase<LogHandlerConfig> {
  enum class WriterType {
    FILE,
    STREAM,
    EVENT,
  };
  enum class StreamType {
    STDOUT,
    STDERR,
  };
  CONFIG_ITEM(name, "");
  CONFIG_ITEM(writer_type, WriterType::FILE);
  // FILE options
  CONFIG_ITEM(file_path, "");  // leave empty if you want `setupLogConfig` to auto generate path
  CONFIG_ITEM(async, true);
  // ROTATE FILE options
  CONFIG_ITEM(rotate, true);
  CONFIG_ITEM(max_files, 100);
  CONFIG_ITEM(max_file_size, 10_MB);
  CONFIG_ITEM(rotate_on_open, false);
  // STREAM options
  CONFIG_ITEM(stream_type, StreamType::STDERR);
  // SHARED options
  CONFIG_ITEM(start_level, LogLevel(folly::LogLevel::MIN_LEVEL));

 public:
  bool operator==(const LogHandlerConfig &o) const {
    return name() == o.name() && writer_type() == o.writer_type() && file_path() == o.file_path() &&
           async() == o.async() && rotate() == o.rotate() && max_files() == o.max_files() &&
           max_file_size() == o.max_file_size() && rotate_on_open() == o.rotate_on_open() &&
           stream_type() == o.stream_type() && start_level() == o.start_level();
  }

  bool operator!=(const LogHandlerConfig &o) const { return !(*this == o); }
};

struct LogConfig : public ConfigBase<LogConfig> {
  static LogCategoryConfig makeRootCategoryConfig() {
    LogCategoryConfig cfg;
    cfg.set_categories({"."});
    cfg.set_level(folly::LogLevel::INFO);
    cfg.set_handlers({"normal", "err", "fatal"});
    return cfg;
  }

  static LogCategoryConfig makeEventCategoryConfig() {
    LogCategoryConfig cfg;
    cfg.set_categories(std::vector<String>{"eventlog"});
    cfg.set_level(folly::LogLevel::INFO);
    cfg.set_propagate(folly::LogLevel::ERR);
    cfg.set_inherit(false);
    cfg.set_handlers({"event"});
    return cfg;
  }

  static LogHandlerConfig makeNormalHandlerConfig() {
    LogHandlerConfig cfg;
    cfg.set_name("normal");
    cfg.set_async(true);
    cfg.set_start_level(folly::LogLevel::MIN_LEVEL);
    return cfg;
  }

  static LogHandlerConfig makeErrHandlerConfig() {
    LogHandlerConfig cfg;
    cfg.set_name("err");
    cfg.set_async(false);
    cfg.set_start_level(folly::LogLevel::ERR);
    return cfg;
  }

  static LogHandlerConfig makeFatalHandlerConfig() {
    LogHandlerConfig cfg;
    cfg.set_name("fatal");
    cfg.set_async(false);
    cfg.set_start_level(folly::LogLevel::FATAL);
    cfg.set_writer_type(LogHandlerConfig::WriterType::STREAM);
    cfg.set_stream_type(LogHandlerConfig::StreamType::STDERR);
    return cfg;
  }

  static LogHandlerConfig makeEventHandlerConfig() {
    LogHandlerConfig cfg;
    cfg.set_name("event");
    cfg.set_writer_type(LogHandlerConfig::WriterType::EVENT);
    cfg.set_async(true);
    cfg.set_start_level(folly::LogLevel::INFO);
    return cfg;
  }

  CONFIG_HOT_UPDATED_ITEM(categories, std::vector<LogCategoryConfig>({makeRootCategoryConfig()}));
  CONFIG_HOT_UPDATED_ITEM(
      handlers,
      std::vector<LogHandlerConfig>({makeNormalHandlerConfig(), makeErrHandlerConfig(), makeFatalHandlerConfig()}));
};

String generateLogConfig(const LogConfig &c, String serverName);
}  // namespace hf3fs::logging

#pragma once

#include <folly/experimental/coro/SharedMutex.h>

#include "common/utils/Coroutine.h"

namespace hf3fs {
namespace detail {
template <typename Guard, typename T>
class CoLockGuard {
 public:
  CoLockGuard(Guard &&guard, T &obj)
      : guard_(std::move(guard)),
        obj_(obj) {}

  T *operator->() { return &obj_; }

  T &operator*() { return obj_; }

 private:
  Guard guard_;
  T &obj_;
};

template <typename T>
using CoExclusiveLockGuard = CoLockGuard<std::unique_lock<folly::coro::SharedMutex>, T>;

template <typename T>
using CoSharedLockGuard = CoLockGuard<folly::coro::SharedLock<folly::coro::SharedMutex>, const T>;
}  // namespace detail

template <typename T>
class CoroSynchronized {
 public:
  using SharedLockPtr = detail::CoSharedLockGuard<T>;
  using LockPtr = detail::CoExclusiveLockGuard<T>;

  template <typename... Args>
  explicit CoroSynchronized(Args &&...args)
      : obj_(std::forward<Args>(args)...) {}

  CoTask<SharedLockPtr> coSharedLock() {
    auto lock = co_await sharedMu_.co_scoped_lock_shared();
    co_return SharedLockPtr(std::move(lock), obj_);
  }

  CoTask<LockPtr> coLock() {
    auto lock = co_await sharedMu_.co_scoped_lock();
    co_return LockPtr(std::move(lock), obj_);
  }

 private:
  folly::coro::SharedMutex sharedMu_;
  T obj_;
};
}  // namespace hf3fs

#pragma once

#include <algorithm>
#include <cassert>
#include <fcntl.h>
#include <folly/Overload.h>
#include <folly/logging/xlog.h>
#include <string>
#include <string_view>
#include <utility>
#include <vector>

#include "client/mgmtd/ICommonMgmtdClient.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Result.h"
#include "common/utils/StatusCode.h"
#include "fbs/meta/Common.h"
#include "fbs/meta/Schema.h"
#include "meta/store/DirEntry.h"
#include "meta/store/Inode.h"

namespace hf3fs::meta::server {

template <typename T>
inline InodeId getInodeId(T &&val) {
  return folly::variant_match(
      std::forward<T>(val),
      [](const Inode &inode) { return inode.id; },
      [](const DirEntry &entry) { return entry.id; },
      [](const InodeId &id) { return id; },
      [](const std::pair<InodeId, Acl> &cachedAcl) { return cachedAcl.first; });
}

template <typename T>
inline Acl getDirectoryAcl(T &&val) {
  return folly::variant_match(
      std::forward<T>(val),
      [](const Inode &inode) {
        assert(inode.isDirectory());
        return inode.acl;
      },
      [](const DirEntry &entry) {
        assert(entry.isDirectory() && entry.dirAcl.has_value());
        return *entry.dirAcl;
      },
      [](const std::pair<InodeId, Acl> &cachedAcl) { return cachedAcl.second; });
}

template <typename T>
inline InodeType getInodeType(T &&val) {
  return folly::variant_match(
      std::forward<T>(val),
      [](const Inode &inode) { return inode.getType(); },
      [](const DirEntry &entry) { return entry.type; },
      [](const std::pair<InodeId, Acl> &) { return InodeType::Directory; });
}

template <typename T>
inline Result<T> checkMetaFound(std::optional<T> val) {
  if (!val.has_value()) {
    return makeError(MetaCode::kNotFound);
  }
  return std::move(val.value());
}

inline bool isFirstMeta(client::ICommonMgmtdClient &mgmtd, flat::NodeId nodeId) {
  auto routing = mgmtd.getRoutingInfo();
  if (!routing) {
    return false;
  }
  auto nodes = routing->getNodeBy(flat::selectNodeByType(flat::NodeType::META) && flat::selectActiveNode());
  auto first =
      std::min_element(nodes.begin(), nodes.end(), [](auto &a, auto &b) { return a.app.nodeId < b.app.nodeId; });
  return first != nodes.end() && first->app.nodeId == nodeId;
}

}  // namespace hf3fs::meta::server

#pragma once

#include <folly/Hash.h>

namespace hf3fs {

using int128_t = __int128;
using uint128_t = unsigned __int128;

}  // namespace hf3fs

template <>
struct std::hash<hf3fs::uint128_t> {
  constexpr size_t operator()(hf3fs::uint128_t const &i) const noexcept {
    auto const hi = static_cast<uint64_t>(i >> 64);
    auto const lo = static_cast<uint64_t>(i);
    return folly::hash::hash_128_to_64(hi, lo);
  }
};
template <>
struct std::hash<hf3fs::int128_t> {
  constexpr size_t operator()(hf3fs::int128_t const &i) const noexcept {
    return hash<hf3fs::uint128_t>{}(static_cast<hf3fs::uint128_t>(i));
  }
};

#pragma once

#include <folly/File.h>
#include <tuple>

namespace hf3fs::logging {

class FileHelper {
 public:
  FileHelper() = default;

  FileHelper(const FileHelper &) = delete;
  FileHelper &operator=(const FileHelper &) = delete;
  ~FileHelper();

  void open(const std::string &fname, bool truncate = false);
  void reopen(bool truncate);
  void flush();
  void close();
  void writev(iovec *iov, int count);
  void write(const void *buf, size_t count);
  size_t size() const;
  const std::string &filename() const;

  //
  // return file path and its extension:
  //
  // "mylog.txt" => ("mylog", ".txt")
  // "mylog" => ("mylog", "")
  // "mylog." => ("mylog.", "")
  // "/dir1/dir2/mylog.txt" => ("/dir1/dir2/mylog", ".txt")
  //
  // the starting dot in filenames is ignored (hidden files):
  //
  // ".mylog" => (".mylog". "")
  // "my_folder/.mylog" => ("my_folder/.mylog", "")
  // "my_folder/.mylog.txt" => ("my_folder/.mylog", ".txt")
  static std::tuple<std::string, std::string> splitByExtension(const std::string &fname);

  operator bool() const { return file_ != nullptr; }

 private:
  int fd() const;

  const int open_tries_ = 5;
  const unsigned int open_interval_ = 10;
  std::unique_ptr<folly::File> file_;
  std::string filename_;
};
}  // namespace hf3fs::logging

#pragma once

#include <cstdint>
#include <string_view>

namespace hf3fs::kv {
// fixed size prefix
inline constexpr uint32_t makePrefixValue(const char (&s)[5]) {
  return s[0] + (static_cast<uint32_t>(s[1]) << 8) + (static_cast<uint32_t>(s[2]) << 16) +
         (static_cast<uint32_t>(s[3]) << 24);
}

// use enum for avoiding duplicated prefixes.
enum class KeyPrefix : uint32_t {
  Unknown = makePrefixValue("UNKW"),
#define DEFINE_PREFIX(name, s) name = makePrefixValue(s),

#include "KeyPrefix-def.h"
};

inline constexpr std::string_view toStr(KeyPrefix prefix) {
  switch (prefix) {
#define DEFINE_PREFIX(name, s) \
  case KeyPrefix::name:        \
    return s;

#include "KeyPrefix-def.h"
    default:
      return "UNKW";
  }
}

}  // namespace hf3fs::kv

#pragma once

#include <libaio.h>
#include <liburing.h>
#include <vector>

#include "storage/aio/BatchReadJob.h"
#include "storage/store/StorageTargets.h"

namespace hf3fs::storage {

class IoStatus {
 public:
  virtual ~IoStatus() = default;

  bool hasUnfinishedBatchReadJob() const { return iterator_; }

  void setAioReadJobIterator(AioReadJobIterator it) { iterator_ = it; }

  bool availableToSubmit() const { return inflight_ < maxEvents_; }

  uint32_t inflight() const { return inflight_; }

  virtual void collect() = 0;

  virtual void submit() = 0;

  virtual void reap(uint32_t minCompleteIn) = 0;

 protected:
  AioReadJobIterator iterator_;
  uint32_t maxEvents_ = 0;
  uint32_t inflight_ = 0;
};

class AioStatus : public IoStatus {
 public:
  ~AioStatus() override;

  Result<Void> init(uint32_t maxEvents);

  void collect() override;

  void submit() override;

  void reap(uint32_t minCompleteIn) override;

 private:
  uint32_t readyToSubmit_ = 0;
  io_context_t aioContext_ = nullptr;
  std::vector<struct iocb> iocbs_;
  std::vector<struct iocb *> availables_;
  std::vector<struct io_event> events_;
};

class IoUringStatus : public IoStatus {
 public:
  ~IoUringStatus() override;

  Result<Void> init(uint32_t maxEvents, const std::vector<int> &fds, const std::vector<struct iovec> &iovecs);

  void collect() override;

  void submit() override;

  void reap(uint32_t minCompleteIn) override;

 private:
  struct io_uring ring_ {};
  std::vector<AioReadJob *> submittingJobs_;
};

}  // namespace hf3fs::storage

#pragma once
#include <algorithm>
#include <climits>
#include <cstddef>

#include "common/monitor/Recorder.h"

namespace hf3fs::memory {

class AllocatedMemoryCounter {
 public:
  AllocatedMemoryCounter();

  ~AllocatedMemoryCounter();

  static void initialize();

  static void shutdown();

  void add(size_t size) {
    allocatedMemory_[0] += size;
    allocatedMemory_[calcBucket(size)] += size;
    tryReport();
  }

  void sub(size_t size) {
    deallocatedMemory_[0] += size;
    deallocatedMemory_[calcBucket(size)] += size;
    tryReport();
  }

 private:
  static size_t calcBucket(size_t size) {
    size_t sizeWidth = SIZE_WIDTH - __builtin_clzll(size);
    size_t bucket = sizeWidth > kMinBucketSizeWidth ? sizeWidth - kMinBucketSizeWidth + 1 : 1;
    return std::min(bucket, kMaxNumBuckets - 1);
  }

  static size_t calcBucketSize(size_t bucketIndex) { return 1ULL << (kMinBucketSizeWidth + bucketIndex - 2); }

  static void initFromEnvVars();

  static void initBucketTagSets();

  void initThreadTagSet();

  void tryReport(bool force = false);

 public:
  static constexpr size_t kMinBucketSizeWidth = 9;
  static constexpr size_t kMinBucketSizeBytes = 1 << (kMinBucketSizeWidth - 1);  // 256 bytes
  static constexpr size_t kMaxNumBuckets = 20;

 private:
  monitor::TagSet threadTagSet_;
  bool initialized_;
  bool reporting_;

  ssize_t allocatedMemory_[kMaxNumBuckets] = {0};
  ssize_t deallocatedMemory_[kMaxNumBuckets] = {0};
};

}  // namespace hf3fs::memory

#pragma once

#include <fmt/format.h>
#include <optional>

namespace hf3fs {
template <typename T>
inline std::optional<std::decay_t<T>> makeOptional(const T *ptr) {
  if (ptr) return std::make_optional<T>(*ptr);
  return std::nullopt;
}

template <typename T>
auto optionalMap(const std::optional<T> &src, auto &&transformer) {
  using DstType = std::decay_t<decltype(transformer(src.value()))>;
  if (src.has_value()) {
    return std::optional<DstType>(transformer(src.value()));
  }
  return std::optional<DstType>();
}

template <typename T>
struct OptionalFmt {
  explicit OptionalFmt(const std::optional<T> &val)
      : val_(val) {}

  std::string toString() const { return fmt::format("{}", *this); }

  const std::optional<T> &val_;
};
}  // namespace hf3fs

FMT_BEGIN_NAMESPACE

template <typename T>
struct formatter<hf3fs::OptionalFmt<T>> : formatter<T> {
  template <typename FormatContext>
  auto format(const hf3fs::OptionalFmt<T> &op, FormatContext &ctx) const {
    if (op.val_.has_value()) {
      return fmt::format_to(ctx.out(), "{}", *op.val_);
    } else {
      return fmt::format_to(ctx.out(), "std::nullopt");
    }
  }
};

FMT_END_NAMESPACE

#pragma once

#include "common/app/AppInfo.h"
#include "common/app/ConfigStatus.h"
#include "common/app/ConfigUpdateRecord.h"
#include "common/logging/LogConfig.h"
#include "common/monitor/Monitor.h"
#include "common/net/Server.h"
#include "common/serde/Serde.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/UtcTimeSerde.h"
#include "memory/common/MemoryAllocatorConfig.h"

namespace hf3fs {
class ApplicationBase {
 public:
  struct Config : public ConfigBase<Config> {
    CONFIG_OBJ(log, logging::LogConfig);
    CONFIG_OBJ(monitor, monitor::Monitor::Config);
    CONFIG_OBJ(memory, memory::MemoryAllocatorConfig);
  };
  int run(int argc, char *argv[]);

  static void handleSignal(int signum);

  using ConfigFlags = std::vector<config::KeyValue>;
  static Result<Void> parseFlags(std::string_view prefix, int *argc, char ***argv, ConfigFlags &flags);

  static Result<Void> initConfig(config::IConfig &cfg,
                                 const String &path,
                                 bool dump,
                                 const std::vector<config::KeyValue> &flags);

  static Result<Void> loadConfig(config::IConfig &cfg, const String &path, const std::vector<config::KeyValue> &flags);

  static Result<Void> updateConfig(const String &configContent, const String &configDesc);

  static Result<Void> hotUpdateConfig(const String &update, bool render);

  static Result<Void> validateConfig(const String &configContent, const String &configDesc);

  static Result<String> getConfigString(std::string_view configKey);

  static Result<std::pair<String, Result<String>>> renderConfig(const String &configContent,
                                                                bool testUpdate,
                                                                bool isHotUpdate);

  static std::optional<flat::AppInfo> getAppInfo();

  static std::optional<app::ConfigUpdateRecord> getLastConfigUpdateRecord();

  static ConfigStatus getConfigStatus();

 protected:
  ApplicationBase();
  ~ApplicationBase() = default;

  virtual void stop() = 0;

  virtual int mainLoop();

  virtual Result<Void> parseFlags(int *argc, char ***argv) = 0;

  virtual Result<Void> initApplication() = 0;

  virtual config::IConfig *getConfig() = 0;

  virtual const flat::AppInfo *info() const { return nullptr; }

  virtual bool configPushable() const { return true; }

  virtual void onConfigUpdated() {}
};

void stopAndJoin(net::Server *server);
}  // namespace hf3fs

#pragma once

#include "Recorder.h"

namespace hf3fs::monitor {

class ScopedCounterWriter {
 public:
  ScopedCounterWriter(monitor::DistributionRecorder &recorder,
                      std::atomic_int64_t &counter,
                      int64_t value,
                      const monitor::TagSet &tagSet)
      : recorder_(recorder),
        counter_(counter),
        value_(value),
        tagSet_(tagSet) {
    int64_t currentCounter = counter_.fetch_add(value_) + value_;
    if (tagSet.size() > 0)
      recorder_.addSample(currentCounter, tagSet_);
    else
      recorder_.addSample(currentCounter);
  }

  ScopedCounterWriter(monitor::DistributionRecorder &recorder,
                      std::atomic_int64_t &counter,
                      int64_t value,
                      const std::string &instance = "")
      : ScopedCounterWriter(recorder,
                            counter,
                            value,
                            instance.empty() ? monitor::TagSet() : monitor::instanceTagSet(instance)) {}

  ~ScopedCounterWriter() {
    int64_t currentCounter = counter_.fetch_add(-value_) - value_;
    if (tagSet_.size() > 0)
      recorder_.addSample(currentCounter, tagSet_);
    else
      recorder_.addSample(currentCounter);
  }

 private:
  monitor::DistributionRecorder &recorder_;
  std::atomic_int64_t &counter_;
  int64_t value_;
  const monitor::TagSet tagSet_;
};

class ScopedLatencyWriter {
 public:
  ScopedLatencyWriter(monitor::LatencyRecorder &recorder, const monitor::TagSet &tagSet)
      : recorder_(recorder),
        startTime_(hf3fs::SteadyClock::now()),
        tagSet_(tagSet) {}

  ScopedLatencyWriter(monitor::LatencyRecorder &recorder, const std::string &instance = "")
      : ScopedLatencyWriter(recorder, instance.empty() ? monitor::TagSet() : monitor::instanceTagSet(instance)) {}

  ~ScopedLatencyWriter() {
    if (tagSet_.size() > 0)
      recorder_.addSample(getElapsedTime(), tagSet_);
    else
      recorder_.addSample(getElapsedTime());
  }

  hf3fs::SteadyClock::duration getElapsedTime() const { return hf3fs::SteadyClock::now() - startTime_; }

 private:
  monitor::LatencyRecorder &recorder_;
  hf3fs::SteadyClock::time_point startTime_;
  const monitor::TagSet tagSet_;
};

}  // namespace hf3fs::monitor

#pragma once

#include <memory>

#include "client/mgmtd/MgmtdClientForClient.h"
#include "client/storage/StorageClient.h"
#include "common/logging/LogConfig.h"
#include "common/net/Client.h"
#include "common/net/Server.h"
#include "common/utils/BackgroundRunner.h"
#include "common/utils/ConfigBase.h"
#include "core/app/ServerAppConfig.h"
#include "core/app/ServerLauncher.h"
#include "core/app/ServerLauncherConfig.h"
#include "core/app/ServerMgmtdClientFetcher.h"

namespace hf3fs::migration::server {

class MigrationServer : public net::Server {
 public:
  static constexpr auto kName = "Migration";
  static constexpr auto kNodeType = flat::NodeType::CLIENT;

  struct CommonConfig : public ApplicationBase::Config {
    CommonConfig() {
      using logging::LogConfig;
      log().set_categories({LogConfig::makeRootCategoryConfig(), LogConfig::makeEventCategoryConfig()});
      log().set_handlers({LogConfig::makeNormalHandlerConfig(),
                          LogConfig::makeErrHandlerConfig(),
                          LogConfig::makeFatalHandlerConfig(),
                          LogConfig::makeEventHandlerConfig()});
    }
  };

  struct Config : public ConfigBase<Config> {
    CONFIG_OBJ(base, net::Server::Config, [](net::Server::Config &c) {
      c.set_groups_length(2);
      c.groups(0).listener().set_listen_port(8000);
      c.groups(0).set_services({"MigrationSerde"});

      c.groups(1).set_network_type(net::Address::TCP);
      c.groups(1).listener().set_listen_port(9000);
      c.groups(1).set_use_independent_thread_pool(true);
      c.groups(1).set_services({"Core"});
    });
    CONFIG_OBJ(background_client, net::Client::Config);
    CONFIG_OBJ(mgmtd_client, ::hf3fs::client::MgmtdClientForClient::Config);
    CONFIG_OBJ(storage_client, storage::client::StorageClient::Config);
  };

  MigrationServer(const Config &config);
  ~MigrationServer() override;

  Result<Void> beforeStart() final;

  Result<Void> beforeStop() final;

 private:
  const Config &config_;
  const ClientId clientId_;

  std::unique_ptr<net::Client> backgroundClient_;
  std::shared_ptr<::hf3fs::client::IMgmtdClientForClient> mgmtdClient_;
};

}  // namespace hf3fs::migration::server

#pragma once

#include "common/serde/Serde.h"

namespace hf3fs::serde {

template <class T>
struct BigEndian {
  BigEndian() = default;
  BigEndian(T v)
      : value(v) {}

  T value;
  operator T() const { return value; }
};

}  // namespace hf3fs::serde

template <class T>
struct hf3fs::serde::SerdeMethod<hf3fs::serde::BigEndian<T>> {
  static auto serdeTo(const BigEndian<T> &o) { return folly::Endian::swap(o.value); }
  static auto serdeToReadable(const BigEndian<T> &o) { return o.value; }
  static Result<BigEndian<T>> serdeFrom(size_t value) { return BigEndian<T>{folly::Endian::swap(value)}; }
  static Result<BigEndian<T>> serdeFromReadable(size_t value) { return BigEndian<T>{value}; }
};

#pragma once

#include "common/serde/Serde.h"
#include "common/utils/StrongType.h"

namespace hf3fs::serde {

template <typename T>
using SerdeToReadableMemberMethodReturnType = std::invoke_result_t<decltype(&T::serdeToReadable), T>;

template <typename T>
using SerdeToMemberMethodReturnType = std::invoke_result_t<decltype(&T::serdeTo), T>;

template <typename T>
using SerdeToReadableReturnType = std::invoke_result_t<decltype(serde::SerdeMethod<T>::serdeToReadable), const T &>;

template <typename T>
using SerdeToReturnType = std::invoke_result_t<decltype(serde::SerdeMethod<T>::serdeTo), const T &>;

template <typename T>
concept ConvertibleToString = std::is_convertible_v<T, std::string>;

template <typename T>
concept WithReadableSerdeMemberMethod =
    !StrongTyped<T> && !ConvertibleToString<T> && requires(const T &t, SerdeToReadableMemberMethodReturnType<T> s) {
  { t.serdeToReadable() } -> std::convertible_to<SerdeToReadableMemberMethodReturnType<T>>;
  { T::serdeFromReadable(s) } -> std::convertible_to<Result<T>>;
};

template <typename T>
concept WithReadableSerdeMethod = !StrongTyped<T> && !ConvertibleToString<T> && !WithReadableSerdeMemberMethod<T> &&
                                  requires(const T &t, SerdeToReadableReturnType<T> s) {
  { serde::SerdeMethod<T>::serdeToReadable(t) } -> std::convertible_to<SerdeToReadableReturnType<T>>;
  { serde::SerdeMethod<T>::serdeFromReadable(s) } -> std::convertible_to<Result<T>>;
};

template <typename T>
concept WithSerdeMemberMethod =
    !StrongTyped<T> && !ConvertibleToString<T> && !WithReadableSerdeMemberMethod<T> && !WithReadableSerdeMethod<T> &&
    requires(const T &t, SerdeToMemberMethodReturnType<T> s) {
  { t.serdeTo() } -> std::convertible_to<SerdeToMemberMethodReturnType<T>>;
  { T::serdeFrom(s) } -> std::convertible_to<Result<T>>;
};

template <typename T>
concept WithSerdeMethod =
    !StrongTyped<T> && !ConvertibleToString<T> && !WithReadableSerdeMemberMethod<T> && !WithReadableSerdeMethod<T> &&
    !WithSerdeMemberMethod<T> && requires(const T &t, SerdeToReturnType<T> s) {
  { serde::SerdeMethod<T>::serdeTo(t) } -> std::convertible_to<SerdeToReturnType<T>>;
  { serde::SerdeMethod<T>::serdeFrom(s) } -> std::convertible_to<Result<T>>;
};

template <typename T>
concept SerdeTypeWithoutSpecializedSerdeMethod =
    serde::SerdeType<T> && !WithReadableSerdeMemberMethod<T> && !WithReadableSerdeMethod<T> &&
    !WithSerdeMemberMethod<T> && !WithSerdeMethod<T>;

}  // namespace hf3fs::serde

namespace hf3fs::analytics {

const std::string kVariantValueIndexColumnSuffix = "ValIdx";
const std::string kResultErrorTypeColumnSuffix = "Error";

}  // namespace hf3fs::analytics

#pragma once

#include "common/serde/CallContext.h"
#include "fbs/storage/Service.h"
#include "storage/service/StorageOperator.h"

namespace hf3fs::storage {

class StorageService : public serde::ServiceWrapper<StorageService, storage::StorageSerde> {
 public:
  StorageService(StorageOperator &storageOperator)
      : storageOperator_(storageOperator) {}

  CoTryTask<BatchReadRsp> batchRead(serde::CallContext &ctx, const BatchReadReq &req) {
    reportReadQueueLatency(ctx);
    if (UNLIKELY(req.payloads.empty())) co_return BatchReadRsp{.tag = req.tag};
    ServiceRequestContext requestCtx{"batchRead", req.tag, req.retryCount, req.userInfo, req.debugFlags};
    co_return co_await storageOperator_.batchRead(requestCtx, req, ctx);
  }

  CoTryTask<WriteRsp> write(serde::CallContext &ctx, const WriteReq &req) {
    reportUpdateQueueLatency(ctx);
    ServiceRequestContext requestCtx{"write", req.tag, req.retryCount, req.userInfo, req.debugFlags};
    co_return co_await storageOperator_.write(requestCtx, req, ctx.transport()->ibSocket());
  }

  CoTryTask<UpdateRsp> update(serde::CallContext &ctx, const UpdateReq &req) {
    reportUpdateQueueLatency(ctx);
    ServiceRequestContext requestCtx{"update", req.tag, req.retryCount, req.userInfo, req.debugFlags};
    co_return co_await storageOperator_.update(requestCtx, req, ctx.transport()->ibSocket());
  }

  CoTryTask<QueryLastChunkRsp> queryLastChunk(serde::CallContext &ctx, const QueryLastChunkReq &req) {
    reportDefaultQueueLatency(ctx);
    if (UNLIKELY(req.payloads.empty())) co_return QueryLastChunkRsp{};
    ServiceRequestContext requestCtx{"queryLastChunk", req.tag, req.retryCount, req.userInfo, req.debugFlags};
    co_return co_await storageOperator_.queryLastChunk(requestCtx, req);
  }

  CoTryTask<TruncateChunksRsp> truncateChunks(serde::CallContext &ctx, const TruncateChunksReq &req) {
    reportDefaultQueueLatency(ctx);
    if (UNLIKELY(req.payloads.empty())) co_return TruncateChunksRsp{};
    ServiceRequestContext requestCtx{"truncateChunks",
                                     req.payloads.front().tag,
                                     req.payloads.front().retryCount,
                                     req.userInfo,
                                     req.debugFlags};
    co_return co_await storageOperator_.truncateChunks(requestCtx, req);
  }

  CoTryTask<RemoveChunksRsp> removeChunks(serde::CallContext &ctx, const RemoveChunksReq &req) {
    reportDefaultQueueLatency(ctx);
    if (UNLIKELY(req.payloads.empty())) co_return RemoveChunksRsp{};
    ServiceRequestContext requestCtx{"removeChunks",
                                     req.payloads.front().tag,
                                     req.payloads.front().retryCount,
                                     req.userInfo,
                                     req.debugFlags};
    co_return co_await storageOperator_.removeChunks(requestCtx, req);
  }

  CoTryTask<TargetSyncInfo> syncStart(serde::CallContext &ctx, const SyncStartReq &req) {
    reportDefaultQueueLatency(ctx);
    return storageOperator_.syncStart(req);
  }

  CoTryTask<SyncDoneRsp> syncDone(serde::CallContext &ctx, const SyncDoneReq &req) {
    reportDefaultQueueLatency(ctx);
    return storageOperator_.syncDone(req);
  }

  CoTryTask<SpaceInfoRsp> spaceInfo(serde::CallContext &ctx, const SpaceInfoReq &req) {
    reportDefaultQueueLatency(ctx);
    return storageOperator_.spaceInfo(req);
  }

  CoTryTask<CreateTargetRsp> createTarget(serde::CallContext &ctx, const CreateTargetReq &req) {
    reportDefaultQueueLatency(ctx);
    return storageOperator_.createTarget(req);
  }

  CoTryTask<OfflineTargetRsp> offlineTarget(serde::CallContext &ctx, const OfflineTargetReq &req) {
    reportDefaultQueueLatency(ctx);
    return storageOperator_.offlineTarget(req);
  }

  CoTryTask<RemoveTargetRsp> removeTarget(serde::CallContext &ctx, const RemoveTargetReq &req) {
    reportDefaultQueueLatency(ctx);
    return storageOperator_.removeTarget(req);
  }

  CoTryTask<QueryChunkRsp> queryChunk(serde::CallContext &ctx, const QueryChunkReq &req) {
    reportDefaultQueueLatency(ctx);
    return storageOperator_.queryChunk(req);
  }

  CoTryTask<GetAllChunkMetadataRsp> getAllChunkMetadata(serde::CallContext &ctx, const GetAllChunkMetadataReq &req) {
    reportDefaultQueueLatency(ctx);
    return storageOperator_.getAllChunkMetadata(req);
  }

 private:
  void reportReadQueueLatency(serde::CallContext &ctx);
  void reportUpdateQueueLatency(serde::CallContext &ctx);
  void reportDefaultQueueLatency(serde::CallContext &ctx);

 private:
  StorageOperator &storageOperator_;
};

}  // namespace hf3fs::storage

#pragma once

#include <functional>
#include <optional>
#include <string_view>

#include "client/storage/StorageClient.h"
#include "client/storage/TargetSelection.h"
#include "common/monitor/Recorder.h"
#include "common/utils/Coroutine.h"
#include "fbs/meta/Schema.h"
#include "fbs/mgmtd/MgmtdTypes.h"

namespace hf3fs::meta {

class FileOperation {
 public:
  struct Recorder {
    Recorder(std::string_view prefix)
        : removeChunksCount(fmt::format("{}.remove_chunks", prefix)),
          removeChunksSize(fmt::format("{}.remove_chunks_size", prefix)),
          queryChunksFailed(fmt::format("{}.query_chunks_failed", prefix)),
          removeChunksFailed(fmt::format("{}.remove_chunks_failed", prefix)),
          truncateChunkFailed(fmt::format("{}.truncate_chunk_failed", prefix)),
          queryLastChunkLatency(fmt::format("{}.query_last_chunk", prefix)),
          queryTotalChunkLatency(fmt::format("{}.query_total_chunks", prefix)),
          removeChunksLatency(fmt::format("{}.remove_chunks_latency", prefix)),
          truncateChunkLatency(fmt::format("{}.truncate_latency", prefix)) {}

    monitor::CountRecorder removeChunksCount;
    monitor::CountRecorder removeChunksSize;
    monitor::CountRecorder queryChunksFailed;
    monitor::CountRecorder removeChunksFailed;
    monitor::CountRecorder truncateChunkFailed;
    monitor::LatencyRecorder queryLastChunkLatency;
    monitor::LatencyRecorder queryTotalChunkLatency;
    monitor::LatencyRecorder removeChunksLatency;
    monitor::LatencyRecorder truncateChunkLatency;
  };

  FileOperation(storage::client::StorageClient &storage,
                const flat::RoutingInfo &routing,
                const flat::UserInfo &userInfo,
                const meta::Inode &inode,
                std::optional<std::reference_wrapper<Recorder>> recorder = {})
      : storage_(storage),
        routing_(routing),
        userInfo_(userInfo),
        inode_(inode),
        recorder_(recorder) {}

  struct QueryResult {
    uint64_t length = 0;
    uint64_t lastChunk = 0;
    uint64_t lastChunkLen = 0;
    uint64_t totalChunkLen = 0;
    uint64_t totalNumChunks = 0;
  };

  CoTryTask<QueryResult> queryChunks(bool queryTotalChunk, bool dynStripe);

  CoTryTask<std::map<flat::ChainId, QueryResult>> queryChunksByChain(bool queryTotalChunk, bool dynStripe);

  CoTryTask<std::pair<uint32_t, bool>> removeChunks(size_t targetLength,
                                                    size_t removeChunksBatchSize,
                                                    bool dynStripe,
                                                    storage::client::RetryOptions retry);

  CoTryTask<void> truncateChunk(size_t targetLength);

  CoTryTask<std::map<uint32_t, uint32_t>> queryAllChunks(uint64_t begin, uint64_t end);

 private:
  storage::client::StorageClient &storage_;
  const flat::RoutingInfo &routing_;
  const flat::UserInfo &userInfo_;
  const meta::Inode &inode_;
  std::optional<std::reference_wrapper<Recorder>> recorder_;
};

}  // namespace hf3fs::meta

#pragma once

#include "common/serde/Serde.h"
#include "common/serde/Service.h"
#include "common/utils/UtcTime.h"
#include "common/utils/Uuid.h"

namespace hf3fs::migration {

/* job state transition
  NotSubmitted --> Pending --> Running --> Succeeded
                    |           ^  ^
                    |           |  |
                    |  +--------+  |
                    |  |           |
                    v  v           v
                   Stopped     Failed
 */

enum JobStatus {
  NotSubmitted = 0,
  Pending,
  Running,
  Failed,
  Stopped,
  Succeeded,
};

struct StartJobReq {
  SERDE_STRUCT_FIELD(id,
                     Uuid::zero());  // start a new job if not found at server side; otherwise resume an existing job
  SERDE_STRUCT_FIELD(path, String{});
  SERDE_STRUCT_FIELD(mtime, UtcTime{});  // only files with mtime greater than this value
};

struct StartJobRsp {
  SERDE_STRUCT_FIELD(status, JobStatus::NotSubmitted);
};

struct StopJobReq {
  SERDE_STRUCT_FIELD(id, Uuid::zero());
};

struct StopJobRsp {
  SERDE_STRUCT_FIELD(status, JobStatus::NotSubmitted);
};

struct JobInfo {
  SERDE_STRUCT_FIELD(id, Uuid::zero());
  SERDE_STRUCT_FIELD(path, String{});
  SERDE_STRUCT_FIELD(mtime, UtcTime{});  // only files with mtime greater than this value
  SERDE_STRUCT_FIELD(status, JobStatus::NotSubmitted);
  SERDE_STRUCT_FIELD(startTime, UtcTime{});
  SERDE_STRUCT_FIELD(finishTime, UtcTime{});
  SERDE_STRUCT_FIELD(numFilesFound, uint64_t{});
  SERDE_STRUCT_FIELD(numFilesCopied, uint64_t{});
  SERDE_STRUCT_FIELD(numSrcFilesRemoved, uint64_t{});
  SERDE_STRUCT_FIELD(bytesOfFilesFound, uint64_t{});
  SERDE_STRUCT_FIELD(bytesOfFilesCopied, uint64_t{});
};

struct ListJobsReq {
  SERDE_STRUCT_FIELD(path, String{});
  SERDE_STRUCT_FIELD(ids, std::vector<Uuid>{});    // only jobs with specific ids
  SERDE_STRUCT_FIELD(status, JobStatus::Running);  // only jobs with this status
  SERDE_STRUCT_FIELD(after, UtcTime{});            // only jobs with start time greater than this value
  SERDE_STRUCT_FIELD(limit, uint32_t{10});         // max number of jobs in response
};

struct ListJobsRsp {
  SERDE_STRUCT_FIELD(jobs, std::vector<JobInfo>{});
};

SERDE_SERVICE(MigrationSerde, 0xF1) {
  SERDE_SERVICE_METHOD(start, 1, StartJobReq, StartJobRsp);
  SERDE_SERVICE_METHOD(stop, 2, StopJobReq, StopJobRsp);
  SERDE_SERVICE_METHOD(list, 3, ListJobsReq, ListJobsRsp);
};

}  // namespace hf3fs::migration

#pragma once

#include <atomic>
#include <folly/Synchronized.h>
#include <folly/logging/xlog.h>
#include <map>
#include <memory>
#include <mutex>
#include <optional>
#include <regex.h>
#include <string>
#include <vector>

#include "common/app/NodeId.h"
#include "common/kv/IKVEngine.h"
#include "common/kv/ITransaction.h"
#include "common/kv/KeyPrefix.h"
#include "common/serde/Serde.h"
#include "common/utils/BackgroundRunner.h"
#include "common/utils/CPUExecutorGroup.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/Coroutine.h"
#include "common/utils/UtcTime.h"
#include "fbs/meta/Common.h"
#include "meta/store/Inode.h"

namespace hf3fs::meta::server {

class Distributor {
 public:
  struct Config : ConfigBase<Config> {
    CONFIG_HOT_UPDATED_ITEM(update_interval, 1_s);
    CONFIG_HOT_UPDATED_ITEM(timeout, 30_s);
  };

  Distributor(const Config &config, flat::NodeId nodeId, std::shared_ptr<kv::IKVEngine> kvEngine)
      : config_(config),
        nodeId_(nodeId),
        kvEngine_(kvEngine) {
    XLOGF_IF(FATAL, !nodeId_, "invalid node id {}", nodeId_);
  }

  ~Distributor() { stopAndJoin(); }

  flat::NodeId nodeId() const { return nodeId_; }

  void start(CPUExecutorGroup &exec);
  void stopAndJoin(bool updateMap = true);

  flat::NodeId getServer(InodeId inodeId);

  CoTryTask<std::pair<bool, kv::Versionstamp>> checkOnServer(kv::IReadWriteTransaction &txn, InodeId inodeId);
  CoTryTask<std::pair<bool, kv::Versionstamp>> checkOnServer(kv::IReadWriteTransaction &txn,
                                                             InodeId inodeId,
                                                             flat::NodeId nodeId);

 private:
  static constexpr auto kPrefix = kv::toStr(kv::KeyPrefix::MetaDistributor);
  static constexpr auto kMapKey = kPrefix;

  struct ServerMap {
    SERDE_STRUCT_FIELD(active, std::vector<flat::NodeId>());
  };

  struct LatestServerMap : ServerMap {
    kv::Versionstamp versionstamp{0};
  };

  struct ServerStatus {
    std::string versionstamp;
    SteadyTime lastUpdate;
  };

  struct PerServerKey {
    static std::string pack(flat::NodeId nodeId);
    static flat::NodeId unpack(std::string_view key);
  };

  CoTryTask<kv::Versionstamp> loadVersion(kv::IReadOnlyTransaction &txn);
  CoTryTask<LatestServerMap> loadServerMap(kv::IReadOnlyTransaction &txn, bool update);

  CoTryTask<void> updateVersion(kv::IReadWriteTransaction &txn);
  CoTryTask<void> updateServerMap(kv::IReadWriteTransaction &txn, const ServerMap &map);

  CoTryTask<void> update(bool exit);
  CoTryTask<bool> update(kv::IReadWriteTransaction &txn, bool exit);

  const Config config_;
  flat::NodeId nodeId_;
  std::atomic<size_t> updated_{0};
  std::shared_ptr<kv::IKVEngine> kvEngine_;
  std::unique_ptr<BackgroundRunner> bgRunner_;

  folly::Synchronized<LatestServerMap> latest_;
  folly::Synchronized<std::map<flat::NodeId, ServerStatus>> servers_;
};

}  // namespace hf3fs::meta::server

#pragma once

#include <atomic>
#include <folly/IPAddressV4.h>
#include <gtest/gtest_prod.h>
#include <map>

#include "common/net/ib/IBConnect.h"
#include "common/net/ib/IBDevice.h"
#include "common/net/ib/IBSocket.h"
#include "common/serde/CallContext.h"
#include "common/serde/Service.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Duration.h"

namespace hf3fs::net {

SERDE_SERVICE(IBConnect, 11) {
  SERDE_SERVICE_METHOD(query, 1, IBQueryReq, IBQueryRsp);
  SERDE_SERVICE_METHOD(connect, 2, IBConnectReq, IBConnectRsp);
};

class IBConnectService : public serde::ServiceWrapper<IBConnectService, IBConnect> {
 public:
  using AcceptFn = std::function<void(std::unique_ptr<IBSocket>)>;

  IBConnectService(const IBSocket::Config &config, AcceptFn accept, std::function<Duration()> acceptTimeout)
      : config_(config),
        accept_(std::move(accept)),
        acceptTimeout_(acceptTimeout) {}

  CoTryTask<IBQueryRsp> query(serde::CallContext &ctx, const IBQueryReq &req);
  CoTryTask<IBConnectRsp> connect(serde::CallContext &ctx, const IBConnectReq &req);

 private:
  FRIEND_TEST(TestRDMA, SlowConnect);
  FRIEND_TEST(TestRDMA, ConnectitonLost);
  friend class IBSocket;

  static std::atomic<uint64_t> &delayMs() {
    static std::atomic<uint64_t> delay{0};
    return delay;
  }

  static std::atomic<bool> &connectionLost() {
    static std::atomic<bool> val{false};
    return val;
  }

  const IBSocket::Config &config_;
  AcceptFn accept_;
  std::function<Duration()> acceptTimeout_;
};

}  // namespace hf3fs::net

#pragma once

#include "common/serde/Serde.h"
#include "common/utils/Coroutine.h"
#include "fbs/meta/Schema.h"

namespace hf3fs::client::cli {

struct InodeRow {
  SERDE_STRUCT_FIELD(timestamp, std::time_t{});
  SERDE_STRUCT_FIELD(inode, meta::Inode{});
};

struct InodeTable {
  SERDE_STRUCT_FIELD(timestamp, std::time_t{});
  SERDE_STRUCT_FIELD(inodes, std::vector<InodeRow>{});

 public:
  bool dumpToFile(const Path &filePath) const;
  bool loadFromFile(const Path &filePath);
  bool dumpToParquetFile(const Path &filePath) const;
  bool loadFromParquetFile(const Path &filePath);
};
static_assert(serde::Serializable<InodeTable>);

std::vector<Path> listFilesFromPath(const Path path);

CoTryTask<Void> dumpInodesFromFdb(const std::string fdbClusterFile,
                                  const uint32_t numInodesPerFile,
                                  const std::string inodeDir,
                                  const bool parquetFormat = false,
                                  const bool dumpAllInodes = false,
                                  const uint32_t threads = 4);

CoTryTask<robin_hood::unordered_set<meta::InodeId>> loadInodeFromFiles(
    const Path inodePath,
    const robin_hood::unordered_set<meta::InodeId> &inodeIdsToPeek,
    const uint32_t parallel,
    const bool parquetFormat = false,
    std::time_t *inodeDumpTime = nullptr);

class Dispatcher;
CoTryTask<void> registerDumpInodesHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include <span>
#include <sys/shm.h>

#include "common/utils/Path.h"
#include "common/utils/Result.h"

namespace hf3fs {
class SysvShm {
 public:
  ~SysvShm();

  static Result<std::shared_ptr<SysvShm>> create(const Path &path, size_t size, int flags);
  std::span<uint8_t> getBuf() const;
  Result<shmid_ds> getStat() const;
  Result<Void> detach();
  Result<Void> remove();

 protected:
  SysvShm();

  int id_ = -1;
  uint8_t *ptr_ = nullptr;
  size_t size_ = 0;
  bool removed_ = false;
};
}  // namespace hf3fs

#pragma once

#include <folly/experimental/coro/Mutex.h>

#include "MgmtdData.h"
#include "common/utils/BackgroundRunner.h"
#include "common/utils/CoroSynchronized.h"
#include "common/utils/DefaultRetryStrategy.h"
#include "core/app/ServerEnv.h"
#include "core/user/UserStoreEx.h"
#include "fbs/mgmtd/NodeInfo.h"
#include "fbs/mgmtd/PersistentNodeInfo.h"
#include "fdb/FDBRetryStrategy.h"
#include "mgmtd/store/MgmtdStore.h"

namespace hf3fs::mgmtd {
struct MgmtdConfig;

using ClientSessionMap = RHStringHashMap<ClientSession>;

struct MgmtdState {
  MgmtdState(std::shared_ptr<core::ServerEnv> env, const MgmtdConfig &config);
  ~MgmtdState();

  UtcTime utcNow();
  Result<Void> validateClusterId(const core::ServiceOperation &ctx, std::string_view clusterId);
  CoTryTask<void> validateAdmin(const core::ServiceOperation &ctx, const flat::UserInfo &userInfo);
  CoTask<std::optional<flat::MgmtdLeaseInfo>> currentLease(UtcTime now);
  flat::NodeId selfId() const;
  kv::FDBRetryStrategy createRetryStrategy();

  const std::shared_ptr<core::ServerEnv> env_;
  flat::NodeInfo selfNodeInfo_;
  flat::PersistentNodeInfo selfPersistentNodeInfo_;
  const MgmtdConfig &config_;

  MgmtdStore store_;
  core::UserStoreEx userStore_;

  class WriterMutexGuard {
   public:
    WriterMutexGuard(std::unique_lock<folly::coro::Mutex> mu, std::string_view m)
        : mu_(std::move(mu)),
          method_(m) {}
    WriterMutexGuard(WriterMutexGuard &&other) = default;
    ~WriterMutexGuard();

   private:
    std::unique_lock<folly::coro::Mutex> mu_;
    std::string_view method_;
    SteadyTime start_ = SteadyClock::now();
  };

  template <NameWrapper method>
  CoTask<WriterMutexGuard> coScopedLock() {
    auto mu = co_await writerMu_.co_scoped_lock();
    co_return WriterMutexGuard(std::move(mu), method);
  }

  CoroSynchronized<MgmtdData> data_;
  CoroSynchronized<ClientSessionMap> clientSessionMap_;

 private:
  // logical lock for protecting the whole processing of a writer operation
  // during which read-modify-write will be performed on `data_`
  folly::coro::Mutex writerMu_;
};
}  // namespace hf3fs::mgmtd

#pragma once

#include <algorithm>
#include <cstdint>
#include <cstring>
#include <folly/Likely.h>
#include <folly/logging/xlog.h>
#include <optional>
#include <string>
#include <string_view>
#include <variant>
#include <vector>

#include "common/kv/ITransaction.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Path.h"
#include "common/utils/Result.h"
#include "common/utils/UtcTime.h"
#include "common/utils/Uuid.h"
#include "fbs/meta/Common.h"
#include "fbs/meta/Schema.h"

namespace hf3fs::meta::server {
using hf3fs::kv::IReadOnlyTransaction;
using hf3fs::kv::IReadWriteTransaction;

class Inode : public meta::Inode {
 public:
  using Base = meta::Inode;
  using Base::Base;

  Inode(Base base)
      : Base(std::move(base)) {}
  Inode(InodeId id, Acl acl, UtcTime time, std::variant<File, Directory, Symlink> type)
      : Base{id, InodeData{type, acl, 1, time, time, time}} {}

  static Inode newFile(InodeId id, Acl acl, Layout layout, UtcTime time) { return Inode(id, acl, time, File(layout)); }

  static Inode newDirectory(InodeId id, InodeId parent, std::string name, Acl acl, Layout layout, UtcTime time) {
    return Inode(id, acl, time, Directory{parent, std::move(layout), std::move(name)});
  }

  static Inode newSymlink(InodeId id, Path target, Uid uid, Gid gid, UtcTime time) {
    static constexpr Permission perm{0777};  // permission of symlink is never used, and won't changed
    return Inode(id, Acl(uid, gid, perm), time, Symlink{std::move(target)});
  }

  /** key format: kInodePrefx + InodeId.key */
  static std::string packKey(InodeId id);
  std::string packKey() const;
  Result<Void> unpackKey(std::string_view key);

  static Result<Inode> newUnpacked(std::string_view key, std::string_view value);

  // The difference of `snapshotLoad` and `load` is the former won't add key of inode into read conflict set.
  static CoTryTask<std::optional<Inode>> snapshotLoad(IReadOnlyTransaction &txn, InodeId id);
  static CoTryTask<std::optional<Inode>> load(IReadOnlyTransaction &txn, InodeId id);

  CoTryTask<void> addIntoReadConflict(IReadWriteTransaction &txn) {
#ifndef NDEBUG
    snapshotLoaded_ = false;
#endif
    co_return co_await txn.addReadConflict(packKey());
  }

  CoTryTask<void> store(IReadWriteTransaction &txn) const;
  /** Remove this inode */
  CoTryTask<void> remove(IReadWriteTransaction &txn) const;

  CoTryTask<DirEntry> snapshotLoadDirEntry(IReadOnlyTransaction &txn) const;

  static CoTryTask<Void> loadAncestors(IReadWriteTransaction &txn, std::vector<Inode> &ancestors, InodeId parent);

 private:
  template <const bool SNAPSHOT>
  static CoTryTask<std::optional<Inode>> loadImpl(IReadOnlyTransaction &txn, InodeId id);

#ifndef NDEBUG
  mutable bool snapshotLoaded_ = false;
#endif
};

static_assert(serde::SerializableToJson<Inode>);
}  // namespace hf3fs::meta::server

#pragma once

#include "common/serde/CallContext.h"
#include "common/serde/Service.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/Semaphore.h"

namespace hf3fs::net {

struct RDMATransmissionReq {
  SERDE_STRUCT_FIELD(uuid, size_t{});
};

struct RDMATransmissionRsp {
  SERDE_STRUCT_FIELD(dummy, Void{});
};

SERDE_SERVICE(RDMAControl, 10) { SERDE_SERVICE_METHOD(apply, 1, RDMATransmissionReq, RDMATransmissionRsp); };

class RDMATransmissionLimiter {
 public:
  RDMATransmissionLimiter(uint32_t maxConcurrentTransmission)
      : semaphore_(maxConcurrentTransmission) {}

  CoTask<void> co_wait();

  void signal(Duration latency);

  void updateMaxConcurrentTransmission(uint32_t value) { semaphore_.changeUsableTokens(value); }

 private:
  Semaphore semaphore_;
  std::atomic<uint32_t> current_{};
};
using RDMATransmissionLimiterPtr = std::shared_ptr<RDMATransmissionLimiter>;

class RDMAControlImpl : public serde::ServiceWrapper<RDMAControlImpl, RDMAControl> {
 public:
  class Config : public ConfigBase<Config> {
    CONFIG_HOT_UPDATED_ITEM(max_concurrent_transmission, 64u);
  };
  RDMAControlImpl(const Config &config)
      : config_(config),
        limiter_(std::make_shared<RDMATransmissionLimiter>(config_.max_concurrent_transmission())),
        guard_(config_.addCallbackGuard(
            [&] { limiter_->updateMaxConcurrentTransmission(config_.max_concurrent_transmission()); })) {}

  CoTryTask<RDMATransmissionRsp> apply(serde::CallContext &, const RDMATransmissionReq &req);

 private:
  const Config &config_;
  RDMATransmissionLimiterPtr limiter_;
  std::unique_ptr<ConfigCallbackGuard> guard_;
};

}  // namespace hf3fs::net

#pragma once

#include <folly/executors/CPUThreadPoolExecutor.h>
#include <folly/experimental/coro/Baton.h>

#include "common/monitor/Recorder.h"
#include "common/utils/BoundedQueue.h"
#include "common/utils/CPUExecutorGroup.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/Coroutine.h"

namespace hf3fs {

class DynamicCoroutinesPool {
 public:
  class Config : public ConfigBase<Config> {
    CONFIG_ITEM(queue_size, 1024u);
    CONFIG_HOT_UPDATED_ITEM(threads_num, 8ul, ConfigCheckers::checkPositive);
    CONFIG_HOT_UPDATED_ITEM(coroutines_num, 64u, ConfigCheckers::checkPositive);
  };

  DynamicCoroutinesPool(const Config &config, std::string_view name = "pool");

  ~DynamicCoroutinesPool() { stopAndJoin(); }

  Result<Void> start();

  Result<Void> stopAndJoin();

  void enqueue(CoTask<void> &&task) { queue_.enqueue(std::make_unique<CoTask<void>>(std::move(task))); }

 protected:
  Result<Void> setCoroutinesNum(uint32_t num);

  CoTask<void> run();

  void afterCoroutineStop();

 private:
  const Config &config_;
  BoundedQueue<std::unique_ptr<CoTask<void>>> queue_;
  std::unique_ptr<ConfigCallbackGuard> guard_;

  std::mutex mutex_;
  folly::CPUThreadPoolExecutor executor_;
  monitor::Recorder::TagRef<monitor::ValueRecorder> coroutinesNumRecorder_;

  std::atomic_flag stopped_{false};
  folly::coro::Baton baton_;
  uint32_t coroutinesNum_{};
  folly::Synchronized<uint32_t, std::mutex> currentRunning_{1};
};

}  // namespace hf3fs

#pragma once

#include <folly/hash/Hash.h>

#include "RobinHood.h"

namespace hf3fs {
struct StringHash {
  using is_transparent = void;  // enable heterogeneous overloads

  [[nodiscard]] auto operator()(std::string_view str) const noexcept -> uint64_t {
    return robin_hood::hash<std::string_view>{}(str);
  }
};

template <typename T>
using RHStringHashFlatMap = robin_hood::unordered_flat_map<std::string, T, StringHash, std::equal_to<>>;

template <typename T>
using RHStringHashNodeMap = robin_hood::unordered_node_map<std::string, T, StringHash, std::equal_to<>>;

template <typename T>
using RHStringHashMap = robin_hood::unordered_map<std::string, T, StringHash, std::equal_to<>>;

using RHStringHashSet = robin_hood::unordered_set<std::string, StringHash, std::equal_to<>>;

struct RobinHoodHasher {
  template <typename T>
  size_t operator()(const T &x) const {
    if constexpr (requires { robin_hood::hash<T>{}; }) {
      return robin_hood::hash<T>{}(x);
    } else {
      return std::hash<T>{}(x);
    }
  }
};

template <typename... Args>
auto robinhoodCombineHash(Args &&...args) {
  return folly::hash::hash_combine_generic(RobinHoodHasher{}, std::forward<Args>(args)...);
}
}  // namespace hf3fs

#pragma once

#include <cstdint>
#include <fmt/format.h>

namespace hf3fs {

struct Varint32 {
  uint32_t value;

  Varint32() = default;
  Varint32(uint32_t value)
      : value(value) {}

  operator uint32_t &() { return value; }
  operator uint32_t() const { return value; }
};

}  // namespace hf3fs

FMT_BEGIN_NAMESPACE

template <>
struct formatter<hf3fs::Varint32> : formatter<uint32_t> {
  template <typename FormatContext>
  auto format(hf3fs::Varint32 val, FormatContext &ctx) const {
    return formatter<uint32_t>::format(static_cast<uint32_t>(val), ctx);
  }
};

FMT_END_NAMESPACE

#pragma once

#include "FuseConfig.h"
#include "common/utils/AtomicSharedPtrTable.h"
#include "fbs/meta/Common.h"
#include "fbs/meta/Schema.h"

namespace hf3fs::fuse {
class UserConfig {
 public:
  UserConfig() = default;
  void init(FuseConfig &config);
  Result<meta::Inode> setConfig(const char *key, const char *val, const meta::UserInfo &ui);
  Result<meta::Inode> lookupConfig(const char *key, const meta::UserInfo &ui);
  Result<meta::Inode> statConfig(meta::InodeId iid, const meta::UserInfo &ui);
  std::pair<std::shared_ptr<std::vector<meta::DirEntry>>, std::shared_ptr<std::vector<std::optional<meta::Inode>>>>
  listConfig(const meta::UserInfo &ui);

 public:
  const FuseConfig &getConfig(const meta::UserInfo &ui);

 public:
  const std::vector<std::string> systemKeys{"storage.net_client.rdma_control.max_concurrent_transmission",
                                            "periodic_sync.enable",
                                            "periodic_sync.interval",
                                            "periodic_sync.flush_write_buf",
                                            "io_worker_coros.hi",
                                            "io_worker_coros.lo",
                                            "max_jobs_per_ioring",
                                            "io_job_deq_timeout"};
  const std::vector<std::string> userKeys{"enable_read_cache",
                                          "readonly",
                                          "dryrun_bench_mode",
                                          "flush_on_stat",
                                          "sync_on_stat",
                                          "attr_timeout",
                                          "entry_timeout",
                                          "negative_timeout",
                                          "symlink_timeout"};

 private:
  Result<std::pair<bool, int>> parseKey(const char *key);
  meta::InodeId configIid(bool isGet, bool isSys, int kidx) {
    return meta::InodeId{(isGet ? meta::InodeId::getConf() : meta::InodeId::setConf()).u64() - 1 -
                         (isSys ? 0 : systemKeys.size()) - kidx};
  }

 private:
  FuseConfig *config_;
  struct LocalConfig {
    LocalConfig(const FuseConfig &globalConfig)
        : config(globalConfig) {}
    std::mutex mtx;
    FuseConfig config;
    std::vector<config::KeyValue> updatedItems;
  };
  std::unique_ptr<AtomicSharedPtrTable<LocalConfig>> configs_;
  std::mutex userMtx_;
  std::set<meta::Uid> users_;

 private:
  std::atomic<int> storageMaxConcXmit_;
};
}  // namespace hf3fs::fuse

#pragma once

#include "ICommonMgmtdClient.h"
#include "MgmtdClient.h"

namespace hf3fs::client {
template <typename Base>
class CommonMgmtdClient : public Base {
 public:
  explicit CommonMgmtdClient(std::shared_ptr<MgmtdClient> client)
      : client_(std::move(client)) {}

  CoTask<void> start(folly::Executor *backgroundExecutor = nullptr, bool startBackground = true) final {
    return client_->start(backgroundExecutor, startBackground);
  }

  CoTask<void> startBackgroundTasks() final { return client_->startBackgroundTasks(); }

  CoTask<void> stop() final { return client_->stop(); }

  std::shared_ptr<RoutingInfo> getRoutingInfo() final { return client_->getRoutingInfo(); }

  CoTryTask<void> refreshRoutingInfo(bool force) final { return client_->refreshRoutingInfo(force); }

  bool addRoutingInfoListener(String name, ICommonMgmtdClient::RoutingInfoListener listener) final {
    return client_->addRoutingInfoListener(std::move(name), std::move(listener));
  }

  bool removeRoutingInfoListener(std::string_view name) final { return client_->removeRoutingInfoListener(name); }

  CoTryTask<mgmtd::ListClientSessionsRsp> listClientSessions() final { return client_->listClientSessions(); }

  CoTryTask<std::optional<flat::ConfigInfo>> getConfig(flat::NodeType nodeType, flat::ConfigVersion version) final {
    return client_->getConfig(nodeType, version);
  }

  CoTryTask<std::vector<flat::TagPair>> getUniversalTags(const String &universalId) final {
    return client_->getUniversalTags(universalId);
  }

  CoTryTask<RHStringHashMap<flat::ConfigVersion>> getConfigVersions() final { return client_->getConfigVersions(); }

  CoTryTask<mgmtd::GetClientSessionRsp> getClientSession(const String &clientId) final {
    return client_->getClientSession(clientId);
  }

 protected:
  std::shared_ptr<MgmtdClient> client_;
};
}  // namespace hf3fs::client

#pragma once

#include <cstdint>
#include <fmt/format.h>

namespace hf3fs {

struct Varint64 {
  uint64_t value;

  Varint64() = default;
  Varint64(uint64_t value)
      : value(value) {}

  operator uint64_t &() { return value; }
  operator uint64_t() const { return value; }
};

}  // namespace hf3fs

FMT_BEGIN_NAMESPACE

template <>
struct formatter<hf3fs::Varint64> : formatter<uint64_t> {
  template <typename FormatContext>
  auto format(hf3fs::Varint64 val, FormatContext &ctx) const {
    return formatter<uint64_t>::format(static_cast<uint64_t>(val), ctx);
  }
};

FMT_END_NAMESPACE

#pragma once

#include <cstdint>
#include <limits>
#include <type_traits>

namespace hf3fs {
template <typename To, typename From>
inline constexpr bool isSafeConvertTo(From from) noexcept {
  static_assert(std::is_integral_v<From> && std::is_integral_v<To>);
  static_assert(sizeof(From) <= sizeof(uint64_t) && sizeof(To) <= sizeof(uint64_t));
  if constexpr (std::is_signed_v<From> == std::is_signed_v<To>) {
    return from <= std::numeric_limits<To>::max() && from >= std::numeric_limits<To>::min();
  } else if constexpr (std::is_signed_v<From>) {
    return from >= 0 && static_cast<uint64_t>(from) <= std::numeric_limits<To>::max();
  } else {
    return from <= static_cast<uint64_t>(std::numeric_limits<To>::max());
  }
}
}  // namespace hf3fs

#pragma once

#include <folly/Likely.h>
#include <gtest/gtest_prod.h>

#include "FDB.h"
#include "FDBTransaction.h"
#include "common/kv/IKVEngine.h"

namespace hf3fs {

namespace meta {
template <typename KV>
class MetaTestBase;
}

namespace kv {
class FDBKVEngine : public IKVEngine {
 public:
  FDBKVEngine(fdb::DB db)
      : db_(std::move(db)) {}

  std::unique_ptr<IReadOnlyTransaction> createReadonlyTransaction() override { return createReadWriteTransaction(); }

  std::unique_ptr<IReadWriteTransaction> createReadWriteTransaction() override {
    fdb::Transaction tr(db_);
    if (UNLIKELY(tr.error())) {
      return nullptr;
    }
    return std::make_unique<FDBTransaction>(std::move(tr));
  }

 private:
  template <typename KV>
  friend class hf3fs::meta::MetaTestBase;

  FRIEND_TEST(TestFDBTransaction, Readonly);

  void setReadonly(bool rdonly) { db_.readonly_ = rdonly; }

  fdb::DB db_;
};
}  // namespace kv

}  // namespace hf3fs

#pragma once

#include <functional>
#include <type_traits>

namespace hf3fs {
template <typename P, typename T>
concept UnaryPredicate = std::is_same_v<bool, std::invoke_result_t<P, const T &>>;

// Selector is an alias of `std::function<bool(const T &)>`.
// Declare it as a new class so we can override its logical operators.
template <typename T, UnaryPredicate<T> F>
class Selector {
 public:
  Selector(F &&f)
      : predicate_(std::move(f)) {}

  bool operator()(const T &t) const { return predicate_(t); }

 private:
  F predicate_;
};

template <typename T, typename U0, typename U1>
inline auto operator&&(Selector<T, U0> &&s0, Selector<T, U1> &&s1) {
  auto predicate = [s0 = std::move(s0), s1 = std::move(s1)](const T &t) { return s0(t) && s1(t); };
  return Selector<T, decltype(predicate)>(std::move(predicate));
}

template <typename T, typename U0, typename U1>
inline auto operator||(Selector<T, U0> &&s0, Selector<T, U1> &&s1) {
  auto predicate = [s0 = std::move(s0), s1 = std::move(s1)](const T &t) { return s0(t) || s1(t); };
  return Selector<T, decltype(predicate)>(std::move(predicate));
}

template <typename T, typename U>
inline auto operator!(Selector<T, U> &&s) {
  auto predicate = [s = std::move(s)](const T &t) { return !s(t); };
  return Selector<T, decltype(predicate)>(std::move(predicate));
}

template <typename T>
inline auto allAccept() {
  static constexpr auto s = [](const T &) { return true; };
  return Selector<T, decltype(s)>(s);
}

template <typename T>
inline auto allReject() {
  static constexpr auto s = [](const T &) { return false; };
  return Selector<T, decltype(s)>(s);
}

template <typename T, UnaryPredicate<T> F>
inline auto makeSelector(F &&f) {
  return Selector<T, F>(std::move(f));
}
}  // namespace hf3fs

#pragma once

#include "common/utils/UtcTime.h"
#include "fbs/mgmtd/TargetInfo.h"

namespace hf3fs::mgmtd {
template <typename T>
class WithTimestamp {
 public:
  WithTimestamp()
      : base_(),
        ts_(SteadyClock::now()) {}

  explicit WithTimestamp(T info)
      : base_(std::move(info)),
        ts_(SteadyClock::now()) {}

  WithTimestamp(T info, SteadyTime ts)
      : base_(std::move(info)),
        ts_(ts) {}

  T &base() { return base_; }
  const T &base() const { return base_; }

  SteadyTime ts() const { return ts_; }
  void updateTs() { ts_ = SteadyClock::now(); }
  void updateTs(SteadyTime ts) { ts_ = ts; }

 private:
  T base_;
  SteadyTime ts_;
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "common/monitor/Reporter.h"
#include "common/monitor/Sample.h"
#include "common/utils/ConfigBase.h"

namespace hf3fs::monitor {

class TaosClient : public Reporter {
 public:
  struct Config : ConfigBase<Config> {
    CONFIG_ITEM(host, "");
    CONFIG_ITEM(user, "");
    CONFIG_ITEM(passwd, "");
    CONFIG_ITEM(db, "");
    CONFIG_ITEM(cfg_dir, "/tmp");
    CONFIG_ITEM(log_dir, "/tmp");
  };

  TaosClient(const Config &config)
      : config_(config) {}
  ~TaosClient() override { stop(); }

  Result<Void> init() final;
  void stop();
  static void cleanUp();

  Result<Void> query(const std::string &sql);
  Result<Void> commit(const std::vector<Sample> &samples) final;

 protected:
  Result<Void> insert(std::string &buffer, std::vector<uintptr_t> &offsets);

 private:
  const Config &config_;
  void *taos_ = nullptr;
};

}  // namespace hf3fs::monitor

#pragma once

#include "MgmtdConfigFetcher.h"
#include "MgmtdLauncherConfig.h"
#include "common/net/Server.h"
#include "core/app/ServerAppConfig.h"
#include "core/app/ServerLauncher.h"
#include "fdb/HybridKvEngineConfig.h"
#include "mgmtd/service/MgmtdOperator.h"
#include "service/MgmtdConfig.h"

namespace hf3fs::mgmtd {

class MgmtdServer : public net::Server {
 public:
  static constexpr auto kName = "Mgmtd";
  static constexpr auto kNodeType = flat::NodeType::MGMTD;

  using AppConfig = core::ServerAppConfig;
  using LauncherConfig = MgmtdLauncherConfig;
  using RemoteConfigFetcher = MgmtdConfigFetcher;
  using Launcher = core::ServerLauncher<MgmtdServer>;

  using CommonConfig = ApplicationBase::Config;
  class Config : public ConfigBase<Config> {
    CONFIG_OBJ(base, net::Server::Config, [](net::Server::Config &c) {
      c.set_groups_length(2);
      c.groups(0).listener().set_listen_port(8000);
      c.groups(0).set_services({"Mgmtd"});

      c.groups(1).set_network_type(net::Address::TCP);
      c.groups(1).listener().set_listen_port(9000);
      c.groups(1).set_use_independent_thread_pool(true);
      c.groups(1).set_services({"Core"});
    });
    CONFIG_OBJ(service, MgmtdConfig);
  };

  explicit MgmtdServer(const Config &config);
  ~MgmtdServer() override;

  using net::Server::start;
  Result<Void> start(const flat::AppInfo &info, std::shared_ptr<kv::IKVEngine> kvEngine);

  Result<Void> beforeStart() final;

  Result<Void> afterStop() final;

 private:
  const Config &config_;

  std::shared_ptr<kv::IKVEngine> kvEngine_;
  std::unique_ptr<MgmtdOperator> mgmtdOperator_;

  friend class testing::MgmtdTestHelper;
};

}  // namespace hf3fs::mgmtd

#pragma once

#include <boost/algorithm/hex.hpp>
#include <boost/algorithm/string.hpp>
#include <fmt/format.h>
#include <span>

#include "Result.h"
#include "String.h"
#include "common/utils/MagicEnum.hpp"

namespace hf3fs {
template <typename T>
requires(std::is_enum_v<T>) inline String toString(T t) { return String(magic_enum::enum_name(t)); }

template <typename T>
requires(std::is_enum_v<T>) inline std::string_view toStringView(T t) { return magic_enum::enum_name(t); }

template <typename Container>
String toHexString(const Container &c) {
  String to;
  boost::algorithm::hex(std::begin(c), std::end(c), std::back_inserter(to));
  return to;
}

String fromHexString(std::string_view c);

auto splitAndTransform(std::string_view src, auto &&delimiterPredicte, auto &&transform) {
  using RetType = std::decay_t<decltype(transform(src))>;
  std::vector<std::string_view> slices;
  boost::split(slices, src, std::forward<decltype(delimiterPredicte)>(delimiterPredicte));
  std::vector<RetType> result;
  result.reserve(slices.size());
  for (auto slice : slices) {
    if (!slice.empty()) {
      result.push_back(transform(slice));
    }
  }
  return result;
}

template <typename T>
using StringMap = std::map<String, T, std::less<>>;

// return empty string if prefix is all of \xff
String getPrefixEnd(String prefix);

void encodeOrderPreservedString(String &buf, std::string_view key, bool mayContainNull);
Result<std::string_view> decodeOrderPreservedString(std::string_view key, String &result, bool mayContainNull);

String encodeOrderPreservedStrings(std::span<const String> keys, bool mayContainNull);
String encodeOrderPreservedStrings(std::initializer_list<std::string_view> keys, bool mayContainNull);
Result<std::vector<String>> decodeOrderPreservedStrings(std::string_view key, bool mayContainNull);
Result<Void> decodeOrderPreservedStrings(std::string_view key, bool mayContainNull, std::span<String> result);

}  // namespace hf3fs

#pragma once

#include <mutex>

#include "FileHelper.h"
#include "IWritableFile.h"

namespace hf3fs::logging {
class RotatingFile : public IWritableFile {
 public:
  struct Options {
    size_t maxFiles = 0;
    size_t maxFileSize = 0;
    bool rotateOnOpen = false;
  };

  RotatingFile(std::string path, const Options &options);

  bool ttyOutput() const override;
  std::string desc() const override;
  ssize_t writevFull(iovec *iov, int count) override;
  ssize_t writeFull(const void *buf, size_t count) override;
  void flush() override;

 private:
  size_t checkRotate(size_t size);
  void rotate();

  const std::string path_;
  const Options options_;

  mutable std::mutex mu_;
  FileHelper file_;
  size_t currentSize_;
};
}  // namespace hf3fs::logging

/* linenoise.h -- VERSION 1.0
 *
 * Guerrilla line editing library against the idea that a line editing lib
 * needs to be 20,000 lines of C code.
 *
 * See linenoise.c for more information.
 *
 * ------------------------------------------------------------------------
 *
 * Copyright (c) 2010-2023, Salvatore Sanfilippo <antirez at gmail dot com>
 * Copyright (c) 2010-2013, Pieter Noordhuis <pcnoordhuis at gmail dot com>
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *  *  Redistributions of source code must retain the above copyright
 *     notice, this list of conditions and the following disclaimer.
 *
 *  *  Redistributions in binary form must reproduce the above copyright
 *     notice, this list of conditions and the following disclaimer in the
 *     documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#ifndef __LINENOISE_H
#define __LINENOISE_H

#ifdef __cplusplus
extern "C" {
#endif

#include <stddef.h> /* For size_t. */

extern char *linenoiseEditMore;

/* The linenoiseState structure represents the state during line editing.
 * We pass this state to functions implementing specific editing
 * functionalities. */
struct linenoiseState {
  int in_completion;     /* The user pressed TAB and we are now in completion
                          * mode, so input is handled by completeLine(). */
  size_t completion_idx; /* Index of next completion to propose. */
  int ifd;               /* Terminal stdin file descriptor. */
  int ofd;               /* Terminal stdout file descriptor. */
  char *buf;             /* Edited line buffer. */
  size_t buflen;         /* Edited line buffer size. */
  const char *prompt;    /* Prompt to display. */
  size_t plen;           /* Prompt length. */
  size_t pos;            /* Current cursor position. */
  size_t oldpos;         /* Previous refresh cursor position. */
  size_t len;            /* Current edited line length. */
  size_t cols;           /* Number of columns in terminal. */
  size_t oldrows;        /* Rows used by last refrehsed line (multiline mode) */
  int history_index;     /* The history index we are currently editing. */
};

typedef struct linenoiseCompletions {
  size_t len;
  char **cvec;
} linenoiseCompletions;

/* Non blocking API. */
int linenoiseEditStart(struct linenoiseState *l,
                       int stdin_fd,
                       int stdout_fd,
                       char *buf,
                       size_t buflen,
                       const char *prompt);
char *linenoiseEditFeed(struct linenoiseState *l);
void linenoiseEditStop(struct linenoiseState *l);
void linenoiseHide(struct linenoiseState *l);
void linenoiseShow(struct linenoiseState *l);

/* Blocking API. */
char *linenoise(const char *prompt);
void linenoiseFree(void *ptr);

/* Completion API. */
typedef void(linenoiseCompletionCallback)(const char *, linenoiseCompletions *);
typedef char *(linenoiseHintsCallback)(const char *, int *color, int *bold);
typedef void(linenoiseFreeHintsCallback)(void *);
void linenoiseSetCompletionCallback(linenoiseCompletionCallback *);
void linenoiseSetHintsCallback(linenoiseHintsCallback *);
void linenoiseSetFreeHintsCallback(linenoiseFreeHintsCallback *);
void linenoiseAddCompletion(linenoiseCompletions *, const char *);

/* History API. */
int linenoiseHistoryAdd(const char *line);
int linenoiseHistorySetMaxLen(int len);
int linenoiseHistorySave(const char *filename);
int linenoiseHistoryLoad(const char *filename);

/* Other utilities. */
void linenoiseClearScreen(void);
void linenoiseSetMultiLine(int ml);
void linenoisePrintKeyCodes(void);
void linenoiseMaskModeEnable(void);
void linenoiseMaskModeDisable(void);

#ifdef __cplusplus
}
#endif

#endif /* __LINENOISE_H */

/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#pragma once

#include <folly/logging/AsyncLogWriter.h>
#include <vector>

#include "IWritableFile.h"

namespace hf3fs::logging {
/**
 * An implementation of `folly::AsyncLogWriter` that writes log messages into a
 * file.
 *
 * See `folly::AsyncLogWriter` for details on asynchronous IO.
 */
class AsyncFileWriter : public folly::AsyncLogWriter {
 public:
  /**
   * Construct an AsyncFileWriter that writes to the specified File object.
   */
  explicit AsyncFileWriter(std::shared_ptr<IWritableFile> file);

  ~AsyncFileWriter();

  /**
   * Returns true if the output steam is a tty.
   */
  bool ttyOutput() const override;

 private:
  void writeToFile(const std::vector<std::string> &ioQueue, size_t numDiscarded);

  void performIO(const std::vector<std::string> &ioQueue, size_t numDiscarded) override;

  std::string getNumDiscardedMsg(size_t numDiscarded);

  std::shared_ptr<IWritableFile> file_;
};
}  // namespace hf3fs::logging

#pragma once

#include "common/app/ApplicationBase.h"
#include "core/utils/ServiceOperation.h"
#include "fbs/core/service/Rpc.h"

namespace hf3fs::core {

struct RenderConfigOperation : ServiceOperationWithMetric<"CoreService", "RenderConfig", "op"> {
  RenderConfigReq req;

  explicit RenderConfigOperation(RenderConfigReq r)
      : req(std::move(r)) {}

  String toStringImpl() const final { return "RenderConfig"; }

  CoTryTask<RenderConfigRsp> handle() {
    auto res = ApplicationBase::renderConfig(req.configTemplate, req.testUpdate, req.isHotUpdate);
    CO_RETURN_ON_ERROR(res);
    auto &[content, updateRes] = *res;
    if (updateRes) {
      co_return RenderConfigRsp::create(std::move(content), Status(StatusCode::kOK), std::move(*updateRes));
    } else {
      co_return RenderConfigRsp::create(std::move(content), std::move(updateRes.error()));
    }
  }
};

}  // namespace hf3fs::core

#pragma once

#include <array>
#include <folly/hash/Hash.h>
#include <mutex>

#include "RobinHoodUtils.h"

namespace hf3fs {

template <class T, std::size_t N>
class Shards {
 public:
  auto position(auto &&...args) { return folly::hash::hash_combine_generic(RobinHoodHasher{}, args...) % N; }

  auto withLockAt(auto &&f, std::size_t pos) {
    auto lock = std::unique_lock(locks_[pos]);
    return f(array_[pos]);
  }

  auto withLock(auto &&f, auto &&...args) { return withLockAt(f, position(args...)); }

  void iterate(auto &&f) {
    for (std::size_t idx = 0; idx < N; ++idx) {
      withLockAt(f, idx);
    }
  }

 private:
  std::array<std::mutex, N> locks_;
  std::array<T, N> array_;
};

}  // namespace hf3fs

#pragma once

#include "IMgmtdClientForClient.h"
#include "IMgmtdClientForServer.h"
#include "RoutingInfo.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/Duration.h"
#include "stubs/common/IStubFactory.h"
#include "stubs/mgmtd/IMgmtdServiceStub.h"

namespace hf3fs::client {
class MgmtdClient {
 public:
  struct Config : public ConfigBase<Config> {
    CONFIG_ITEM(mgmtd_server_addresses, std::vector<net::Address>{});
    CONFIG_ITEM(work_queue_size, 100, ConfigCheckers::checkPositive);
    CONFIG_ITEM(network_type, std::optional<net::Address::Type>{});

    CONFIG_HOT_UPDATED_ITEM(enable_auto_refresh, true);
    CONFIG_HOT_UPDATED_ITEM(auto_refresh_interval, 10_s);
    CONFIG_HOT_UPDATED_ITEM(enable_auto_heartbeat, true);
    CONFIG_HOT_UPDATED_ITEM(auto_heartbeat_interval, 10_s);
    CONFIG_HOT_UPDATED_ITEM(enable_auto_extend_client_session, true);
    CONFIG_HOT_UPDATED_ITEM(auto_extend_client_session_interval, 10_s);
    CONFIG_HOT_UPDATED_ITEM(accept_incomplete_routing_info_during_mgmtd_bootstrapping, true);
  };

  using MgmtdStub = mgmtd::IMgmtdServiceStub;
  using MgmtdStubFactory = stubs::IStubFactory<MgmtdStub>;
  using RoutingInfoListener = ICommonMgmtdClient::RoutingInfoListener;
  using ConfigListener = IMgmtdClientForServer::ConfigListener;
  using HeartbeatPayload = IMgmtdClientForServer::HeartbeatPayload;
  using ClientSessionPayload = IMgmtdClientForClient::ClientSessionPayload;

  MgmtdClient(String clusterId, std::unique_ptr<MgmtdStubFactory> stubFactory, const Config &config);
  ~MgmtdClient();

  // backgroundExecutor == nullptr means using the current executor
  CoTask<void> start(folly::Executor *backgroundExecutor = nullptr, bool startBackground = true);

  CoTask<void> startBackgroundTasks();

  CoTask<void> stop();

  std::shared_ptr<RoutingInfo> getRoutingInfo();

  CoTryTask<void> refreshRoutingInfo(bool force);

  CoTryTask<mgmtd::HeartbeatRsp> heartbeat();

  Result<Void> triggerHeartbeat();

  void setAppInfoForHeartbeat(flat::AppInfo info);

  bool addRoutingInfoListener(String name, RoutingInfoListener listener);

  bool removeRoutingInfoListener(std::string_view name);

  void setServerConfigListener(ConfigListener listener);

  void setClientConfigListener(ConfigListener listener);

  void updateHeartbeatPayload(HeartbeatPayload payload);

  CoTryTask<mgmtd::SetChainsRsp> setChains(const flat::UserInfo &userInfo,
                                           const std::vector<flat::ChainSetting> &chains);

  CoTryTask<mgmtd::SetChainTableRsp> setChainTable(const flat::UserInfo &userInfo,
                                                   flat::ChainTableId tableId,
                                                   const std::vector<flat::ChainId> &chains,
                                                   const String &desc);

  void setClientSessionPayload(ClientSessionPayload payload);

  CoTryTask<void> extendClientSession();

  CoTryTask<mgmtd::ListClientSessionsRsp> listClientSessions();

  CoTryTask<flat::ConfigVersion> setConfig(const flat::UserInfo &userInfo,
                                           flat::NodeType nodeType,
                                           const String &content,
                                           const String &desc);

  CoTryTask<std::optional<flat::ConfigInfo>> getConfig(flat::NodeType nodeType, flat::ConfigVersion version);

  CoTryTask<RHStringHashMap<flat::ConfigVersion>> getConfigVersions();

  CoTryTask<void> enableNode(const flat::UserInfo &userInfo, flat::NodeId nodeId);

  CoTryTask<void> disableNode(const flat::UserInfo &userInfo, flat::NodeId nodeId);

  CoTryTask<void> registerNode(const flat::UserInfo &userInfo, flat::NodeId nodeId, flat::NodeType type);

  CoTryTask<void> unregisterNode(const flat::UserInfo &userInfo, flat::NodeId nodeId);

  CoTryTask<flat::NodeInfo> setNodeTags(const flat::UserInfo &userInfo,
                                        flat::NodeId nodeId,
                                        const std::vector<flat::TagPair> &tags,
                                        flat::SetTagMode mode);

  CoTryTask<std::vector<flat::TagPair>> setUniversalTags(const flat::UserInfo &userInfo,
                                                         const String &universalId,
                                                         const std::vector<flat::TagPair> &tags,
                                                         flat::SetTagMode mode);

  CoTryTask<std::vector<flat::TagPair>> getUniversalTags(const String &universalId);

  CoTryTask<mgmtd::GetClientSessionRsp> getClientSession(const String &clientId);

  CoTryTask<flat::ChainInfo> rotateLastSrv(const flat::UserInfo &userInfo, flat::ChainId chainId);

  CoTryTask<flat::ChainInfo> rotateAsPreferredOrder(const flat::UserInfo &userInfo, flat::ChainId chainId);

  CoTryTask<flat::ChainInfo> setPreferredTargetOrder(const flat::UserInfo &userInfo,
                                                     flat::ChainId chainId,
                                                     const std::vector<flat::TargetId> &preferredTargetOrder);

  CoTryTask<mgmtd::ListOrphanTargetsRsp> listOrphanTargets();

  CoTryTask<flat::ChainInfo> updateChain(const flat::UserInfo &userInfo,
                                         flat::ChainId cid,
                                         flat::TargetId tid,
                                         mgmtd::UpdateChainReq::Mode mode);

  struct Impl;

 private:
  std::unique_ptr<Impl> impl_;
};
}  // namespace hf3fs::client

#pragma once

#include "RoutingInfo.h"
#include "fbs/mgmtd/Rpc.h"

namespace hf3fs::client {
// ICommonMgmtdClient provides the common facilities:
// 1. start and stop
// 2. RoutingInfo
class ICommonMgmtdClient {
 public:
  virtual ~ICommonMgmtdClient() = default;

  virtual CoTask<void> start(folly::Executor *backgroundExecutor = nullptr, bool startBackground = true) { co_return; }

  virtual CoTask<void> startBackgroundTasks() { co_return; }

  virtual CoTask<void> stop() { co_return; }

  virtual std::shared_ptr<RoutingInfo> getRoutingInfo() = 0;

  // manual calls
  virtual CoTryTask<void> refreshRoutingInfo(bool force) = 0;

  using RoutingInfoListener = std::function<void(std::shared_ptr<RoutingInfo>)>;
  virtual bool addRoutingInfoListener(String name, RoutingInfoListener listener) = 0;
  virtual bool removeRoutingInfoListener(std::string_view name) = 0;

  virtual CoTryTask<mgmtd::ListClientSessionsRsp> listClientSessions() = 0;

  virtual CoTryTask<std::optional<flat::ConfigInfo>> getConfig(flat::NodeType nodeType,
                                                               flat::ConfigVersion version) = 0;

  virtual CoTryTask<std::vector<flat::TagPair>> getUniversalTags(const String &universalId) = 0;

  virtual CoTryTask<RHStringHashMap<flat::ConfigVersion>> getConfigVersions() = 0;

  virtual CoTryTask<mgmtd::GetClientSessionRsp> getClientSession(const String &clientId) = 0;
};
}  // namespace hf3fs::client

//-----------------------------------------------------------------------------
// MurmurHash3 was written by Austin Appleby, and is placed in the public
// domain. The author hereby disclaims copyright to this source code.

#ifndef _MURMURHASH3_H_
#define _MURMURHASH3_H_

//-----------------------------------------------------------------------------
// Platform-specific functions and macros

// Microsoft Visual Studio

#if defined(_MSC_VER) && (_MSC_VER < 1600)

typedef unsigned char uint8_t;
typedef unsigned int uint32_t;
typedef unsigned __int64 uint64_t;

// Other compilers

#else  // defined(_MSC_VER)

#include <stdint.h>

#endif  // !defined(_MSC_VER)

//-----------------------------------------------------------------------------

void MurmurHash3_x86_32(const void *key, int len, uint32_t seed, void *out);

void MurmurHash3_x86_128(const void *key, int len, uint32_t seed, void *out);

void MurmurHash3_x64_128(const void *key, int len, uint32_t seed, void *out);

//-----------------------------------------------------------------------------

#endif  // _MURMURHASH3_H_

#pragma once

#include "MgmtdTypes.h"
#include "common/serde/SerdeComparisons.h"
#include "common/serde/SerdeHelper.h"

namespace hf3fs::flat {
struct LocalTargetInfo : public serde::SerdeHelper<LocalTargetInfo> {
  bool operator==(const LocalTargetInfo &other) const { return serde::equals(*this, other); }

  SERDE_STRUCT_FIELD(targetId, TargetId(0));
  SERDE_STRUCT_FIELD(localState, LocalTargetState(LocalTargetState::INVALID));
  SERDE_STRUCT_FIELD(diskIndex, std::optional<uint32_t>{});
  SERDE_STRUCT_FIELD(usedSize, uint64_t{});
  SERDE_STRUCT_FIELD(chainVersion, ChainVersion{});
  SERDE_STRUCT_FIELD(lowSpace, false);
};
}  // namespace hf3fs::flat

#pragma once

#include <folly/logging/xlog.h>

#include "Semaphore.h"

namespace hf3fs {
class SemaphoreGuard : public folly::NonCopyableNonMovable {
 public:
  SemaphoreGuard(Semaphore &semaphore)
      : semaphore_(semaphore),
        acquired_(false) {}

  ~SemaphoreGuard() {
    if (acquired_) semaphore_.signal();
  }

  void wait() {
    semaphore_.wait();
    acquired_ = true;
  }

  bool tryWait() {
    acquired_ = semaphore_.try_wait();
    return acquired_;
  }

  folly::coro::Task<void> coWait() {
    acquired_ = semaphore_.try_wait();
    if (acquired_) co_return;

    XLOGF(DBG5, "Cannot acquire semaphore {} right now, waiting", fmt::ptr(&semaphore_));
    co_await semaphore_.co_wait();
    acquired_ = true;
  }

 private:
  Semaphore &semaphore_;
  bool acquired_;
};
}  // namespace hf3fs

#pragma once

#include <folly/io/async/Request.h>
namespace hf3fs {

class RequestInfo : public folly::RequestData {
 public:
  static constexpr const char *kTokenName = "hf3fs::RequestInfo";
  static folly::RequestToken const &token() {
    static folly::RequestToken const token(kTokenName);
    return token;
  }

  static inline RequestInfo *get() {
    return dynamic_cast<RequestInfo *>(folly::RequestContext::get()->getContextData(token()));
  }

  bool hasCallback() override { return false; }

  virtual std::string describe() const = 0;
  virtual bool canceled() const = 0;
};

}  // namespace hf3fs

#pragma once

#include "common/net/Server.h"
#include "monitor_collector/service/MonitorCollectorService.h"

namespace hf3fs::monitor {

class MonitorCollectorServer : public net::Server {
 public:
  static constexpr auto kName = "MonitorCollector";

  struct Config : public ConfigBase<Config> {
    CONFIG_OBJ(base, net::Server::Config, [](net::Server::Config &c) {
      c.set_groups_length(1);
      c.groups(0).listener().set_listen_port(10000);
      c.groups(0).set_network_type(hf3fs::net::Address::TCP);
      c.groups(0).set_services({"MonitorCollector"});
    });
    CONFIG_OBJ(monitor_collector, monitor::MonitorCollectorService::Config);
  };

  MonitorCollectorServer(const Config &config);
  ~MonitorCollectorServer() override;

  // set up monitor collector server.
  Result<Void> beforeStart() final;

  // tear down monitor collector server.
  Result<Void> beforeStop() final;

 private:
  std::unique_ptr<MonitorCollectorOperator> monitorCollectorOperator_;
  const Config &config_;
};

}  // namespace hf3fs::monitor

#pragma once

#include "common/app/AppInfo.h"
#include "common/kv/IKVEngine.h"
#include "common/utils/CPUExecutorGroup.h"
#include "stubs/common/IStubFactory.h"
#include "stubs/mgmtd/IMgmtdServiceStub.h"

namespace hf3fs::core {
class ServerEnv {
 public:
  const flat::AppInfo &appInfo() const { return appInfo_; }

  void setAppInfo(flat::AppInfo appInfo);

  const std::shared_ptr<kv::IKVEngine> &kvEngine() const { return kvEngine_; }

  void setKvEngine(std::shared_ptr<kv::IKVEngine> engine);

  using MgmtdStubFactory = stubs::IStubFactory<mgmtd::IMgmtdServiceStub>;
  const std::shared_ptr<MgmtdStubFactory> &mgmtdStubFactory() const { return mgmtdStubFactory_; }

  void setMgmtdStubFactory(std::shared_ptr<MgmtdStubFactory> factory);

  using UtcTimeGenerator = std::function<UtcTime()>;
  const UtcTimeGenerator &utcTimeGenerator() const { return utcTimeGenerator_; }

  void setUtcTimeGenerator(UtcTimeGenerator generator);

  CPUExecutorGroup *backgroundExecutor() const { return backgroundExecutor_; }

  void setBackgroundExecutor(CPUExecutorGroup *executor);

  using ConfigUpdater = std::function<Result<Void>(const String &, const String &)>;
  const ConfigUpdater &configUpdater() const { return configUpdater_; }

  void setConfigUpdater(ConfigUpdater updater);

  using ConfigValidater = std::function<Result<Void>(const String &, const String &)>;
  const ConfigValidater &configValidater() const { return configValidater_; }

  void setConfigValidater(ConfigValidater validater);

 private:
  flat::AppInfo appInfo_;
  std::shared_ptr<kv::IKVEngine> kvEngine_;
  std::shared_ptr<MgmtdStubFactory> mgmtdStubFactory_;
  UtcTimeGenerator utcTimeGenerator_ = &UtcClock::now;
  CPUExecutorGroup *backgroundExecutor_ = nullptr;
  ConfigUpdater configUpdater_;
  ConfigValidater configValidater_;
};
}  // namespace hf3fs::core

#pragma once

#include "CommonMgmtdClient.h"
#include "IMgmtdClientForAdmin.h"

namespace hf3fs::client {
class MgmtdClientForAdmin : public CommonMgmtdClient<IMgmtdClientForAdmin> {
 public:
  struct Config : MgmtdClient::Config {
    Config() {
      set_enable_auto_refresh(false);
      set_enable_auto_heartbeat(false);
      set_enable_auto_extend_client_session(false);
      set_accept_incomplete_routing_info_during_mgmtd_bootstrapping(true);
    }
  };

  explicit MgmtdClientForAdmin(std::shared_ptr<MgmtdClient> client)
      : CommonMgmtdClient(std::move(client)) {}

  MgmtdClientForAdmin(String clusterId,
                      std::unique_ptr<MgmtdClient::MgmtdStubFactory> stubFactory,
                      const Config &config)
      : CommonMgmtdClient(std::make_shared<MgmtdClient>(std::move(clusterId), std::move(stubFactory), config)) {}

  CoTryTask<mgmtd::SetChainsRsp> setChains(const flat::UserInfo &userInfo,
                                           const std::vector<flat::ChainSetting> &chains) final {
    return client_->setChains(userInfo, chains);
  }

  CoTryTask<mgmtd::SetChainTableRsp> setChainTable(const flat::UserInfo &userInfo,
                                                   flat::ChainTableId tableId,
                                                   const std::vector<flat::ChainId> &chains,
                                                   const String &desc) final {
    return client_->setChainTable(userInfo, tableId, chains, desc);
  }

  CoTryTask<flat::ConfigVersion> setConfig(const flat::UserInfo &userInfo,
                                           flat::NodeType nodeType,
                                           const String &content,
                                           const String &desc) final {
    return client_->setConfig(userInfo, nodeType, content, desc);
  }

  CoTryTask<void> enableNode(const flat::UserInfo &userInfo, flat::NodeId nodeId) final {
    return client_->enableNode(userInfo, nodeId);
  }

  CoTryTask<void> disableNode(const flat::UserInfo &userInfo, flat::NodeId nodeId) final {
    return client_->disableNode(userInfo, nodeId);
  }

  CoTryTask<void> registerNode(const flat::UserInfo &userInfo, flat::NodeId nodeId, flat::NodeType type) final {
    return client_->registerNode(userInfo, nodeId, type);
  }

  CoTryTask<flat::NodeInfo> setNodeTags(const flat::UserInfo &userInfo,
                                        flat::NodeId nodeId,
                                        const std::vector<flat::TagPair> &tags,
                                        flat::SetTagMode mode) final {
    return client_->setNodeTags(userInfo, nodeId, tags, mode);
  }

  CoTryTask<void> unregisterNode(const flat::UserInfo &userInfo, flat::NodeId nodeId) final {
    return client_->unregisterNode(userInfo, nodeId);
  }

  CoTryTask<std::vector<flat::TagPair>> setUniversalTags(const flat::UserInfo &userInfo,
                                                         const String &universalId,
                                                         const std::vector<flat::TagPair> &tags,
                                                         flat::SetTagMode mode) final {
    return client_->setUniversalTags(userInfo, universalId, tags, mode);
  }

  CoTryTask<flat::ChainInfo> rotateLastSrv(const flat::UserInfo &userInfo, flat::ChainId chainId) final {
    return client_->rotateLastSrv(userInfo, chainId);
  }

  CoTryTask<flat::ChainInfo> rotateAsPreferredOrder(const flat::UserInfo &userInfo, flat::ChainId chainId) final {
    return client_->rotateAsPreferredOrder(userInfo, chainId);
  }

  CoTryTask<flat::ChainInfo> setPreferredTargetOrder(const flat::UserInfo &userInfo,
                                                     flat::ChainId chainId,
                                                     const std::vector<flat::TargetId> &preferredTargetOrder) final {
    return client_->setPreferredTargetOrder(userInfo, chainId, preferredTargetOrder);
  }

  CoTryTask<flat::ChainInfo> updateChain(const flat::UserInfo &userInfo,
                                         flat::ChainId cid,
                                         flat::TargetId tid,
                                         mgmtd::UpdateChainReq::Mode mode) final {
    return client_->updateChain(userInfo, cid, tid, mode);
  }

  CoTryTask<mgmtd::ListOrphanTargetsRsp> listOrphanTargets() final { return client_->listOrphanTargets(); }
};
}  // namespace hf3fs::client

#pragma once

#include "ChainTargetInfo.h"
#include "LocalTargetInfo.h"
#include "common/serde/SerdeComparisons.h"

namespace hf3fs::flat {
struct TargetInfo : public serde::SerdeHelper<TargetInfo> {
  bool operator==(const TargetInfo &other) const { return serde::equals(*this, other); }

  SERDE_STRUCT_FIELD(targetId, TargetId(0));
  SERDE_STRUCT_FIELD(publicState, PublicTargetState(PublicTargetState::INVALID));
  SERDE_STRUCT_FIELD(localState, LocalTargetState(LocalTargetState::INVALID));
  SERDE_STRUCT_FIELD(chainId, ChainId(0));
  SERDE_STRUCT_FIELD(nodeId, std::optional<NodeId>{});
  SERDE_STRUCT_FIELD(diskIndex, std::optional<uint32_t>{});
  SERDE_STRUCT_FIELD(usedSize, uint64_t{});
};
}  // namespace hf3fs::flat

#pragma once

#include <fmt/core.h>
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <folly/hash/Hash.h>
#include <folly/io/async/EventBase.h>
#include <folly/io/coro/ServerSocket.h>
#include <folly/io/coro/Transport.h>
#include <folly/logging/xlog.h>
#include <folly/net/NetworkSocket.h>
#include <netinet/in.h>
#include <sys/socket.h>

#include "common/utils/Address.h"
#include "common/utils/UtcTime.h"

namespace hf3fs::net {

using namespace std::chrono_literals;

using EventBase = folly::EventBase;

using IOBufQueue = folly::IOBufQueue;
using IOBuf = folly::IOBuf;
using IOBufPtr = std::unique_ptr<folly::IOBuf>;

using NamedThreadFactory = folly::NamedThreadFactory;
using CPUThreadPoolExecutor = folly::CPUThreadPoolExecutor;

template <size_t N>
struct MethodName {
  constexpr MethodName(const char (&str)[N]) { std::copy_n(str, N, string); }
  std::string str() const { return {string, N - 1}; }
  constexpr std::string_view string_view() const { return {string, N - 1}; }
  char string[N];
};

}  // namespace hf3fs::net

#pragma once

#include "common/monitor/LogReporter.h"
#include "common/monitor/Reporter.h"
#include "common/monitor/Sample.h"
#include "common/net/Client.h"
#include "common/utils/Address.h"
#include "common/utils/ConfigBase.h"

namespace hf3fs::monitor {

class MonitorCollectorClient : public Reporter {
 public:
  struct Config : ConfigBase<Config> {
    CONFIG_ITEM(remote_ip, "");  // format: 127.0.0.1:10000
    CONFIG_OBJ(client, net::Client::Config);
  };

  MonitorCollectorClient(const Config &config)
      : config_(config) {}
  ~MonitorCollectorClient() override { stop(); }

  Result<Void> init() final;
  void stop();

  Result<Void> commit(const std::vector<Sample> &samples) final;

 private:
  const Config &config_;
  std::unique_ptr<net::Client> client_;
  std::unique_ptr<serde::ClientContext> ctx_;
};

}  // namespace hf3fs::monitor

#pragma once

#include <map>
#include <string>
#include <sys/resource.h>

#include "common/serde/Serde.h"
#include "common/utils/Result.h"

namespace hf3fs {

struct SysResource {
  static Result<std::string> hostname(bool physicalMachineName = true);

  static uint32_t pid();

  static Result<Void> increaseProcessFDLimit(uint64_t limit);

  static Result<std::string> exec(std::string_view cmd);

  struct DiskInfo {
    SERDE_STRUCT_FIELD(deviceId, uint32_t{});
    SERDE_STRUCT_FIELD(uuid, std::string{});
    SERDE_STRUCT_FIELD(manufacturer, std::string{});
    SERDE_STRUCT_FIELD(devicePath, Path{});
    SERDE_STRUCT_FIELD(mountPath, Path{});
  };
  static Result<std::vector<DiskInfo>> scanDiskInfo();

  static Result<std::map<unsigned long, std::string>> fileSystemUUID(bool force = false);

  static std::map<String, String> getAllEnvs(std::string_view prefix = {});

  static Result<rlimit> getRlimit(__rlimit_resource_t resource);

  static int64_t getOpenedFDCount();
};

}  // namespace hf3fs

#pragma once

#include "analytics/StructuredTraceLog.h"
#include "client/storage/StorageClient.h"
#include "common/kv/TransactionRetry.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/CoroutinesPool.h"
#include "common/utils/Duration.h"
#include "common/utils/PriorityCoroutinePool.h"
#include "common/utils/Size.h"
#include "core/user/UserCache.h"
#include "meta/components/Distributor.h"
#include "meta/components/Forward.h"
#include "meta/components/SessionManager.h"
#include "meta/event/Event.h"
#include "meta/store/Utils.h"

namespace hf3fs::meta::server {

using kv::TransactionRetry;

struct GcConfig : ConfigBase<GcConfig> {
  CONFIG_HOT_UPDATED_ITEM(enable, true);
  CONFIG_HOT_UPDATED_ITEM(scan_interval, 200_ms);
  CONFIG_HOT_UPDATED_ITEM(scan_batch, 4096);
  CONFIG_HOT_UPDATED_ITEM(remove_chunks_batch_size, 32);
  CONFIG_HOT_UPDATED_ITEM(gc_file_delay, 5_min);
  CONFIG_HOT_UPDATED_ITEM(gc_file_concurrent, 32ul);
  CONFIG_HOT_UPDATED_ITEM(gc_directory_delay, 0_s);
  CONFIG_HOT_UPDATED_ITEM(gc_directory_concurrent, 4ul);
  CONFIG_HOT_UPDATED_ITEM(gc_directory_entry_batch, 32ul);
  CONFIG_HOT_UPDATED_ITEM(gc_directory_entry_concurrent, 4ul);
  CONFIG_HOT_UPDATED_ITEM(retry_delay, 10_min);
  // disable gc delay if free space is below 5%
  CONFIG_HOT_UPDATED_ITEM(gc_delay_free_space_threshold, 5);
  CONFIG_HOT_UPDATED_ITEM(check_session, true);
  CONFIG_HOT_UPDATED_ITEM(distributed_gc, true);  // random select a GC directory
  CONFIG_HOT_UPDATED_ITEM(txn_low_priority, false);

  // small file or large file
  CONFIG_HOT_UPDATED_ITEM(small_file_chunks, (uint64_t)32);
  CONFIG_HOT_UPDATED_ITEM(large_file_chunks, (uint64_t)128);

  CONFIG_HOT_UPDATED_ITEM(recursive_perm_check, true);

  CONFIG_OBJ(workers, PriorityCoroutinePoolConfig, [](auto &c) {
    c.set_coroutines_num(8);
    c.set_queue_size(1024);
  });
  CONFIG_OBJ(retry_remove_chunks, storage::client::RetryOptions, [](auto &c) {
    c.set_init_wait_time(10_s);
    c.set_max_wait_time(10_s);
    c.set_max_retry_time(30_s);
  });
};

struct Config : ConfigBase<Config> {
  CONFIG_HOT_UPDATED_ITEM(readonly, false);
  CONFIG_HOT_UPDATED_ITEM(authenticate, false);
  CONFIG_HOT_UPDATED_ITEM(grv_cache, false);

  CONFIG_OBJ(gc, GcConfig);
  CONFIG_OBJ(session_manager, SessionManager::Config);
  CONFIG_OBJ(distributor, Distributor::Config);
  CONFIG_OBJ(forward, Forward::Config);
  CONFIG_OBJ(event_trace_log, analytics::StructuredTraceLog<MetaEventTrace>::Config);

  CONFIG_HOT_UPDATED_ITEM(max_symlink_depth, 4L, ConfigCheckers::checkPositive);
  CONFIG_HOT_UPDATED_ITEM(max_symlink_count, 10L, ConfigCheckers::checkPositive);
  CONFIG_HOT_UPDATED_ITEM(max_directory_depth, 64L, ConfigCheckers::checkPositive);
  CONFIG_HOT_UPDATED_ITEM(acl_cache_time, 15_s);
  CONFIG_HOT_UPDATED_ITEM(list_default_limit, 128);
  CONFIG_HOT_UPDATED_ITEM(sync_on_prune_session, false);
  CONFIG_HOT_UPDATED_ITEM(max_remove_chunks_per_request, 32u, ConfigCheckers::checkPositive);
  CONFIG_HOT_UPDATED_ITEM(allow_stat_deleted_inodes, true);
  CONFIG_HOT_UPDATED_ITEM(ignore_length_hint, false);
  CONFIG_HOT_UPDATED_ITEM(time_granularity, 1_s);
  CONFIG_HOT_UPDATED_ITEM(dynamic_stripe, false);
  CONFIG_HOT_UPDATED_ITEM(dynamic_stripe_initial, 16u, ConfigCheckers::checkPositive);
  CONFIG_HOT_UPDATED_ITEM(dynamic_stripe_growth, 2u);
  CONFIG_HOT_UPDATED_ITEM(batch_stat_concurrent, 8u);
  CONFIG_HOT_UPDATED_ITEM(batch_stat_by_path_concurrent, 4u);
  CONFIG_HOT_UPDATED_ITEM(max_batch_operations, 4096u);
  CONFIG_HOT_UPDATED_ITEM(enable_new_chunk_engine, false);
  CONFIG_HOT_UPDATED_ITEM(allow_owner_change_immutable, false);

  // deperated
  CONFIG_HOT_UPDATED_ITEM(check_file_hole, false);
  CONFIG_OBJ(background_hole_checker, CoroutinesPoolBase::Config, [](CoroutinesPoolBase::Config &c) {
    c.set_coroutines_num(16);
    c.set_queue_size(4096);
  });

  CONFIG_HOT_UPDATED_ITEM(inodeId_check_unique, true);
  CONFIG_HOT_UPDATED_ITEM(inodeId_abort_on_duplicate, false);

  // replace file with new inode on O_TRUNC
  CONFIG_HOT_UPDATED_ITEM(otrunc_replace_file, true);
  CONFIG_HOT_UPDATED_ITEM(otrunc_replace_file_threshold, 1_GB);

  // statfs
  CONFIG_HOT_UPDATED_ITEM(statfs_cache_time, 60_s);
  CONFIG_HOT_UPDATED_ITEM(statfs_update_interval, 5_s);
  CONFIG_HOT_UPDATED_ITEM(statfs_space_imbalance_threshold, 5);

  // iflags
  CONFIG_HOT_UPDATED_ITEM(iflags_chain_allocation, false);
  CONFIG_HOT_UPDATED_ITEM(iflags_chunk_engine, true);

  // recursive remove
  CONFIG_HOT_UPDATED_ITEM(recursive_remove_check_owner, true);
  CONFIG_HOT_UPDATED_ITEM(recursive_remove_perm_check, (size_t)1024);
  CONFIG_HOT_UPDATED_ITEM(allow_directly_move_to_trash, false);

  // idempotent operation
  CONFIG_HOT_UPDATED_ITEM(idempotent_record_expire, 30_min);
  CONFIG_HOT_UPDATED_ITEM(idempotent_record_clean, 1_min);
  CONFIG_HOT_UPDATED_ITEM(idempotent_remove, true);
  CONFIG_HOT_UPDATED_ITEM(idempotent_rename, false);

  CONFIG_HOT_UPDATED_ITEM(operation_timeout, 5_s);

  CONFIG_OBJ(retry_transaction, TransactionRetry);
  CONFIG_OBJ(retry_remove_chunks, storage::client::RetryOptions, [](auto &c) {
    c.set_init_wait_time(10_s);
    c.set_max_wait_time(10_s);
    c.set_max_retry_time(30_s);
  });

  CONFIG_OBJ(user_cache, core::UserCache::Config);
};

}  // namespace hf3fs::meta::server

#pragma once

#include "common/utils/UtcTime.h"

#define CO_INVOKE_OP(NORMAL_PATH_LOG_LEVEL, op, peer, ...)                          \
  LOG_OP_##NORMAL_PATH_LOG_LEVEL(op, "start execution. from:{}", peer);             \
  auto guard = op.startRecord();                                                    \
  auto result = co_await op.handle(__VA_ARGS__);                                    \
  if (result.hasError()) {                                                          \
    guard.reportWithCode(result.error().code());                                    \
    LOG_OP_ERR(op, "failed: {}. latency: {}", result.error(), *guard.latency());    \
    CO_RETURN_ERROR(result);                                                        \
  } else {                                                                          \
    guard.reportWithCode(StatusCode::kOK);                                          \
    LOG_OP_##NORMAL_PATH_LOG_LEVEL(op, "succeeded. latency: {}", *guard.latency()); \
    co_return result;                                                               \
  }

#define CO_INVOKE_OP_INFO(...) CO_INVOKE_OP(INFO, __VA_ARGS__)
#define CO_INVOKE_OP_DBG(...) CO_INVOKE_OP(DBG, __VA_ARGS__)

#pragma once

#include <atomic>
#include <condition_variable>
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <mutex>

#include "common/utils/ConfigBase.h"
#include "storage/service/TargetMap.h"

namespace hf3fs::storage {

struct Components;

class CheckWorker {
 public:
  class Config : public ConfigBase<Config> {
    CONFIG_HOT_UPDATED_ITEM(update_target_size_interval, 10_s);
    CONFIG_HOT_UPDATED_ITEM(emergency_recycling_ratio, 0.95);
    CONFIG_HOT_UPDATED_ITEM(disk_low_space_threshold, 0.96);
    CONFIG_HOT_UPDATED_ITEM(disk_reject_create_chunk_threshold, 0.98);
  };

  CheckWorker(const Config &config, Components &components);

  // start check worker.
  Result<Void> start(const std::vector<Path> &targetPaths, const std::vector<std::string> &manufacturers);

  // stop check worker. End all tasks immediately.
  Result<Void> stopAndJoin();

  // check hf3fs path writable or not.
  static bool checkWritable(const Path &path);

 protected:
  void loop(const std::vector<Path> &targetPaths, const std::vector<std::string> &manufacturers);

 private:
  ConstructLog<"storage::CheckWorker"> constructLog_;
  const Config &config_;
  Components &components_;
  folly::CPUThreadPoolExecutor executors_;

  std::mutex mutex_;
  std::condition_variable cond_;
  std::atomic<bool> stopping_ = false;
  std::atomic<bool> started_ = false;
  std::atomic<bool> stopped_ = false;
};

}  // namespace hf3fs::storage

#pragma once
#include <cstddef>

#define GET_MEMORY_ALLOCATOR_FUNC_NAME "getMemoryAllocator"

namespace hf3fs::memory {

class MemoryAllocatorInterface {
 public:
  virtual ~MemoryAllocatorInterface() = default;
  virtual void *allocate(size_t size) = 0;
  virtual void deallocate(void *mem) = 0;
  virtual void *memalign(size_t alignment, size_t size) = 0;
  virtual void logstatus(char *buf, size_t size) = 0;
  virtual bool profiling(bool active, const char *prefix) = 0;
};

using GetMemoryAllocatorFunc = MemoryAllocatorInterface *(*)();

}  // namespace hf3fs::memory

#pragma once

#include <condition_variable>
#include <folly/MPMCQueue.h>
#include <mutex>
#include <thread>
#include <vector>

#include "common/monitor/Monitor.h"
#include "common/serde/Service.h"
#include "common/utils/ConfigBase.h"
#include "fbs/monitor_collector/MonitorCollectorService.h"

namespace hf3fs::monitor {
class MonitorCollectorOperator;
class MonitorCollectorService : public serde::ServiceWrapper<MonitorCollectorService, MonitorCollector> {
 public:
  class Config : public ConfigBase<Config> {
    CONFIG_OBJ(reporter, hf3fs::monitor::Monitor::ReporterConfig, [](hf3fs::monitor::Monitor::ReporterConfig &c) {
      c.set_type("clickhouse");
    });
    CONFIG_ITEM(conn_threads, 32);
    CONFIG_ITEM(queue_capacity, 204800);
    CONFIG_ITEM(batch_commit_size, 4096);
    CONFIG_ITEM(blacklisted_metric_names, std::set<std::string>{});
  };

  MonitorCollectorService(MonitorCollectorOperator &monitorCollectorOperator)
      : monitorCollectorOperator_(monitorCollectorOperator) {}

  CoTryTask<MonitorCollectorRsp> write(serde::CallContext &ctx, std::vector<Sample> &samples);

 private:
  MonitorCollectorOperator &monitorCollectorOperator_;
};

}  // namespace hf3fs::monitor

#pragma once

#include "LocalTargetInfoWithNodeId.h"
#include "NodeInfoWrapper.h"
#include "TargetInfo.h"
#include "fbs/mgmtd/ChainInfo.h"
#include "fbs/mgmtd/ChainTable.h"
#include "fbs/mgmtd/MgmtdTypes.h"

namespace hf3fs::core {
struct ServiceOperation;
}

namespace hf3fs::mgmtd {
struct MgmtdConfig;
class MgmtdTargetInfoLoader;

namespace testing {
class MgmtdTestHelper;
}

using NodeMap = robin_hood::unordered_map<flat::NodeId, NodeInfoWrapper>;
using ChainTableVersionMap = std::map<flat::ChainTableVersion, flat::ChainTable>;
using ChainTableMap = robin_hood::unordered_map<flat::ChainTableId, ChainTableVersionMap>;
using ChainMap = robin_hood::unordered_map<flat::ChainId, flat::ChainInfo>;
using TargetMap = robin_hood::unordered_map<flat::TargetId, TargetInfo>;
using NewBornChains = robin_hood::unordered_map<flat::ChainId, SteadyTime>;
using OrphanTargetsByTargetId = robin_hood::unordered_map<flat::TargetId, flat::TargetInfo>;
using OrphanTargetsByNodeId = robin_hood::unordered_map<flat::NodeId, robin_hood::unordered_set<flat::TargetId>>;

struct RoutingInfo {
  bool routingInfoChanged = false;                  // periodically promote routingInfoVersion
  flat::RoutingInfoVersion routingInfoVersion{1};   // ensure version 0 is less than any valid version
  NodeMap nodeMap;                                  // persistent
  ChainTableMap chainTables;                        // persistent
  ChainMap chains;                                  // persistent
  NewBornChains newBornChains;                      // temporal
  OrphanTargetsByTargetId orphanTargetsByTargetId;  // temporal
  OrphanTargetsByNodeId orphanTargetsByNodeId;      // temporal

  void applyChainTargetChanges(core::ServiceOperation &ctx,
                               const std::vector<flat::ChainInfo> &changedChains,
                               SteadyTime now);

  void localUpdateTargets(flat::NodeId nodeId,
                          const std::vector<flat::LocalTargetInfo> &targets,
                          const MgmtdConfig &config);

  flat::ChainInfo &getChain(flat::ChainId cid);
  const flat::ChainInfo &getChain(flat::ChainId cid) const;

  void insertNewChain(const flat::ChainInfo &chain);

  void insertNewTarget(flat::ChainId cid, const flat::ChainTargetInfo &cti);

  void removeTarget(flat::ChainId cid, flat::TargetId tid);

  const TargetMap &getTargets() const { return targets; }

  auto updateTarget(flat::TargetId tid, auto &&func) {
    XLOGF_IF(DFATAL, !targets.contains(tid), "tid = {}", tid);
    auto &ti = targets[tid];
    return func(ti);
  }

  void reset(flat::RoutingInfoVersion routingInfoVersion,
             NodeMap allNodes,
             ChainTableMap allChainTables,
             ChainMap allChains,
             TargetMap allTargets);

 private:
  void eraseOrphanTarget(flat::TargetId tid);

  TargetMap targets;  // derived

  friend class testing::MgmtdTestHelper;
};

}  // namespace hf3fs::mgmtd

#pragma once

#include "ChainRef.h"
#include "ChainTable.h"
#include "MgmtdTypes.h"
#include "NodeInfo.h"
#include "TargetInfo.h"
#include "common/utils/RobinHoodUtils.h"

namespace hf3fs::flat {
struct RoutingInfo : public serde::SerdeHelper<RoutingInfo> {
  // return nullptr if not found
  const ChainTable *getChainTable(ChainTableId tableId, ChainTableVersion tableVersion = ChainTableVersion(0)) const;
  ChainTable *getChainTable(ChainTableId tableId, ChainTableVersion tableVersion = ChainTableVersion(0));

  // return std::nullopt if not found
  std::optional<ChainId> getChainId(ChainRef ref) const;

  // return nullptr if not found
  const ChainInfo *getChain(ChainId id) const;
  ChainInfo *getChain(ChainId id);

  // return nullptr if not found
  const ChainInfo *getChain(ChainRef ref) const;
  ChainInfo *getChain(ChainRef ref);

  // return nullptr if not found
  const NodeInfo *getNode(NodeId id) const;
  NodeInfo *getNode(NodeId id);

  // return nullptr if not found
  const TargetInfo *getTarget(TargetId id) const;
  TargetInfo *getTarget(TargetId id);

  using NodeMap = robin_hood::unordered_map<NodeId, NodeInfo>;
  // ordered
  using ChainTableVersionMap = std::map<ChainTableVersion, ChainTable>;
  using ChainTableMap = robin_hood::unordered_map<ChainTableId, ChainTableVersionMap>;
  using ChainMap = robin_hood::unordered_map<ChainId, ChainInfo>;
  using TargetMap = robin_hood::unordered_map<TargetId, TargetInfo>;

  SERDE_STRUCT_FIELD(routingInfoVersion, RoutingInfoVersion(0));
  SERDE_STRUCT_FIELD(bootstrapping, bool(false));
  SERDE_STRUCT_FIELD(nodes, NodeMap{});
  SERDE_STRUCT_FIELD(chainTables, ChainTableMap{});
  SERDE_STRUCT_FIELD(chains, ChainMap{});
  SERDE_STRUCT_FIELD(targets, TargetMap{});
};
}  // namespace hf3fs::flat

#pragma once

#include "common/app/AppInfo.h"
#include "common/utils/Selector.h"
#include "fbs/mgmtd/MgmtdTypes.h"

namespace hf3fs::client {
struct ServiceInfo {
  String name;
  uint16_t id{0};
  flat::NodeId nodeId{0};
  flat::NodeStatus nodeStatus{flat::NodeStatus::HEARTBEAT_CONNECTING};
  std::vector<net::Address> endpoints;

  std::vector<net::Address> filterAddress(net::Address::Type type) const {
    std::vector<net::Address> addresses;
    std::copy_if(endpoints.begin(), endpoints.end(), std::back_inserter(addresses), [&](const auto &addr) {
      return addr.type == type;
    });
    return addresses;
  }

  ServiceInfo() = default;
};
}  // namespace hf3fs::client

#pragma once

#include <folly/CancellationToken.h>

#include "client/mgmtd/MgmtdClientForServer.h"
#include "common/net/Server.h"
#include "core/app/ServerAppConfig.h"
#include "core/app/ServerLauncher.h"
#include "core/app/ServerLauncherConfig.h"
#include "core/app/ServerMgmtdClientFetcher.h"
#include "storage/service/Components.h"
#include "storage/service/ReliableForwarding.h"
#include "storage/service/ReliableUpdate.h"
#include "storage/service/StorageOperator.h"

namespace hf3fs::test {
struct StorageServerHelper;
}

namespace hf3fs::storage {

class StorageServer : public net::Server {
 public:
  static constexpr auto kName = "Storage";
  static constexpr auto kNodeType = flat::NodeType::STORAGE;

  using AppConfig = core::ServerAppConfig;
  struct LauncherConfig : public core::ServerLauncherConfig {
    LauncherConfig() { mgmtd_client() = hf3fs::client::MgmtdClientForServer::Config{}; }
  };
  using RemoteConfigFetcher = core::launcher::ServerMgmtdClientFetcher;
  using Launcher = core::ServerLauncher<StorageServer>;

  using CommonConfig = ApplicationBase::Config;
  using Config = Components::Config;
  StorageServer(const Components::Config &config);
  ~StorageServer() override;

  // set up storage server.
  Result<Void> beforeStart() final;

  // before server stop.
  Result<Void> beforeStop() final;

  // tear down storage server.
  Result<Void> afterStop() final;

  using net::Server::start;
  hf3fs::Result<Void> start(const flat::AppInfo &info,
                            std::unique_ptr<::hf3fs::net::Client> client,
                            std::shared_ptr<::hf3fs::client::MgmtdClient> mgmtdClient);

 private:
  friend struct test::StorageServerHelper;
  ConstructLog<"storage::StorageServer"> constructLog_;
  Components components_;
};

}  // namespace hf3fs::storage

#pragma once

#include <cstdint>
#include <vector>

#include "common/serde/Serde.h"
#include "fbs/meta/Schema.h"
namespace hf3fs::client::cli {

struct DirEntryRow {
  SERDE_STRUCT_FIELD(timestamp, std::time_t{});
  SERDE_STRUCT_FIELD(entry, meta::DirEntry{});
};

struct DirEntryTable {
  SERDE_STRUCT_FIELD(timestamp, std::time_t{});
  SERDE_STRUCT_FIELD(entries, std::vector<DirEntryRow>());

 public:
  bool dumpToParquetFile(const Path &filePath) const;
};
static_assert(serde::Serializable<DirEntryTable>);

CoTryTask<Void> dumpDirEntriesFromFdb(const std::string fdbClusterFile,
                                      const uint32_t numEntriesPerDir,
                                      const std::string dentryDir,
                                      const uint32_t threads);
class Dispatcher;
CoTryTask<void> registerDumpDirEntriesHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "CommonMgmtdClient.h"
#include "IMgmtdClientForServer.h"

namespace hf3fs::client {
class MgmtdClientForServer : public CommonMgmtdClient<IMgmtdClientForServer> {
 public:
  struct Config : MgmtdClient::Config {
    Config() {
      set_enable_auto_refresh(true);
      set_enable_auto_heartbeat(true);
      set_enable_auto_extend_client_session(false);
    }
  };

  explicit MgmtdClientForServer(std::shared_ptr<MgmtdClient> client)
      : CommonMgmtdClient(std::move(client)) {}

  MgmtdClientForServer(String clusterId,
                       std::unique_ptr<MgmtdClient::MgmtdStubFactory> stubFactory,
                       const Config &config)
      : CommonMgmtdClient(std::make_shared<MgmtdClient>(std::move(clusterId), std::move(stubFactory), config)) {}

  CoTryTask<mgmtd::HeartbeatRsp> heartbeat() final { return client_->heartbeat(); }

  Result<Void> triggerHeartbeat() final { return client_->triggerHeartbeat(); }

  void setAppInfoForHeartbeat(flat::AppInfo info) final { client_->setAppInfoForHeartbeat(std::move(info)); }

  void setConfigListener(ConfigListener listener) final { client_->setServerConfigListener(std::move(listener)); }

  void updateHeartbeatPayload(HeartbeatPayload payload) final { client_->updateHeartbeatPayload(std::move(payload)); }
};
}  // namespace hf3fs::client

#pragma once

#include <map>

#include "common/net/Network.h"
#include "common/net/tcp/TcpSocket.h"
#include "common/utils/BoundedQueue.h"
#include "common/utils/ConfigBase.h"

namespace hf3fs::net::sync {

class ConnectionPool {
 public:
  struct Config : public ConfigBase<Config> {
    CONFIG_HOT_UPDATED_ITEM(max_connection_num, 16ul);
    CONFIG_HOT_UPDATED_ITEM(tcp_connect_timeout, 1000_ms);
  };

  ConnectionPool(const Config &config)
      : config_(config) {}

  Result<std::unique_ptr<TcpSocket>> acquire(Address addr);

  void restore(Address addr, std::unique_ptr<TcpSocket> socket);

 private:
  using Connections = BoundedQueue<std::unique_ptr<TcpSocket>>;
  Connections *getConnections(Address addr);

 private:
  [[maybe_unused]] const Config &config_;
  std::mutex mutex_;
  robin_hood::unordered_map<Address, std::unique_ptr<Connections>> map_;
  folly::ThreadLocal<robin_hood::unordered_map<Address, Connections *>> tlsCache_;
};

}  // namespace hf3fs::net::sync

#pragma once

#include "IStubFactory.h"
#include "common/serde/ClientContext.h"

namespace hf3fs::stubs {
using ClientContextCreator = std::function<serde::ClientContext(net::Address)>;

template <template <typename Ctx> typename Stub>
requires requires { typename Stub<serde::ClientContext>::InterfaceType; }
class RealStubFactory : public IStubFactory<typename Stub<serde::ClientContext>::InterfaceType> {
 public:
  using StubType = Stub<serde::ClientContext>;
  using InterfaceType = typename StubType::InterfaceType;

  explicit RealStubFactory(ClientContextCreator creator)
      : creator_(std::move(creator)) {}

  std::unique_ptr<InterfaceType> create(net::Address addr) override {
    return std::make_unique<StubType>(creator_(addr));
  }

 private:
  ClientContextCreator creator_;
};
}  // namespace hf3fs::stubs

#pragma once

#include <folly/Range.h>

#include "common/utils/Result.h"
#include "storage/aio/BatchReadJob.h"
#include "storage/store/ChunkMetadata.h"
#include "storage/store/ChunkStore.h"
#include "storage/update/UpdateJob.h"

namespace hf3fs::storage {

class ChunkReplica {
 public:
  // prepare aio read.
  static Result<Void> aioPrepareRead(ChunkStore &store, AioReadJob &job);

  // finish aio read.
  static Result<Void> aioFinishRead(ChunkStore &store, AioReadJob &job);

  // do write.
  static Result<uint32_t> update(ChunkStore &store, UpdateJob &job, folly::CPUThreadPoolExecutor &executor);

  static Result<Void> updateChecksum(ChunkInfo &chunkInfo,
                                     UpdateIO writeIO,
                                     uint32_t chunkSizeBeforeWrite,
                                     bool isAppendWrite);

  // commit the version of this chunk.
  static Result<uint32_t> commit(ChunkStore &store, UpdateJob &job);
};

}  // namespace hf3fs::storage

#pragma once

#include "common/app/NodeId.h"
#include "common/utils/Address.h"
#include "common/utils/Result.h"
#include "common/utils/StrongType.h"
#include "common/utils/Uuid.h"

namespace hf3fs::flat {
enum class PublicTargetState : uint8_t {
  INVALID = 0,
  SERVING = 1,
  LASTSRV = 2,
  SYNCING = 4,
  WAITING = 8,
  OFFLINE = 16,
  MIN = INVALID,
  MAX = OFFLINE
};

enum class LocalTargetState : uint8_t {
  INVALID = 0,
  UPTODATE = 1,
  ONLINE = 2,
  OFFLINE = 4,
  MIN = INVALID,
  MAX = OFFLINE
};

enum class NodeStatus : uint8_t {
  PRIMARY_MGMTD = 0,
  HEARTBEAT_CONNECTED = 1,
  HEARTBEAT_CONNECTING = 2,
  HEARTBEAT_FAILED = 3,
  DISABLED = 4,
  MIN = HEARTBEAT_CONNECTED,
  MAX = DISABLED
};

enum class NodeType : uint8_t { MGMTD = 0, META = 1, STORAGE = 2, CLIENT = 3, FUSE = 4, MIN = MGMTD, MAX = FUSE };

enum class SetTagMode : uint8_t { REPLACE = 0, UPSERT = 1, REMOVE = 2, MIN = REPLACE, MAX = REMOVE };

STRONG_TYPEDEF(uint64_t, TargetId);
STRONG_TYPEDEF(uint64_t, HeartbeatVersion);
STRONG_TYPEDEF(uint64_t, RoutingInfoVersion);
STRONG_TYPEDEF(uint64_t, ConfigVersion);
STRONG_TYPEDEF(uint64_t, ClientSessionVersion);

// ChainVersion is space sensitive and uint32 is enough for changes of one chain
STRONG_TYPEDEF(uint32_t, ChainVersion);

STRONG_TYPEDEF(uint32_t, ChainTableId);
STRONG_TYPEDEF(uint32_t, ChainTableVersion);
STRONG_TYPEDEF(uint32_t, ChainId);

}  // namespace hf3fs::flat

#pragma once

#include <folly/logging/LogCategory.h>
#include <folly/logging/LogFormatter.h>
#include <folly/logging/LogMessage.h>
#include <folly/logging/StandardLogHandlerFactory.h>

namespace hf3fs::logging {
class LogFormatter : public folly::LogFormatter {
 public:
  std::string formatMessage(const folly::LogMessage &message, const folly::LogCategory *) override;
};

class LogFormatterFactory : public folly::StandardLogHandlerFactory::FormatterFactory {
 public:
  bool processOption(folly::StringPiece /* name */, folly::StringPiece /* value */) override { return false; }
  std::shared_ptr<folly::LogFormatter> createFormatter(
      const std::shared_ptr<folly::LogWriter> & /* logWriter */) override {
    return std::make_shared<LogFormatter>();
  }
};

class EventLogFormatter : public folly::LogFormatter {
 public:
  std::string formatMessage(const folly::LogMessage &message, const folly::LogCategory *) override;
};

class EventLogFormatterFactory : public folly::StandardLogHandlerFactory::FormatterFactory {
 public:
  bool processOption(folly::StringPiece /* name */, folly::StringPiece /* value */) override { return false; }
  std::shared_ptr<folly::LogFormatter> createFormatter(
      const std::shared_ptr<folly::LogWriter> & /* logWriter */) override {
    return std::make_shared<EventLogFormatter>();
  }
};

}  // namespace hf3fs::logging

#pragma once

#include <folly/File.h>

#include "IWritableFile.h"

namespace hf3fs::logging {
class SingleFile : public IWritableFile {
 public:
  SingleFile(folly::File &&file)
      : file_(std::move(file)) {}

  bool ttyOutput() const override;
  std::string desc() const override;
  ssize_t writevFull(iovec *iov, int count) override;
  ssize_t writeFull(const void *buf, size_t count) override;
  void flush() override;

 private:
  folly::File file_;
};
}  // namespace hf3fs::logging

#pragma once

#include <atomic>
#include <condition_variable>
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <mutex>

#include "storage/service/TargetMap.h"

namespace hf3fs::storage {

struct Components;

class AllocateWorker {
 public:
  class Config : public ConfigBase<Config> {
    CONFIG_HOT_UPDATED_ITEM(min_remain_groups, 4ul);
    CONFIG_HOT_UPDATED_ITEM(max_remain_groups, 8ul);
    CONFIG_HOT_UPDATED_ITEM(min_remain_ultra_groups, 0ul);  // greater than 4MiB
    CONFIG_HOT_UPDATED_ITEM(max_remain_ultra_groups, 4ul);
    CONFIG_HOT_UPDATED_ITEM(max_reserved_chunks, 1_GB);
  };

  AllocateWorker(const Config &config, Components &components);

  Result<Void> start();

  Result<Void> stopAndJoin();

 protected:
  void loop();

 private:
  ConstructLog<"storage::AllocateWorker"> constructLog_;
  const Config &config_;
  Components &components_;
  folly::CPUThreadPoolExecutor executors_;

  std::mutex mutex_;
  std::condition_variable cond_;
  std::atomic<bool> stopping_ = false;
  std::atomic<bool> started_ = false;
  std::atomic<bool> stopped_ = false;
};

}  // namespace hf3fs::storage

#pragma once

#include <asm-generic/ioctl.h>
#include <cstdint>
#include <linux/limits.h>
#include <sys/ioctl.h>
#include <sys/types.h>

namespace hf3fs::lib::fuse {

#define HF3FS_IOCTYPE_ID 'h'
#define HF3FS_IOCTL_PUNCH_HOLE_MAX 1000

struct Hf3fsIoctlGetMountNameArg {
  char str[32];
};

struct Hf3fsIoctlHardlinkArg {
  ino_t ino;
  char str[NAME_MAX];
};

struct Hf3fsIoctlPunchHoleArg {
  int n;
  size_t flags;
  size_t start[HF3FS_IOCTL_PUNCH_HOLE_MAX];
  size_t end[HF3FS_IOCTL_PUNCH_HOLE_MAX];
};

struct Hf3fsIoctlMove {
  uint64_t srcParent;
  char srcName[NAME_MAX + 1];
  uint64_t dstParent;
  char dstName[NAME_MAX + 1];
  bool moveToTrash;
};

struct Hf3fsIoctlRemove {
  uint64_t parent;
  char name[NAME_MAX + 1];
  bool recursive;
};

enum {
  HF3FS_IOC_GET_MOUNT_NAME = _IOR(HF3FS_IOCTYPE_ID, 0, Hf3fsIoctlGetMountNameArg),
  HF3FS_IOC_GET_PATH_OFFSET = _IOR(HF3FS_IOCTYPE_ID, 1, uint32_t),
  HF3FS_IOC_GET_MAGIC_NUM = _IOR(HF3FS_IOCTYPE_ID, 2, uint32_t),
  HF3FS_IOC_GET_IOCTL_VERSION = _IOR(HF3FS_IOCTYPE_ID, 3, uint32_t),

  HF3FS_IOC_RECURSIVE_RM = _IOR(HF3FS_IOCTYPE_ID, 10, uint32_t),
  HF3FS_IOC_FSYNC = _IOR(HF3FS_IOCTYPE_ID, 11, uint32_t),
  HF3FS_IOC_HARDLINK = _IOW(HF3FS_IOCTYPE_ID, 12, Hf3fsIoctlHardlinkArg),
  HF3FS_IOC_PUNCH_HOLE = _IOW(HF3FS_IOCTYPE_ID, 13, Hf3fsIoctlPunchHoleArg),
  HF3FS_IOC_MOVE = _IOW(HF3FS_IOCTYPE_ID, 14, Hf3fsIoctlMove),
  HF3FS_IOC_REMOVE = _IOW(HF3FS_IOCTYPE_ID, 15, Hf3fsIoctlRemove),
};

}  // namespace hf3fs::lib::fuse

#pragma once

#include <vector>

#include "IEnv.h"
#include "common/utils/ArgParse.h"
#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher {
 public:
  using Args = std::vector<String>;
  using OutputRow = std::vector<String>;
  using OutputTable = std::vector<OutputRow>;
  using ParserGetter = std::function<argparse::ArgumentParser()>;
  using Handler = std::function<CoTryTask<OutputTable>(IEnv &, const argparse::ArgumentParser &, const Args &)>;

  Dispatcher();
  ~Dispatcher();
  CoTryTask<OutputTable> run(IEnv &env, const Args &args) const;

  CoTryTask<void> registerHandler(String usage,
                                  String help,
                                  ParserGetter parserGetter,
                                  Handler handler,
                                  bool replace = false);
  CoTryTask<void> registerHandler(ParserGetter parserGetter, Handler handler, bool replace = false);

  template <typename Handler>
  CoTryTask<void> registerHandler() {
    co_return co_await registerHandler(&Handler::getParser, &Handler::handle);
  }

  std::map<String, String, std::less<>> getUsages() const;

  CoTryTask<void> run(IEnv &env,
                      const std::function<String()> &promptGetter,
                      std::string_view cmd,
                      bool verbose,
                      bool profile,
                      bool breakMultiLineCommandOnFailure);

 private:
  struct HandlerInfo;

  CoTryTask<const HandlerInfo *> findMethod(const String &name) const;
  CoTryTask<OutputTable> handleHelp(String method) const;

  std::map<String, std::unique_ptr<HandlerInfo>> handlers_;
};
}  // namespace hf3fs::client::cli

#pragma once

#include <folly/FileUtil.h>
#include <string>

namespace hf3fs::logging {
class IWritableFile {
 public:
  virtual ~IWritableFile() = default;

  virtual bool ttyOutput() const = 0;
  virtual std::string desc() const = 0;
  virtual ssize_t writevFull(iovec *iov, int count) = 0;
  virtual ssize_t writeFull(const void *buf, size_t count) = 0;
  virtual void flush() = 0;
};
}  // namespace hf3fs::logging

#pragma once

#include "common/kv/IKVEngine.h"

namespace hf3fs::kv::fdb {
struct FDBConfig;
class FDBContext;
}  // namespace hf3fs::kv::fdb

namespace hf3fs::kv {
struct HybridKvEngineConfig;
class HybridKvEngine : public kv::IKVEngine {
 public:
  static std::shared_ptr<HybridKvEngine> fromMem();
  static std::shared_ptr<HybridKvEngine> fromFdb(const kv::fdb::FDBConfig &config);
  static std::shared_ptr<HybridKvEngine> from(bool useMemKV, const kv::fdb::FDBConfig &config);
  static std::shared_ptr<HybridKvEngine> from(const HybridKvEngineConfig &config);

  // use `config` if explicitly set, else fallback to use `useMemKV` and `fdbConfig`.
  static std::shared_ptr<HybridKvEngine> from(const HybridKvEngineConfig &config,
                                              bool useMemKV,
                                              const kv::fdb::FDBConfig &fdbConfig);

  // config contains:
  // 1. use_memkv (fallback mode)
  // 2. fdb (fallback mode)
  // 3. kv_engine (new)
  static std::shared_ptr<HybridKvEngine> fromSuperConfig(const auto &cfg) {
    return from(cfg.kv_engine(), cfg.use_memkv(), cfg.fdb());
  }

  ~HybridKvEngine();

  std::unique_ptr<kv::IReadOnlyTransaction> createReadonlyTransaction() override;
  std::unique_ptr<kv::IReadWriteTransaction> createReadWriteTransaction() override;

 private:
  HybridKvEngine();

  kv::IKVEngine &pick() const;

  std::shared_ptr<kv::fdb::FDBContext> fdbContext_;
  std::vector<std::unique_ptr<kv::IKVEngine>> kvEngines_;
};

}  // namespace hf3fs::kv

#pragma once

#include <cstdint>
#include <string_view>

namespace hf3fs {
// NOTE: `StatusCode` is used as the namespace name so here we use `status_code_t` as the type name.
using status_code_t = uint16_t;

#define RAW_STATUS(name, value)                   \
  namespace StatusCode {                          \
  inline constexpr status_code_t k##name = value; \
  }

#define STATUS(ns, name, value)                     \
  namespace ns##Code {                              \
    inline constexpr status_code_t k##name = value; \
  }

#include "StatusCodeDetails.h"

#undef STATUS
#undef RAW_STATUS

enum class StatusCodeType {
  Invalid = -1,
  Common = 0,
  Transaction,
  RPC,
  Meta,
  Storage,
  Mgmtd,
  MgmtdClient,
  StorageClient,
  ClientAgent,
  Cli,
  KvService,
};

namespace StatusCode {
std::string_view toString(status_code_t code);

StatusCodeType typeOf(status_code_t code);

int toErrno(status_code_t code);
}  // namespace StatusCode
}  // namespace hf3fs

#pragma once

#include "CommonMgmtdClient.h"
#include "IMgmtdClientForClient.h"

namespace hf3fs::client {
class MgmtdClientForClient : public CommonMgmtdClient<IMgmtdClientForClient> {
 public:
  struct Config : MgmtdClient::Config {
    Config() {
      set_enable_auto_refresh(true);
      set_enable_auto_heartbeat(false);
      set_enable_auto_extend_client_session(true);
    }
  };

  explicit MgmtdClientForClient(std::shared_ptr<MgmtdClient> client)
      : CommonMgmtdClient(std::move(client)) {}

  MgmtdClientForClient(String clusterId,
                       std::unique_ptr<MgmtdClient::MgmtdStubFactory> stubFactory,
                       const Config &config)
      : CommonMgmtdClient(std::make_shared<MgmtdClient>(std::move(clusterId), std::move(stubFactory), config)) {}

  CoTryTask<void> extendClientSession() final { return client_->extendClientSession(); }

  void setClientSessionPayload(ClientSessionPayload payload) { client_->setClientSessionPayload(std::move(payload)); }

  void setConfigListener(ConfigListener listener) final { client_->setClientConfigListener(std::move(listener)); }
};
}  // namespace hf3fs::client

/*
 * error_definitions.h
 *
 * This source file is part of the FoundationDB open source project
 *
 * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifdef ERROR

// SOMEDAY: Split this into flow, fdbclient, fdbserver error headers?

// Error codes defined here are primarily for programmatic use, not debugging: a separate
// error should be defined if and only if there is a sensible situation in which code could
// catch and react specifically to that error.  So for example there is only one
// internal_error code even though there are a huge number of internal errors; extra
// information is logged in the trace file.

// 1xxx Normal failure (plausibly these should not even be "errors", but they are failures of
//   the way operations are currently defined)
// clang-format off
ERROR( success, 0, "Success" )
ERROR( end_of_stream, 1, "End of stream" )
ERROR( operation_failed, 1000, "Operation failed")
ERROR( wrong_shard_server, 1001, "Shard is not available from this server")
ERROR( operation_obsolete, 1002, "Operation result no longer necessary")
ERROR( cold_cache_server, 1003, "Cache server is not warm for this range")
ERROR( timed_out, 1004, "Operation timed out" )
ERROR( coordinated_state_conflict, 1005, "Conflict occurred while changing coordination information" )
ERROR( all_alternatives_failed, 1006, "All alternatives failed" )
ERROR( transaction_too_old, 1007, "Transaction is too old to perform reads or be committed" )
ERROR( no_more_servers, 1008, "Not enough physical servers available" )
ERROR( future_version, 1009, "Request for future version" )
ERROR( movekeys_conflict, 1010, "Conflicting attempts to change data distribution" )
ERROR( tlog_stopped, 1011, "TLog stopped" )
ERROR( server_request_queue_full, 1012, "Server request queue is full" )
ERROR( not_committed, 1020, "Transaction not committed due to conflict with another transaction" )
ERROR( commit_unknown_result, 1021, "Transaction may or may not have committed" )
ERROR( transaction_cancelled, 1025, "Operation aborted because the transaction was cancelled" )
ERROR( connection_failed, 1026, "Network connection failed" )
ERROR( coordinators_changed, 1027, "Coordination servers have changed" )
ERROR( new_coordinators_timed_out, 1028, "New coordination servers did not respond in a timely way" )
ERROR( watch_cancelled, 1029, "Watch cancelled because storage server watch limit exceeded" )
ERROR( request_maybe_delivered, 1030, "Request may or may not have been delivered" )
ERROR( transaction_timed_out, 1031, "Operation aborted because the transaction timed out" )
ERROR( too_many_watches, 1032, "Too many watches currently set" )
ERROR( locality_information_unavailable, 1033, "Locality information not available" )
ERROR( watches_disabled, 1034, "Watches cannot be set if read your writes is disabled" )
ERROR( default_error_or, 1035, "Default error for an ErrorOr object" )
ERROR( accessed_unreadable, 1036, "Read or wrote an unreadable key" )
ERROR( process_behind, 1037, "Storage process does not have recent mutations" )
ERROR( database_locked, 1038, "Database is locked" )
ERROR( cluster_version_changed, 1039, "The protocol version of the cluster has changed" )
ERROR( external_client_already_loaded, 1040, "External client has already been loaded" )
ERROR( lookup_failed, 1041, "DNS lookup failed" )
ERROR( proxy_memory_limit_exceeded, 1042, "CommitProxy commit memory limit exceeded" )
ERROR( shutdown_in_progress, 1043, "Operation no longer supported due to shutdown" )
ERROR( serialization_failed, 1044, "Failed to deserialize an object" )
ERROR( connection_unreferenced, 1048, "No peer references for connection" )
ERROR( connection_idle, 1049, "Connection closed after idle timeout" )
ERROR( disk_adapter_reset, 1050, "The disk queue adpater reset" )
ERROR( batch_transaction_throttled, 1051, "Batch GRV request rate limit exceeded")
ERROR( dd_cancelled, 1052, "Data distribution components cancelled")
ERROR( dd_not_found, 1053, "Data distributor not found")
ERROR( wrong_connection_file, 1054, "Connection file mismatch")
ERROR( version_already_compacted, 1055, "The requested changes have been compacted away")
ERROR( local_config_changed, 1056, "Local configuration file has changed. Restart and apply these changes" )
ERROR( failed_to_reach_quorum, 1057, "Failed to reach quorum from configuration database nodes. Retry sending these requests" )
ERROR( unsupported_format_version, 1058, "Format version not supported" )
ERROR( unknown_change_feed, 1059, "Change feed not found" )
ERROR( change_feed_not_registered, 1060, "Change feed not registered" )
ERROR( granule_assignment_conflict, 1061, "Conflicting attempts to assign blob granules" )
ERROR( change_feed_cancelled, 1062, "Change feed was cancelled" )
ERROR( blob_granule_file_load_error, 1063, "Error loading a blob file during granule materialization" )
ERROR( blob_granule_transaction_too_old, 1064, "Read version is older than blob granule history supports" )
ERROR( blob_manager_replaced, 1065, "This blob manager has been replaced." )
ERROR( change_feed_popped, 1066, "Tried to read a version older than what has been popped from the change feed" )
ERROR( remote_kvs_cancelled, 1067, "The remote key-value store is cancelled" )
ERROR( page_header_wrong_page_id, 1068, "Page header does not match location on disk" )
ERROR( page_header_checksum_failed, 1069, "Page header checksum failed" )
ERROR( page_header_version_not_supported, 1070, "Page header version is not supported" )
ERROR( page_encoding_not_supported, 1071, "Page encoding type is not supported or not valid" )
ERROR( page_decoding_failed, 1072, "Page content decoding failed" )
ERROR( unexpected_encoding_type, 1073, "Page content decoding failed" )
ERROR( encryption_key_not_found, 1074, "Encryption key not found" )

ERROR( broken_promise, 1100, "Broken promise" )
ERROR( operation_cancelled, 1101, "Asynchronous operation cancelled" )
ERROR( future_released, 1102, "Future has been released" )
ERROR( connection_leaked, 1103, "Connection object leaked" )
ERROR( never_reply, 1104, "Never reply to the request" )

ERROR( recruitment_failed, 1200, "Recruitment of a server failed" )   // Be careful, catching this will delete the data of a storage server or tlog permanently
ERROR( move_to_removed_server, 1201, "Attempt to move keys to a storage server that was removed" )
ERROR( worker_removed, 1202, "Normal worker shut down" )   // Be careful, catching this will delete the data of a storage server or tlog permanently
ERROR( cluster_recovery_failed, 1203, "Cluster recovery failed")
ERROR( master_max_versions_in_flight, 1204, "Master hit maximum number of versions in flight" )
ERROR( tlog_failed, 1205, "Cluster recovery terminating because a TLog failed" )   // similar to tlog_stopped, but the tlog has actually died
ERROR( worker_recovery_failed, 1206, "Recovery of a worker process failed" )
ERROR( please_reboot, 1207, "Reboot of server process requested" )
ERROR( please_reboot_delete, 1208, "Reboot of server process requested, with deletion of state" )
ERROR( commit_proxy_failed, 1209, "Master terminating because a CommitProxy failed" )
ERROR( resolver_failed, 1210, "Cluster recovery terminating because a Resolver failed" )
ERROR( server_overloaded, 1211, "Server is under too much load and cannot respond" )
ERROR( backup_worker_failed, 1212, "Cluster recovery terminating because a backup worker failed")
ERROR( tag_throttled, 1213, "Transaction tag is being throttled" )
ERROR( grv_proxy_failed, 1214, "Cluster recovery terminating because a GRVProxy failed" )
ERROR( dd_tracker_cancelled, 1215, "The data distribution tracker has been cancelled" )
ERROR( failed_to_progress, 1216, "Process has failed to make sufficient progress" )
ERROR( invalid_cluster_id, 1217, "Attempted to join cluster with a different cluster ID" )
ERROR( restart_cluster_controller, 1218, "Restart cluster controller process" )
ERROR( please_reboot_remote_kv_store, 1219, "Need to reboot the storage engine process as it died abnormally")

// 15xx Platform errors
ERROR( platform_error, 1500, "Platform error" )
ERROR( large_alloc_failed, 1501, "Large block allocation failed" )
ERROR( performance_counter_error, 1502, "QueryPerformanceCounter error" )

ERROR( io_error, 1510, "Disk i/o operation failed" )
ERROR( file_not_found, 1511, "File not found" )
ERROR( bind_failed, 1512, "Unable to bind to network" )
ERROR( file_not_readable, 1513, "File could not be read" )
ERROR( file_not_writable, 1514, "File could not be written" )
ERROR( no_cluster_file_found, 1515, "No cluster file found in current directory or default location" )
ERROR( file_too_large, 1516, "File too large to be read" )
ERROR( non_sequential_op, 1517, "Non sequential file operation not allowed" )
ERROR( http_bad_response, 1518, "HTTP response was badly formed" )
ERROR( http_not_accepted, 1519, "HTTP request not accepted" )
ERROR( checksum_failed, 1520, "A data checksum failed" )
ERROR( io_timeout, 1521, "A disk IO operation failed to complete in a timely manner" )
ERROR( file_corrupt, 1522, "A structurally corrupt data file was detected" )
ERROR( http_request_failed, 1523, "HTTP response code not received or indicated failure" )
ERROR( http_auth_failed, 1524, "HTTP request failed due to bad credentials" )
ERROR( http_bad_request_id, 1525, "HTTP response contained an unexpected X-Request-ID header" )

// 2xxx Attempt (presumably by a _client_) to do something illegal.  If an error is known to
// be internally caused, it should be 41xx
ERROR( client_invalid_operation, 2000, "Invalid API call" )
ERROR( commit_read_incomplete, 2002, "Commit with incomplete read" )
ERROR( test_specification_invalid, 2003, "Invalid test specification" )
ERROR( key_outside_legal_range, 2004, "Key outside legal range" )
ERROR( inverted_range, 2005, "Range begin key larger than end key" )
ERROR( invalid_option_value, 2006, "Option set with an invalid value" )
ERROR( invalid_option, 2007, "Option not valid in this context" )
ERROR( network_not_setup, 2008, "Action not possible before the network is configured" )
ERROR( network_already_setup, 2009, "Network can be configured only once" )
ERROR( read_version_already_set, 2010, "Transaction already has a read version set" )
ERROR( version_invalid, 2011, "Version not valid" )
ERROR( range_limits_invalid, 2012, "Range limits not valid" )
ERROR( invalid_database_name, 2013, "Database name must be 'DB'" )
ERROR( attribute_not_found, 2014, "Attribute not found" )
ERROR( future_not_set, 2015, "Future not ready" )
ERROR( future_not_error, 2016, "Future not an error" )
ERROR( used_during_commit, 2017, "Operation issued while a commit was outstanding" )
ERROR( invalid_mutation_type, 2018, "Unrecognized atomic mutation type" )
ERROR( attribute_too_large, 2019, "Attribute too large for type int" )
ERROR( transaction_invalid_version, 2020, "Transaction does not have a valid commit version" )
ERROR( no_commit_version, 2021, "Transaction is read-only and therefore does not have a commit version" )
ERROR( environment_variable_network_option_failed, 2022, "Environment variable network option could not be set" )
ERROR( transaction_read_only, 2023, "Attempted to commit a transaction specified as read-only" )
ERROR( invalid_cache_eviction_policy, 2024, "Invalid cache eviction policy, only random and lru are supported" )
ERROR( network_cannot_be_restarted, 2025, "Network can only be started once" )
ERROR( blocked_from_network_thread, 2026, "Detected a deadlock in a callback called from the network thread" )
ERROR( invalid_config_db_range_read, 2027, "Invalid configuration database range read" )
ERROR( invalid_config_db_key, 2028, "Invalid configuration database key provided" )
ERROR( invalid_config_path, 2029, "Invalid configuration path" )
ERROR( mapper_bad_index, 2030, "The index in K[] or V[] is not a valid number or out of range" )
ERROR( mapper_no_such_key, 2031, "A mapped key is not set in database" )
ERROR( mapper_bad_range_decriptor, 2032, "\"{...}\" must be the last element of the mapper tuple" )
ERROR( quick_get_key_values_has_more, 2033, "One of the mapped range queries is too large" )
ERROR( quick_get_value_miss, 2034, "Found a mapped key that is not served in the same SS" )
ERROR( quick_get_key_values_miss, 2035, "Found a mapped range that is not served in the same SS" )
ERROR( blob_granule_no_ryw, 2036, "Blob Granule Read Transactions must be specified as ryw-disabled" )
ERROR( blob_granule_not_materialized, 2037, "Blob Granule Read was not materialized" )
ERROR( get_mapped_key_values_has_more, 2038, "getMappedRange does not support continuation for now" )
ERROR( get_mapped_range_reads_your_writes, 2039, "getMappedRange tries to read data that were previously written in the transaction" )
ERROR( checkpoint_not_found, 2040, "Checkpoint not found" )
ERROR( key_not_tuple, 2041, "The key cannot be parsed as a tuple" );
ERROR( value_not_tuple, 2042, "The value cannot be parsed as a tuple" );
ERROR( mapper_not_tuple, 2043, "The mapper cannot be parsed as a tuple" );

ERROR( incompatible_protocol_version, 2100, "Incompatible protocol version" )
ERROR( transaction_too_large, 2101, "Transaction exceeds byte limit" )
ERROR( key_too_large, 2102, "Key length exceeds limit" )
ERROR( value_too_large, 2103, "Value length exceeds limit" )
ERROR( connection_string_invalid, 2104, "Connection string invalid" )
ERROR( address_in_use, 2105, "Local address in use" )
ERROR( invalid_local_address, 2106, "Invalid local address" )
ERROR( tls_error, 2107, "TLS error" )
ERROR( unsupported_operation, 2108, "Operation is not supported" )
ERROR( too_many_tags, 2109, "Too many tags set on transaction" )
ERROR( tag_too_long, 2110, "Tag set on transaction is too long" )
ERROR( too_many_tag_throttles, 2111, "Too many tag throttles have been created" )
ERROR( special_keys_cross_module_read, 2112, "Special key space range read crosses modules. Refer to the `special_key_space_relaxed' transaction option for more details." )
ERROR( special_keys_no_module_found, 2113, "Special key space range read does not intersect a module. Refer to the `special_key_space_relaxed' transaction option for more details." )
ERROR( special_keys_write_disabled, 2114, "Special Key space is not allowed to write by default. Refer to the `special_key_space_enable_writes` transaction option for more details." )
ERROR( special_keys_no_write_module_found, 2115, "Special key space key or keyrange in set or clear does not intersect a module" )
ERROR( special_keys_cross_module_clear, 2116, "Special key space clear crosses modules" )
ERROR( special_keys_api_failure, 2117, "Api call through special keys failed. For more information, call get on special key 0xff0xff/error_message to get a json string of the error message." )
ERROR( client_lib_invalid_metadata, 2118, "Invalid client library metadata." )
ERROR( client_lib_already_exists, 2119, "Client library with same identifier already exists on the cluster." )
ERROR( client_lib_not_found, 2120, "Client library for the given identifier not found." )
ERROR( client_lib_not_available, 2121, "Client library exists, but is not available for download." )
ERROR( client_lib_invalid_binary, 2122, "Invalid client library binary." )

ERROR( tenant_name_required, 2130, "Tenant name must be specified to access data in the cluster" )
ERROR( tenant_not_found, 2131, "Tenant does not exist" )
ERROR( tenant_already_exists, 2132, "A tenant with the given name already exists" )
ERROR( tenant_not_empty, 2133, "Cannot delete a non-empty tenant" )
ERROR( invalid_tenant_name, 2134, "Tenant name cannot begin with \\xff");
ERROR( tenant_prefix_allocator_conflict, 2135, "The database already has keys stored at the prefix allocated for the tenant");
ERROR( tenants_disabled, 2136, "Tenants have been disabled in the cluster");
ERROR( unknown_tenant, 2137, "Tenant is not available from this server")

// 2200 - errors from bindings and official APIs
ERROR( api_version_unset, 2200, "API version is not set" )
ERROR( api_version_already_set, 2201, "API version may be set only once" )
ERROR( api_version_invalid, 2202, "API version not valid" )
ERROR( api_version_not_supported, 2203, "API version not supported" )
ERROR( exact_mode_without_limits, 2210, "EXACT streaming mode requires limits, but none were given" )

ERROR( invalid_tuple_data_type, 2250, "Unrecognized data type in packed tuple")
ERROR( invalid_tuple_index, 2251, "Tuple does not have element at specified index")
ERROR( key_not_in_subspace, 2252, "Cannot unpack key that is not in subspace" )
ERROR( manual_prefixes_not_enabled, 2253, "Cannot specify a prefix unless manual prefixes are enabled" )
ERROR( prefix_in_partition, 2254, "Cannot specify a prefix in a partition" )
ERROR( cannot_open_root_directory, 2255, "Root directory cannot be opened" )
ERROR( directory_already_exists, 2256, "Directory already exists" )
ERROR( directory_does_not_exist, 2257, "Directory does not exist" )
ERROR( parent_directory_does_not_exist, 2258, "Directory's parent does not exist" )
ERROR( mismatched_layer, 2259, "Directory has already been created with a different layer string" )
ERROR( invalid_directory_layer_metadata, 2260, "Invalid directory layer metadata" )
ERROR( cannot_move_directory_between_partitions, 2261, "Directory cannot be moved between partitions" )
ERROR( cannot_use_partition_as_subspace, 2262, "Directory partition cannot be used as subspace" )
ERROR( incompatible_directory_version, 2263, "Directory layer was created with an incompatible version" )
ERROR( directory_prefix_not_empty, 2264, "Database has keys stored at the prefix chosen by the automatic prefix allocator" )
ERROR( directory_prefix_in_use, 2265, "Directory layer already has a conflicting prefix" )
ERROR( invalid_destination_directory, 2266, "Target directory is invalid" )
ERROR( cannot_modify_root_directory, 2267, "Root directory cannot be modified" )
ERROR( invalid_uuid_size, 2268, "UUID is not sixteen bytes");

// 2300 - backup and restore errors
ERROR( backup_error, 2300, "Backup error")
ERROR( restore_error, 2301, "Restore error")
ERROR( backup_duplicate, 2311, "Backup duplicate request")
ERROR( backup_unneeded, 2312, "Backup unneeded request")
ERROR( backup_bad_block_size, 2313, "Backup file block size too small")
ERROR( backup_invalid_url, 2314, "Backup Container URL invalid")
ERROR( backup_invalid_info, 2315, "Backup Container info invalid")
ERROR( backup_cannot_expire, 2316, "Cannot expire requested data from backup without violating minimum restorability")
ERROR( backup_auth_missing, 2317, "Cannot find authentication details (such as a password or secret key) for the specified Backup Container URL")
ERROR( backup_auth_unreadable, 2318, "Cannot read or parse one or more sources of authentication information for Backup Container URLs")
ERROR( backup_does_not_exist, 2319, "Backup does not exist")
ERROR( backup_not_filterable_with_key_ranges, 2320, "Backup before 6.3 cannot be filtered with key ranges")
ERROR( backup_not_overlapped_with_keys_filter, 2321, "Backup key ranges doesn't overlap with key ranges filter")
ERROR( restore_invalid_version, 2361, "Invalid restore version")
ERROR( restore_corrupted_data, 2362, "Corrupted backup data")
ERROR( restore_missing_data, 2363, "Missing backup data")
ERROR( restore_duplicate_tag, 2364, "Restore duplicate request")
ERROR( restore_unknown_tag, 2365, "Restore tag does not exist")
ERROR( restore_unknown_file_type, 2366, "Unknown backup/restore file type")
ERROR( restore_unsupported_file_version, 2367, "Unsupported backup file version")
ERROR( restore_bad_read, 2368, "Unexpected number of bytes read")
ERROR( restore_corrupted_data_padding, 2369, "Backup file has unexpected padding bytes")
ERROR( restore_destination_not_empty, 2370, "Attempted to restore into a non-empty destination database")
ERROR( restore_duplicate_uid, 2371, "Attempted to restore using a UID that had been used for an aborted restore")
ERROR( task_invalid_version, 2381, "Invalid task version")
ERROR( task_interrupted, 2382, "Task execution stopped due to timeout, abort, or completion by another worker")
ERROR( invalid_encryption_key_file, 2383, "The provided encryption key file has invalid contents" )

ERROR( key_not_found, 2400, "Expected key is missing")
ERROR( json_malformed, 2401, "JSON string was malformed")
ERROR( json_eof_expected, 2402, "JSON string did not terminate where expected")

// 2500 - disk snapshot based backup errors
ERROR( snap_disable_tlog_pop_failed,  2500, "Failed to disable tlog pops")
ERROR( snap_storage_failed,  2501, "Failed to snapshot storage nodes")
ERROR( snap_tlog_failed,  2502, "Failed to snapshot TLog nodes")
ERROR( snap_coord_failed,  2503, "Failed to snapshot coordinator nodes")
ERROR( snap_enable_tlog_pop_failed,  2504, "Failed to enable tlog pops")
ERROR( snap_path_not_whitelisted, 2505, "Snapshot create binary path not whitelisted")
ERROR( snap_not_fully_recovered_unsupported, 2506, "Unsupported when the cluster is not fully recovered")
ERROR( snap_log_anti_quorum_unsupported, 2507, "Unsupported when log anti quorum is configured")
ERROR( snap_with_recovery_unsupported, 2508, "Cluster recovery during snapshot operation not supported")
ERROR( snap_invalid_uid_string, 2509, "The given uid string is not a 32-length hex string")

// 27XX - Encryption operations errors
ERROR( encrypt_ops_error, 2700, "Encryption operation error")
ERROR( encrypt_header_metadata_mismatch, 2701, "Encryption header metadata mismatch")
ERROR( encrypt_key_not_found, 2702, "Expected encryption key is missing")
ERROR( encrypt_key_ttl_expired, 2703, "Expected encryption key TTL has expired")
ERROR( encrypt_header_authtoken_mismatch, 2704, "Encryption header authentication token mismatch")
ERROR( encrypt_update_cipher, 2705, "Attempt to update encryption cipher key")
ERROR( encrypt_invalid_id, 2706, "Invalid encryption domainId or encryption cipher key id")

// 4xxx Internal errors (those that should be generated only by bugs) are decimal 4xxx
ERROR( unknown_error, 4000, "An unknown error occurred" )  // C++ exception not of type Error
ERROR( internal_error, 4100, "An internal error occurred" )
ERROR( not_implemented, 4200, "Not implemented yet" )
// clang-format on

#undef ERROR
#endif

#pragma once

#include "common/app/ApplicationBase.h"
#include "common/serde/SerdeHelper.h"
#include "common/utils/RobinHoodUtils.h"

namespace hf3fs::core {
struct EchoMessage : public serde::SerdeHelper<EchoMessage> {
  SERDE_STRUCT_FIELD(str, std::string{});
};

using EchoReq = EchoMessage;
using EchoRsp = EchoMessage;

struct GetConfigReq : public serde::SerdeHelper<GetConfigReq> {
  SERDE_STRUCT_FIELD(configKey, std::string{});
};

struct GetConfigRsp : public serde::SerdeHelper<GetConfigRsp> {
  SERDE_STRUCT_FIELD(config, std::string{});
};

struct RenderConfigReq : public serde::SerdeHelper<RenderConfigReq> {
  SERDE_STRUCT_FIELD(configTemplate, std::string{});
  SERDE_STRUCT_FIELD(testUpdate, bool{true});
  SERDE_STRUCT_FIELD(isHotUpdate, bool{true});
};

struct RenderConfigRsp : public serde::SerdeHelper<RenderConfigRsp> {
  SERDE_STRUCT_FIELD(configAfterRender, std::string{});
  SERDE_STRUCT_FIELD(updateStatus, Status(StatusCode::kOK));
  SERDE_STRUCT_FIELD(configAfterUpdate, std::string{});
};

struct HotUpdateConfigReq : public serde::SerdeHelper<HotUpdateConfigReq> {
  SERDE_STRUCT_FIELD(update, String{});
  SERDE_STRUCT_FIELD(render, false);
};

struct HotUpdateConfigRsp : public serde::SerdeHelper<HotUpdateConfigRsp> {
  SERDE_STRUCT_FIELD(dummy, Void{});
};

struct GetLastConfigUpdateRecordReq : public serde::SerdeHelper<GetLastConfigUpdateRecordReq> {
  SERDE_STRUCT_FIELD(dummy, Void{});
};

struct GetLastConfigUpdateRecordRsp : public serde::SerdeHelper<GetLastConfigUpdateRecordRsp> {
  SERDE_STRUCT_FIELD(record, std::optional<app::ConfigUpdateRecord>{});
};

DEFINE_SERDE_HELPER_STRUCT(ShutdownReq) {
  //
  SERDE_STRUCT_FIELD(graceful, true);
};

DEFINE_SERDE_HELPER_STRUCT(ShutdownRsp) { SERDE_STRUCT_FIELD(dummy, Void{}); };
}  // namespace hf3fs::core

#pragma once

#include <atomic>
#include <condition_variable>
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <mutex>

#include "fbs/storage/Common.h"
#include "storage/service/TargetMap.h"

namespace hf3fs::storage {

struct Components;

class DumpWorker {
 public:
  struct Config : ConfigBase<Config> {
    CONFIG_HOT_UPDATED_ITEM(dump_root_path, Path{});
    CONFIG_HOT_UPDATED_ITEM(dump_interval, 1_d);
    CONFIG_HOT_UPDATED_ITEM(high_cpu_usage_threshold, 100);  // num of cores.
  };

  DumpWorker(const Config &config, Components &components);

  // start dump worker.
  Result<Void> start(flat::NodeId nodeId);

  // stop dump worker. End all tasks immediately.
  Result<Void> stopAndJoin();

 protected:
  void loop();

  Result<Void> dump(const Path &rootPath);

  Result<Void> profilerStart(const Path &rootPath);

 private:
  ConstructLog<"storage::DumpWorker"> constructLog_;
  const Config &config_;
  Components &components_;
  folly::CPUThreadPoolExecutor executors_;

  std::mutex mutex_;
  std::condition_variable cond_;
  std::atomic<bool> stopping_ = false;
  std::atomic<bool> started_ = false;
  std::atomic<bool> stopped_ = false;
  flat::NodeId nodeId_{};
};

}  // namespace hf3fs::storage

#pragma once

#include <memory>

#include "FDB.h"
#include "FDBConfig.h"
#include "common/utils/ConfigBase.h"

namespace hf3fs::kv::fdb {
// FDBContext encapsulates the setup and cleanup of FoundationDB.
// It is designed to be used in the main function.
class FDBContext {
 public:
  static std::shared_ptr<FDBContext> create(const FDBConfig &config);

  ~FDBContext();

  DB getDB() const;

  int64_t maxDbCount() const { return config_.enableMultipleClient() ? config_.multipleClientThreadNum() : 1; }

 private:
  explicit FDBContext(const FDBConfig &config);
  FDBConfig config_;  // don't support hot update config
  std::thread networkThread;
};
}  // namespace hf3fs::kv::fdb

#pragma once

#include <folly/fibers/Baton.h>
#include <memory>

namespace hf3fs {
class DestructionGuard : public std::enable_shared_from_this<DestructionGuard> {
 public:
  folly::fibers::Baton destructed;

  static std::shared_ptr<DestructionGuard> create() {
    struct X : DestructionGuard {};
    auto ptr = std::make_shared<X>();
    return std::shared_ptr<DestructionGuard>(std::move(ptr));
  }

 protected:
  DestructionGuard() = default;
};
}  // namespace hf3fs

#pragma once

#ifdef ENABLE_FUSE_APPLICATION

#include "FuseAppConfig.h"
#include "FuseConfig.h"
#include "FuseConfigFetcher.h"
#include "FuseLauncherConfig.h"
#include "common/app/ApplicationBase.h"
#include "core/app/ServerLauncher.h"

namespace hf3fs::fuse {
class FuseApplication : public ApplicationBase {
 public:
  static constexpr auto kName = "Fuse";
  static constexpr auto kNodeType = flat::NodeType::FUSE;

  using AppConfig = FuseAppConfig;
  using LauncherConfig = FuseLauncherConfig;
  using RemoteConfigFetcher = FuseConfigFetcher;
  using Launcher = core::ServerLauncher<FuseApplication>;

  using Config = FuseConfig;

  FuseApplication();
  ~FuseApplication();

 private:
  Result<Void> parseFlags(int *argc, char ***argv) final;

  Result<Void> initApplication() final;

  void stop() final;

  int mainLoop() final;

  config::IConfig *getConfig() final;

  const flat::AppInfo *info() const final;

  bool configPushable() const final;

  void onConfigUpdated() final;

 private:
  Result<Void> initServer();

  Result<Void> startServer();

  struct Impl;
  std::unique_ptr<Impl> impl_;
};
}  // namespace hf3fs::fuse

#endif

#pragma once

#include <chrono>
#include <condition_variable>
#include <cstdint>
#include <mutex>

namespace hf3fs {
class SimpleSemaphore {
 public:
  struct Options {};

  SimpleSemaphore(Options opts);

  bool post();
  void wait();
  bool try_wait();
  bool try_wait_for(std::chrono::milliseconds time);

 private:
  bool fastWait();

  std::mutex mu_;
  int64_t count_{0};

  struct Waiter;
  Waiter *head = nullptr;
};
}  // namespace hf3fs

#pragma once

#include <span>

#include "TypeTraits.h"

namespace hf3fs {
template <template <typename...> typename DstContainer, typename T, typename F>
auto transformTo(std::span<T> src, F &&transformer) {
  using R = std::invoke_result_t<F, const T &>;
  if constexpr (std::is_same_v<DstContainer<R>, std::vector<R>>) {
    std::vector<R> dst;
    dst.reserve(src.size());
    std::transform(src.begin(), src.end(), std::back_inserter(dst), std::forward<F>(transformer));
    return dst;
  } else {
    DstContainer<R> dst;
    std::transform(src.begin(), src.end(), std::back_inserter(dst), std::forward<F>(transformer));
  }
}
}  // namespace hf3fs

#pragma once

#include "Schema.h"
#include "fbs/core/user/User.h"
#include "fbs/meta/Common.h"

namespace hf3fs::lib::agent {

using hf3fs::Result;

template <typename ValType>
struct OneValRsp {
  SERDE_STRUCT_TYPED_FIELD(ValType, val, ValType{});
};
using EmptyRsp = OneValRsp<Void>;

struct ProcUser {
  SERDE_STRUCT_TYPED_FIELD(ProcessInfo, proc, ProcessInfo{});
  SERDE_STRUCT_TYPED_FIELD(flat::UserInfo, userInfo, flat::UserInfo{});
};

template <bool OptionalPath>
struct PathAt {
  using PathType = std::conditional_t<OptionalPath, std::optional<Path>, Path>;
  SERDE_STRUCT_TYPED_FIELD(int, dirfd, 0);
  SERDE_STRUCT_TYPED_FIELD(PathType, path, PathType{});
};

struct StatFsReq {
  SERDE_STRUCT_TYPED_FIELD(ProcUser, procUser, ProcUser{});
  SERDE_STRUCT_TYPED_FIELD(PathAt<true>, pathAt, PathAt<true>{});
};
using StatFsRsp = OneValRsp<StatFs>;

struct StatReq {
  SERDE_STRUCT_TYPED_FIELD(ProcUser, procUser, ProcUser{});
  SERDE_STRUCT_TYPED_FIELD(PathAt<true>, pathAt, PathAt<true>{});
  SERDE_STRUCT_TYPED_FIELD(bool, followLastSymlink, false);
  SERDE_STRUCT_TYPED_FIELD(bool, ignoreCache, false);
};
using StatRsp = OneValRsp<Stat>;

struct OpenReq {
  SERDE_STRUCT_TYPED_FIELD(ProcUser, procUser, ProcUser{});
  SERDE_STRUCT_TYPED_FIELD(PathAt<false>, pathAt, PathAt<false>{});
  SERDE_STRUCT_TYPED_FIELD(int, flags, 0);
  SERDE_STRUCT_TYPED_FIELD(meta::Permission, mode, meta::Permission{});
};
using OpenRsp = OneValRsp<uint>;

struct CloseReq {
  SERDE_STRUCT_TYPED_FIELD(ProcessInfo, proc, ProcessInfo{});
  SERDE_STRUCT_TYPED_FIELD(int, fd, 0);
};
using CloseRsp = EmptyRsp;

struct SetAttrsReq {
  SERDE_STRUCT_TYPED_FIELD(ProcUser, procUser, ProcUser{});
  SERDE_STRUCT_TYPED_FIELD(PathAt<true>, pathAt, PathAt<true>{});
  SERDE_STRUCT_TYPED_FIELD(bool, followLastSymlink, false);
  SERDE_STRUCT_TYPED_FIELD(std::optional<flat::Uid>, uid, std::nullopt);
  SERDE_STRUCT_TYPED_FIELD(std::optional<flat::Gid>, gid, std::nullopt);
  SERDE_STRUCT_TYPED_FIELD(std::optional<meta::Permission>, perm, std::nullopt);
  SERDE_STRUCT_TYPED_FIELD(std::optional<UtcTime>, atime, std::nullopt);
  SERDE_STRUCT_TYPED_FIELD(std::optional<UtcTime>, mtime, std::nullopt);
};
using SetAttrsRsp = EmptyRsp;

struct DupReq {
  SERDE_STRUCT_TYPED_FIELD(ProcessInfo, proc, ProcessInfo{});
  SERDE_STRUCT_TYPED_FIELD(int, srcfd, 0);
  SERDE_STRUCT_TYPED_FIELD(int, dstfd, 0);
  SERDE_STRUCT_TYPED_FIELD(int, flags, 0);
};
using DupRsp = OneValRsp<uint>;

struct MkdirsReq {
  SERDE_STRUCT_TYPED_FIELD(ProcUser, procUser, ProcUser{});
  SERDE_STRUCT_TYPED_FIELD(PathAt<false>, pathAt, PathAt<false>{});
  SERDE_STRUCT_TYPED_FIELD(meta::Permission, perm, meta::Permission{0});
  SERDE_STRUCT_TYPED_FIELD(bool, recursive, false);
};
using MkdirsRsp = EmptyRsp;

struct SymlinkReq {
  SERDE_STRUCT_TYPED_FIELD(ProcUser, procUser, ProcUser{});
  SERDE_STRUCT_TYPED_FIELD(PathAt<false>, pathAt, PathAt<false>{});
  SERDE_STRUCT_TYPED_FIELD(Path, target, Path{});
};
using SymlinkRsp = EmptyRsp;

struct RemoveReq {
  SERDE_STRUCT_TYPED_FIELD(ProcUser, procUser, ProcUser{});
  SERDE_STRUCT_TYPED_FIELD(PathAt<false>, pathAt, PathAt<false>{});
  SERDE_STRUCT_TYPED_FIELD(bool, recursive, false);
  SERDE_STRUCT_TYPED_FIELD(std::optional<bool>, isDir, std::nullopt);
};
using RemoveRsp = EmptyRsp;

struct RenameReq {
  SERDE_STRUCT_TYPED_FIELD(ProcUser, procUser, ProcUser{});
  SERDE_STRUCT_TYPED_FIELD(PathAt<false>, srcPathAt, PathAt<false>{});
  SERDE_STRUCT_TYPED_FIELD(PathAt<false>, dstPathAt, PathAt<false>{});
};
using RenameRsp = EmptyRsp;

struct ListReq {
  SERDE_STRUCT_TYPED_FIELD(ProcUser, procUser, ProcUser{});
  SERDE_STRUCT_TYPED_FIELD(PathAt<true>, pathAt, PathAt<true>{});
  SERDE_STRUCT_TYPED_FIELD(std::optional<String>, prev, std::nullopt);
};
using ListRsp = OneValRsp<DirEntList>;

struct GetRealPathReq {
  SERDE_STRUCT_TYPED_FIELD(ProcUser, procUser, ProcUser{});
  SERDE_STRUCT_TYPED_FIELD(PathAt<false>, pathAt, PathAt<false>{});
  SERDE_STRUCT_TYPED_FIELD(bool, absolute, false);
};
using GetRealPathRsp = OneValRsp<Path>;

struct FsyncReq {
  SERDE_STRUCT_TYPED_FIELD(ProcessInfo, proc, ProcessInfo{});
  SERDE_STRUCT_TYPED_FIELD(int, fd, 0);
};
using FsyncRsp = EmptyRsp;

struct FtruncateReq {
  SERDE_STRUCT_TYPED_FIELD(ProcessInfo, proc, ProcessInfo{});
  SERDE_STRUCT_TYPED_FIELD(int, fd, 0);
  SERDE_STRUCT_TYPED_FIELD(long, length, 0);
};
using FtruncateRsp = EmptyRsp;

struct LinkReq {
  SERDE_STRUCT_TYPED_FIELD(ProcUser, procUser, ProcUser{});
  SERDE_STRUCT_TYPED_FIELD(PathAt<false>, oldPathAt, PathAt<false>{});
  SERDE_STRUCT_TYPED_FIELD(PathAt<false>, newPathAt, PathAt<false>{});
  SERDE_STRUCT_TYPED_FIELD(bool, followLastSymlink, false);
};
using LinkRsp = EmptyRsp;

struct IovAllocReq {
  SERDE_STRUCT_TYPED_FIELD(ProcUser, procUser, ProcUser{});
  SERDE_STRUCT_TYPED_FIELD(ulong, bytes, 0);
  SERDE_STRUCT_TYPED_FIELD(std::optional<int>, numa, std::nullopt);
  SERDE_STRUCT_TYPED_FIELD(bool, global, false);
  SERDE_STRUCT_TYPED_FIELD(ulong, blockSize, 0);
};
using IovAllocRsp = OneValRsp<AllocatedIov>;

struct IovFreeReq {
  SERDE_STRUCT_TYPED_FIELD(ProcUser, procUser, ProcUser{});
  SERDE_STRUCT_TYPED_FIELD(int, iovDesc, 0);
  SERDE_STRUCT_TYPED_FIELD(Uuid, iovId, Uuid{});
  SERDE_STRUCT_TYPED_FIELD(long, offInBuf, 0);
  SERDE_STRUCT_TYPED_FIELD(ulong, bytes, 0);
};
using IovFreeRsp = EmptyRsp;

struct SeekInFileReq {
  SERDE_STRUCT_TYPED_FIELD(ProcUser, procUser, ProcUser{});
  SERDE_STRUCT_TYPED_FIELD(int, fd, 0);
  SERDE_STRUCT_TYPED_FIELD(long, offset, 0);
  SERDE_STRUCT_TYPED_FIELD(int, whence, 0);
  SERDE_STRUCT_TYPED_FIELD(ulong, readahead, 0);
};
using SeekInFileRsp = OneValRsp<long>;

struct PioVReq {
  SERDE_STRUCT_TYPED_FIELD(ProcUser, procUser, ProcUser{});
  SERDE_STRUCT_TYPED_FIELD(bool, read, false);
  SERDE_STRUCT_TYPED_FIELD(std::vector<IoInfo>, iov, std::vector<IoInfo>{});
  SERDE_STRUCT_TYPED_FIELD(bool, updateOffset, false);
};
using PioVRsp = OneValRsp<PioVRes>;

struct SharedFileHandlesReq {
  SERDE_STRUCT_TYPED_FIELD(ProcessInfo, proc, ProcessInfo{});
  SERDE_STRUCT_TYPED_FIELD(std::vector<int>, fds, std::vector<int>{});
};
using SharedFileHandlesRsp = OneValRsp<String>;

struct OpenWithFileHandlesReq {
  SERDE_STRUCT_TYPED_FIELD(ProcUser, procUser, ProcUser{});
  SERDE_STRUCT_TYPED_FIELD(String, fhs, String{});
};
using OpenWithFileHandlesRsp = OneValRsp<std::vector<int>>;

struct OpenIovecHandleReq {
  SERDE_STRUCT_TYPED_FIELD(ProcUser, procUser, ProcUser{});
  SERDE_STRUCT_TYPED_FIELD(AllocatedIov, aiov, AllocatedIov{});
};
using OpenIovecHandleRsp = OneValRsp<int>;

SERDE_SERVICE(ClientAgentSerde, 10) {
  SERDE_SERVICE_METHOD(statFs, 1, StatFsReq, StatFsRsp);
  SERDE_SERVICE_METHOD(stat, 2, StatReq, StatRsp);
  SERDE_SERVICE_METHOD(open, 3, OpenReq, OpenRsp);
  SERDE_SERVICE_METHOD(close, 4, CloseReq, CloseRsp);
  SERDE_SERVICE_METHOD(setAttrs, 5, SetAttrsReq, SetAttrsRsp);
  SERDE_SERVICE_METHOD(dup, 6, DupReq, DupRsp);
  SERDE_SERVICE_METHOD(mkdirs, 7, MkdirsReq, MkdirsRsp);
  SERDE_SERVICE_METHOD(symlink, 8, SymlinkReq, SymlinkRsp);
  SERDE_SERVICE_METHOD(remove, 9, RemoveReq, RemoveRsp);
  SERDE_SERVICE_METHOD(rename, 10, RenameReq, RenameRsp);
  SERDE_SERVICE_METHOD(list, 11, ListReq, ListRsp);
  SERDE_SERVICE_METHOD(getRealPath, 12, GetRealPathReq, GetRealPathRsp);
  SERDE_SERVICE_METHOD(fsync, 13, FsyncReq, FsyncRsp);
  SERDE_SERVICE_METHOD(ftruncate, 14, FtruncateReq, FtruncateRsp);
  SERDE_SERVICE_METHOD(link, 15, LinkReq, LinkRsp);

  SERDE_SERVICE_METHOD(iovalloc, 101, IovAllocReq, IovAllocRsp);
  SERDE_SERVICE_METHOD(iovfree, 102, IovFreeReq, IovFreeRsp);
  SERDE_SERVICE_METHOD(seekInFile, 103, SeekInFileReq, SeekInFileRsp);
  SERDE_SERVICE_METHOD(piov, 104, PioVReq, PioVRsp);

  SERDE_SERVICE_METHOD(sharedFileHandles, 201, SharedFileHandlesReq, SharedFileHandlesRsp);
  SERDE_SERVICE_METHOD(openWithFileHandles, 202, OpenWithFileHandlesReq, OpenWithFileHandlesRsp);
  SERDE_SERVICE_METHOD(openIovecHandle, 203, OpenIovecHandleReq, OpenIovecHandleRsp);
};
}  // namespace hf3fs::lib::agent

#pragma once

#include "UtcTime.h"
#include "common/serde/Serde.h"

namespace hf3fs::serde {

template <>
struct SerdeMethod<UtcTime> {
  static auto serdeTo(UtcTime t) { return t.toMicroseconds(); }
  static Result<UtcTime> serdeFrom(int64_t t) { return UtcTime::fromMicroseconds(t); }
};

}  // namespace hf3fs::serde

#pragma once

#include <string>

#include "common/utils/AtomicSharedPtrTable.h"
#include "fbs/meta/Schema.h"
#include "lib/common/Shm.h"

namespace hf3fs::fuse {
class IovTable {
 public:
  IovTable() = default;
  void init(const Path &mount, int cap);
  Result<std::pair<meta::Inode, std::shared_ptr<lib::ShmBuf>>> addIov(const char *key,
                                                                      const Path &shmPath,
                                                                      pid_t pid,
                                                                      const meta::UserInfo &ui,
                                                                      folly::Executor::KeepAlive<> exec,
                                                                      storage::client::StorageClient &sc);
  Result<std::shared_ptr<lib::ShmBuf>> rmIov(const char *key, const meta::UserInfo &ui);
  Result<meta::Inode> lookupIov(const char *key, const meta::UserInfo &ui);
  std::optional<int> iovDesc(meta::InodeId iid);
  Result<meta::Inode> statIov(int key, const meta::UserInfo &ui);

 public:
  std::pair<std::shared_ptr<std::vector<meta::DirEntry>>, std::shared_ptr<std::vector<std::optional<meta::Inode>>>>
  listIovs(const meta::UserInfo &ui);

 public:
  std::string mountName;
  std::shared_mutex shmLock;
  robin_hood::unordered_map<Uuid, int> shmsById;
  std::unique_ptr<AtomicSharedPtrTable<lib::ShmBuf>> iovs;

 private:
  mutable std::shared_mutex iovdLock_;
  robin_hood::unordered_map<std::string, int> iovds_;
};
}  // namespace hf3fs::fuse

#pragma once

#include "common/serde/ClientContext.h"
#include "common/serde/MessagePacket.h"
#include "common/utils/Coroutine.h"
#include "fbs/meta/Common.h"
#include "fbs/meta/Service.h"

namespace hf3fs::meta {

class IMetaServiceStub {
 public:
  using InterfaceType = IMetaServiceStub;

  virtual ~IMetaServiceStub() = default;

#define IMETA_STUB_METHOD(NAME, REQ, RESP)                             \
  virtual CoTryTask<RESP> NAME(const REQ &req,                         \
                               const net::UserRequestOptions &options, \
                               serde::Timestamp *timestamp = nullptr) = 0

  IMETA_STUB_METHOD(statFs, StatFsReq, StatFsRsp);
  IMETA_STUB_METHOD(stat, StatReq, StatRsp);
  IMETA_STUB_METHOD(create, CreateReq, CreateRsp);
  IMETA_STUB_METHOD(mkdirs, MkdirsReq, MkdirsRsp);
  IMETA_STUB_METHOD(symlink, SymlinkReq, SymlinkRsp);
  IMETA_STUB_METHOD(hardLink, HardLinkReq, HardLinkRsp);
  IMETA_STUB_METHOD(remove, RemoveReq, RemoveRsp);
  IMETA_STUB_METHOD(open, OpenReq, OpenRsp);
  IMETA_STUB_METHOD(sync, SyncReq, SyncRsp);
  IMETA_STUB_METHOD(close, CloseReq, CloseRsp);
  IMETA_STUB_METHOD(rename, RenameReq, RenameRsp);
  IMETA_STUB_METHOD(list, ListReq, ListRsp);
  IMETA_STUB_METHOD(truncate, TruncateReq, TruncateRsp);
  IMETA_STUB_METHOD(getRealPath, GetRealPathReq, GetRealPathRsp);
  IMETA_STUB_METHOD(setAttr, SetAttrReq, SetAttrRsp);
  IMETA_STUB_METHOD(pruneSession, PruneSessionReq, PruneSessionRsp);
  IMETA_STUB_METHOD(dropUserCache, DropUserCacheReq, DropUserCacheRsp);
  IMETA_STUB_METHOD(authenticate, AuthReq, AuthRsp);
  IMETA_STUB_METHOD(lockDirectory, LockDirectoryReq, LockDirectoryRsp);
  IMETA_STUB_METHOD(testRpc, TestRpcReq, TestRpcRsp);
  IMETA_STUB_METHOD(batchStat, BatchStatReq, BatchStatRsp);
  IMETA_STUB_METHOD(batchStatByPath, BatchStatByPathReq, BatchStatByPathRsp);

#undef IMETA_STUB_METHOD
};

}  // namespace hf3fs::meta

#pragma once

#include "client/meta/MetaClient.h"
#include "client/mgmtd/MgmtdClientForClient.h"
#include "client/storage/StorageClient.h"
#include "common/app/ApplicationBase.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/CoroutinesPool.h"

namespace hf3fs::fuse {
struct FuseConfig : public ConfigBase<FuseConfig> {
#ifdef ENABLE_FUSE_APPLICATION
  CONFIG_OBJ(common, ApplicationBase::Config);
#else
  CONFIG_ITEM(cluster_id, "");
  CONFIG_ITEM(token_file, "");
  CONFIG_ITEM(mountpoint, "");
  CONFIG_ITEM(allow_other, true);
  CONFIG_OBJ(ib_devices, net::IBDevice::Config);
  CONFIG_OBJ(log, logging::LogConfig);
  CONFIG_OBJ(monitor, monitor::Monitor::Config);
#endif
  CONFIG_HOT_UPDATED_ITEM(enable_priority, false);
  CONFIG_HOT_UPDATED_ITEM(enable_interrupt, false);
  CONFIG_HOT_UPDATED_ITEM(attr_timeout, (double)30);
  CONFIG_HOT_UPDATED_ITEM(entry_timeout, (double)30);
  CONFIG_HOT_UPDATED_ITEM(negative_timeout, (double)5);
  CONFIG_HOT_UPDATED_ITEM(symlink_timeout, (double)5);
  CONFIG_HOT_UPDATED_ITEM(readonly, false);
  CONFIG_HOT_UPDATED_ITEM(memset_before_read, false);
  CONFIG_HOT_UPDATED_ITEM(enable_read_cache, true);
  CONFIG_HOT_UPDATED_ITEM(fsync_length_hint, false);  // for test
  CONFIG_HOT_UPDATED_ITEM(fdatasync_update_length, false);
  CONFIG_ITEM(max_idle_threads, 10);
  CONFIG_ITEM(max_threads, 256);
  CONFIG_ITEM(max_readahead, 16_MB);
  CONFIG_ITEM(max_background, 32);
  CONFIG_ITEM(enable_writeback_cache, false);
  CONFIG_OBJ(client, net::Client::Config);
  CONFIG_OBJ(mgmtd, client::MgmtdClientForClient::Config);
  CONFIG_OBJ(storage, storage::client::StorageClient::Config);
  CONFIG_OBJ(meta, meta::client::MetaClient::Config, [&](auto &cfg) { cfg.set_dynamic_stripe(true); });
  CONFIG_ITEM(remount_prefix, (std::optional<std::string>)std::nullopt);
  CONFIG_ITEM(iov_limit, 1_MB);
  CONFIG_ITEM(io_jobq_size, 1024);
  CONFIG_ITEM(batch_io_coros, 128);
  CONFIG_ITEM(rdma_buf_pool_size, 1024);
  CONFIG_ITEM(time_granularity, 1_s);
  CONFIG_HOT_UPDATED_ITEM(check_rmrf, true);
  CONFIG_ITEM(notify_inval_threads, 32);

  CONFIG_ITEM(max_uid, 1_M);

  CONFIG_HOT_UPDATED_ITEM(chunk_size_limit, 0_KB);

  CONFIG_SECT(io_jobq_sizes, {
    CONFIG_ITEM(hi, 32);
    CONFIG_ITEM(lo, 4096);
  });

  CONFIG_SECT(io_worker_coros, {
    CONFIG_HOT_UPDATED_ITEM(hi, 8);
    CONFIG_HOT_UPDATED_ITEM(lo, 8);
  });

  CONFIG_HOT_UPDATED_ITEM(io_job_deq_timeout, 1_ms);

  CONFIG_OBJ(storage_io, storage::client::IoOptions);

  CONFIG_HOT_UPDATED_ITEM(submit_wait_jitter, 1_ms);
  CONFIG_HOT_UPDATED_ITEM(max_jobs_per_ioring, 32);

  CONFIG_SECT(io_bufs, {
    CONFIG_ITEM(max_buf_size, 1_MB);
    CONFIG_ITEM(max_readahead, 256_KB);
    CONFIG_ITEM(write_buf_size, 1_MB);
  });

  CONFIG_HOT_UPDATED_ITEM(flush_on_stat, true);
  CONFIG_HOT_UPDATED_ITEM(sync_on_stat, true);
  CONFIG_HOT_UPDATED_ITEM(dryrun_bench_mode, false);

  struct PeriodSync : public ConfigBase<PeriodSync> {
    CONFIG_HOT_UPDATED_ITEM(enable, true);
    CONFIG_HOT_UPDATED_ITEM(interval, 30_s);
    CONFIG_HOT_UPDATED_ITEM(limit, 1000u);
    CONFIG_HOT_UPDATED_ITEM(flush_write_buf, true);
    CONFIG_OBJ(worker, CoroutinesPoolBase::Config, [](auto &cfg) { cfg.set_coroutines_num(4); });
  };
  CONFIG_OBJ(periodic_sync, PeriodSync);
};
}  // namespace hf3fs::fuse

#pragma once

#include <shared_mutex>

namespace hf3fs {

#ifndef PTHREAD_RWLOCK_WRITER_NONRECURSIVE_INITIALIZER_NP
#error "only support GNU pthread rwlock!"
#endif

class FairSharedMutex : public std::shared_mutex {
 public:
  FairSharedMutex() {
    *reinterpret_cast<pthread_rwlock_t *>(native_handle()) = PTHREAD_RWLOCK_WRITER_NONRECURSIVE_INITIALIZER_NP;
  }
};

}  // namespace hf3fs

#ifndef RAW_STATUS
#define RAW_STATUS(...)
#endif

#ifndef STATUS
#define STATUS(...)
#endif

#define COMMON_STATUS(...) RAW_STATUS(__VA_ARGS__)
#define TRANSACTION_STATUS(...) STATUS(Transaction, __VA_ARGS__)
#define RPC_STATUS(...) STATUS(RPC, __VA_ARGS__)
#define META_STATUS(...) STATUS(Meta, __VA_ARGS__)
#define STORAGE_STATUS(...) STATUS(Storage, __VA_ARGS__)
#define STORAGE_CLIENT_STATUS(...) STATUS(StorageClient, __VA_ARGS__)
#define MGMTD_STATUS(...) STATUS(Mgmtd, __VA_ARGS__)
#define MGMTD_CLIENT_STATUS(...) STATUS(MgmtdClient, __VA_ARGS__)
#define CLIENT_AGENT_STATUS(...) STATUS(ClientAgent, __VA_ARGS__)
#define CLI_STATUS(...) STATUS(Cli, __VA_ARGS__)
#define KV_SERVICE_STATUS(...) STATUS(KvService, __VA_ARGS__)

COMMON_STATUS(OK, 0)
COMMON_STATUS(NotImplemented, 1)
COMMON_STATUS(DataCorruption, 2)
COMMON_STATUS(InvalidArg, 3)
COMMON_STATUS(InvalidConfig, 4)
COMMON_STATUS(QueueEmpty, 5)
COMMON_STATUS(QueueFull, 6)
COMMON_STATUS(QueueConflict, 7)
COMMON_STATUS(QueueInvalidItem, 8)
COMMON_STATUS(MonitorInitFailed, 12)
COMMON_STATUS(MonitorQueryFailed, 13)
COMMON_STATUS(MonitorWriteFailed, 14)
COMMON_STATUS(ConfigInvalidType, 15)
COMMON_STATUS(ConfigInvalidValue, 16)
COMMON_STATUS(ConfigUpdateFailed, 17)
COMMON_STATUS(ConfigValidateFailed, 18)
COMMON_STATUS(ConfigRedundantKey, 19)
COMMON_STATUS(ConfigKeyNotFound, 20)
COMMON_STATUS(AuthenticationFail, 25)
COMMON_STATUS(NotEnoughMemory, 26)
COMMON_STATUS(Interrupted, 27)
COMMON_STATUS(InvalidFormat, 33)
COMMON_STATUS(ReadOnlyMode, 34)
COMMON_STATUS(SerdeInsufficientLength, 40)
COMMON_STATUS(SerdeMissingFieldsAtEnd, 41)
COMMON_STATUS(SerdeVariantIndexExceeded, 42)
COMMON_STATUS(SerdeUnknownEnumValue, 43)
COMMON_STATUS(SerdeNotContainer, 44)
COMMON_STATUS(SerdeNotString, 45)
COMMON_STATUS(SerdeNotNumber, 46)
COMMON_STATUS(SerdeNotInteger, 47)
COMMON_STATUS(SerdeNotTable, 48)
COMMON_STATUS(SerdeKeyNotFound, 49)
COMMON_STATUS(SerdeInvalidJson, 50)
COMMON_STATUS(SerdeInvalidToml, 51)
COMMON_STATUS(NoApplication, 52)
COMMON_STATUS(CannotPushConfig, 53)
COMMON_STATUS(KVStoreNotFound, 60)
COMMON_STATUS(KVStoreGetError, 61)
COMMON_STATUS(KVStoreSetError, 62)
COMMON_STATUS(KVStoreOpenFailed, 63)
COMMON_STATUS(TokenMismatch, 64)
COMMON_STATUS(TokenDuplicated, 65)
COMMON_STATUS(TokenStale, 66)
COMMON_STATUS(TooManyTokens, 67)
COMMON_STATUS(KVStoreIterateError, 68)
COMMON_STATUS(IOError, 69)
COMMON_STATUS(FaultInjection, 70)
COMMON_STATUS(ConfigParseError, 71)
COMMON_STATUS(OSError, 72)
COMMON_STATUS(FoundBug, 998)
COMMON_STATUS(Unknown, 999)

/* 1xxx: transaction errors. */
TRANSACTION_STATUS(Failed, 1000)
TRANSACTION_STATUS(Conflict, 1001)
TRANSACTION_STATUS(Throttled, 1002)
TRANSACTION_STATUS(TooOld, 1003)
TRANSACTION_STATUS(NetworkError, 1004)
TRANSACTION_STATUS(Canceled, 1005)
TRANSACTION_STATUS(MaybeCommitted, 1006)
TRANSACTION_STATUS(Retryable, 1007)
TRANSACTION_STATUS(ResourceConstrained, 1008)
TRANSACTION_STATUS(ProcessBehind, 1009)
TRANSACTION_STATUS(FutureVersion, 1010)

/* 2xxx: RPC errors. */
RPC_STATUS(InvalidMessageType, 2000)
RPC_STATUS(RequestIsEmpty, 2001)
RPC_STATUS(VerifyRequestFailed, 2002)
RPC_STATUS(VerifyResponseFailed, 2003)
RPC_STATUS(Timeout, 2005)
RPC_STATUS(InvalidAddr, 2006)
RPC_STATUS(SendFailed, 2007)
RPC_STATUS(InvalidServiceID, 2008)
RPC_STATUS(InvalidMethodID, 2009)
RPC_STATUS(SocketError, 2010)
RPC_STATUS(ListenFailed, 2011)
RPC_STATUS(RequestRefused, 2012)
RPC_STATUS(SocketClosed, 2013)
RPC_STATUS(ConnectFailed, 2014)
RPC_STATUS(IBInitFailed, 2015)
RPC_STATUS(IBDeviceNotFound, 2016)
RPC_STATUS(RDMAPostFailed, 2017)
RPC_STATUS(RDMAError, 2018)
RPC_STATUS(RDMANoBuf, 2019)
RPC_STATUS(InvalidServiceName, 2020)
RPC_STATUS(IBDeviceNotInitialized, 2021)
RPC_STATUS(EpollInitError, 2022)
RPC_STATUS(EpollAddError, 2023)
RPC_STATUS(EpollDelError, 2024)
RPC_STATUS(EpollWakeUpError, 2025)
RPC_STATUS(EpollWaitError, 2026)
RPC_STATUS(IBOpenPortFailed, 2027)

/* 3xxx: meta errors. */
// NOTE: update ErrorHandling in fbs/meta/Utils.h after add new error codes.
META_STATUS(NotFound, 3000)
META_STATUS(NotEmpty, 3001)
META_STATUS(NotDirectory, 3003)
META_STATUS(TooManySymlinks, 3005)
META_STATUS(IsDirectory, 3006)
META_STATUS(Exists, 3007)
META_STATUS(NoPermission, 3008)
META_STATUS(Inconsistent, 3009)
META_STATUS(NotFile, 3010)
META_STATUS(BadFileSystem, 3011)
META_STATUS(InodeIdAllocFailed, 3012)
META_STATUS(InvalidFileLayout, 3013)
META_STATUS(FileHasHole, 3014)
META_STATUS(OTruncFailed, 3015)
META_STATUS(MoreChunksToRemove, 3016)
META_STATUS(NameTooLong, 3017)
META_STATUS(RequestCanceled, 3018)
META_STATUS(Busy, 3019)
// NOTE: add expected error code under Expected
META_STATUS(Expected, 3100)
META_STATUS(NoLock, 3101)
META_STATUS(FileTooLarge, 3102)
// NOTE: add retryable error code under Retryable
META_STATUS(Retryable, 3200)
META_STATUS(ForwardFailed, 3201)
META_STATUS(ForwardTimeout, 3202)
META_STATUS(OperationTimeout, 3203)
// NOTE: add not retryable error under NotRetryable
META_STATUS(NotRetryable, 3300)
META_STATUS(FoundBug, 3999)

/* 4xxx: storage errors. */
STORAGE_STATUS(ChunkMetadataUnpackError, 4000)
STORAGE_STATUS(ChunkMetadataNotFound, 4001)
STORAGE_STATUS(ChunkMetadataGetError, 4002)
STORAGE_STATUS(ChunkMetadataSetError, 4003)
STORAGE_STATUS(ChunkNotCommit, 4004)
STORAGE_STATUS(ChunkNotClean, 4005)
STORAGE_STATUS(ChunkStaleUpdate, 4006)
STORAGE_STATUS(ChunkMissingUpdate, 4007)
STORAGE_STATUS(ChunkCommittedUpdate, 4008)
STORAGE_STATUS(ChunkOpenFailed, 4009)
STORAGE_STATUS(ChunkReadFailed, 4010)
STORAGE_STATUS(ChunkWriteFailed, 4011)
STORAGE_STATUS(ChunkAdvanceUpdate, 4012)
STORAGE_STATUS(ChunkApplyFailed, 4013)
STORAGE_STATUS(PunchHoleFailed, 4014)
STORAGE_STATUS(ChunkSizeMismatch, 4015)
STORAGE_STATUS(StorageStatFailed, 4016)
STORAGE_STATUS(StorageUUIDMismatch, 4017)
STORAGE_STATUS(StorageInitFailed, 4018)
STORAGE_STATUS(MetaStoreOpenFailed, 4020)
STORAGE_STATUS(MetaStoreInvalidIterator, 4021)
STORAGE_STATUS(ChunkNotReadyToRemove, 4022)
STORAGE_STATUS(ChunkStaleCommit, 4023)
STORAGE_STATUS(ChunkInvalidChunkSize, 4024)
STORAGE_STATUS(TargetOffline, 4030)
STORAGE_STATUS(TargetNotFound, 4031)
STORAGE_STATUS(TargetStateInvalid, 4032)
STORAGE_STATUS(NoSuccessorTarget, 4033)
STORAGE_STATUS(NoSuccessorAddr, 4034)
STORAGE_STATUS(ChunkStoreInitFailed, 4040)
STORAGE_STATUS(AioSubmitFailed, 4050)
STORAGE_STATUS(AioGetEventsFailed, 4051)
STORAGE_STATUS(BufferSizeExceeded, 4060)
STORAGE_STATUS(SyncStartFailed, 4070)
STORAGE_STATUS(SyncSendStartFailed, 4071)
STORAGE_STATUS(SyncSendDoneFailed, 4072)
STORAGE_STATUS(ChecksumMismatch, 4080)
STORAGE_STATUS(ChainVersionMismatch, 4081)
STORAGE_STATUS(ChunkVersionMismatch, 4082)
STORAGE_STATUS(ChannelIsLocked, 4090)
STORAGE_STATUS(ChunkIsLocked, 4091)

/* 5xxx: mgmtd errors. */
MGMTD_STATUS(NotPrimary, 5000)
MGMTD_STATUS(NodeNotFound, 5001)
MGMTD_STATUS(HeartbeatFail, 5002)
MGMTD_STATUS(ClusterIdMismatch, 5003)
MGMTD_STATUS(RegisterFail, 5004)
MGMTD_STATUS(InvalidRoutingInfoVersion, 5005)
MGMTD_STATUS(InvalidConfigVersion, 5006)
MGMTD_STATUS(InvalidChainTable, 5007)
MGMTD_STATUS(InvalidChainTableVersion, 5008)
MGMTD_STATUS(HeartbeatVersionStale, 5009)
MGMTD_STATUS(ClientSessionVersionStale, 5010)
MGMTD_STATUS(InvalidTag, 5011)
MGMTD_STATUS(ChainNotFound, 5012)
MGMTD_STATUS(NodeTypeMismatch, 5013)
MGMTD_STATUS(ExtendClientSessionMismatch, 5014)
MGMTD_STATUS(ClientSessionNotFound, 5015)
MGMTD_STATUS(NotAdmin, 5016)
MGMTD_STATUS(TargetNotFound, 5017)
MGMTD_STATUS(TargetExisted, 5018)

/* 6xxx: mgmtd client errors. */
MGMTD_CLIENT_STATUS(PrimaryMgmtdNotFound, 6000)
MGMTD_CLIENT_STATUS(WorkQueueFull, 6001)
MGMTD_CLIENT_STATUS(MetaServiceNotAvailable, 6002)
MGMTD_CLIENT_STATUS(Exit, 6003)  // internal used
MGMTD_CLIENT_STATUS(RoutingInfoNotReady, 6004)

/* 7xxx: storage client errors. */
STORAGE_CLIENT_STATUS(InitFailed, 7000)
STORAGE_CLIENT_STATUS(MemoryError, 7001)
STORAGE_CLIENT_STATUS(InvalidArg, 7002)
STORAGE_CLIENT_STATUS(NotInitialized, 7003)
STORAGE_CLIENT_STATUS(RoutingError, 7004)
STORAGE_CLIENT_STATUS(NotAvailable, 7005)
STORAGE_CLIENT_STATUS(CommError, 7006)
STORAGE_CLIENT_STATUS(ChunkNotFound, 7007)
STORAGE_CLIENT_STATUS(Timeout, 7008)
STORAGE_CLIENT_STATUS(BadConfig, 7009)
STORAGE_CLIENT_STATUS(RemoteIOError, 7010)
STORAGE_CLIENT_STATUS(ServerError, 7011)
STORAGE_CLIENT_STATUS(ResourceBusy, 7012)
STORAGE_CLIENT_STATUS(DuplicateUpdate, 7013)
STORAGE_CLIENT_STATUS(RoutingVersionMismatch, 7014)
STORAGE_CLIENT_STATUS(ChecksumMismatch, 7015)
STORAGE_CLIENT_STATUS(NoRDMAInterface, 7016)
STORAGE_CLIENT_STATUS(ProtocolMismatch, 7017)
STORAGE_CLIENT_STATUS(RequestCanceled, 7018)
STORAGE_CLIENT_STATUS(ReadOnlyServer, 7019)
STORAGE_CLIENT_STATUS(ChunkNotCommit, 7020)
STORAGE_CLIENT_STATUS(NoSpace, 7021)
STORAGE_CLIENT_STATUS(FoundBug, 7999)

/* 8xxx: client agent & fuse errors */
CLIENT_AGENT_STATUS(TooManyOpenFiles, 8000)
CLIENT_AGENT_STATUS(HoleInIoOutcome, 8001)
CLIENT_AGENT_STATUS(FailedToStart, 8002)
CLIENT_AGENT_STATUS(OperationDisabled, 8003)
CLIENT_AGENT_STATUS(IovNotRegistered, 8004)
CLIENT_AGENT_STATUS(IovShmFail, 8006)

/*10xxx: cli errors */
CLI_STATUS(WrongUsage, 10000)

/*11xxx: object storage errors */
KV_SERVICE_STATUS(UpdateConflict, 11000)
KV_SERVICE_STATUS(OperatingByOthers, 11001)
KV_SERVICE_STATUS(StoreNotFound, 11002)
KV_SERVICE_STATUS(StoreNotAvailable, 11003)
KV_SERVICE_STATUS(StoreLoaded, 11004)
KV_SERVICE_STATUS(ClusterIdMismatch, 11005)
KV_SERVICE_STATUS(NotPrimary, 11006)
KV_SERVICE_STATUS(TableInfoStale, 11007)
KV_SERVICE_STATUS(HeartbeatVersionStale, 11008)
KV_SERVICE_STATUS(UnknownWorker, 11009)
KV_SERVICE_STATUS(StoreLeaseEmpty, 11010)
KV_SERVICE_STATUS(StoreLeaseHeld, 11011)
KV_SERVICE_STATUS(TableNotFound, 11012)
KV_SERVICE_STATUS(TableIdMismatch, 11013)
KV_SERVICE_STATUS(NoAvailableWorker, 11014)
KV_SERVICE_STATUS(NoPermission, 11015)
KV_SERVICE_STATUS(NoAvailableMaster, 11016)
KV_SERVICE_STATUS(StoreTypeMismatch, 11017)
KV_SERVICE_STATUS(WriteStale, 11018)
KV_SERVICE_STATUS(Exists, 11019)
KV_SERVICE_STATUS(NotFound, 11020)
KV_SERVICE_STATUS(OutOfRange, 11021)
KV_SERVICE_STATUS(Deleted, 11022)
KV_SERVICE_STATUS(ServiceStopping, 11023)

#undef STATUS
#undef RAW_STATUS

#pragma once

#include <cstdalign>
#include <cstdlib>

#include "GlobalMemoryAllocator.h"

#ifdef OVERRIDE_CXX_NEW_DELETE

// Override global new/delete with custom memory allocator.
void *operator new(size_t size) { return hf3fs::memory::allocate(size); }

void operator delete(void *mem) noexcept { hf3fs::memory::deallocate(mem); }

void *operator new[](size_t size) { return hf3fs::memory::allocate(size); }

void operator delete[](void *mem) noexcept { hf3fs::memory::deallocate(mem); }

#endif

#pragma once

#include <parquet/schema.h>

#include "common/serde/Serde.h"
#include "common/utils/Coroutine.h"
#include "fbs/meta/Schema.h"
#include "fbs/storage/Common.h"

namespace hf3fs::client::cli {

class ChunkMetaRow {
  SERDE_STRUCT_FIELD(timestamp, std::time_t{});
  SERDE_STRUCT_FIELD(chainId, storage::ChainId{});
  SERDE_STRUCT_FIELD(targetId, storage::TargetId{});
  SERDE_STRUCT_FIELD(inodeId, meta::InodeId{});
  SERDE_STRUCT_FIELD(chunkmeta, storage::ChunkMeta{});
};

struct ChunkMetaTable {
  SERDE_STRUCT_FIELD(chainId, storage::ChainId{});
  SERDE_STRUCT_FIELD(targetId, storage::TargetId{});
  SERDE_STRUCT_FIELD(timestamp, std::time_t{});
  SERDE_STRUCT_FIELD(chunks, std::vector<ChunkMetaRow>{});

 public:
  bool dumpToFile(const Path &filePath, bool jsonFormat = false) const;
  bool loadFromFile(const Path &filePath, bool jsonFormat = false);
  bool dumpToParquetFile(const Path &filePath) const;
  bool loadFromParquetFile(const Path &filePath);
};
static_assert(serde::Serializable<ChunkMetaTable>);

class Dispatcher;
CoTryTask<void> registerDumpChunkMetaHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/app/ApplicationBase.h"
#include "core/utils/ServiceOperation.h"
#include "fbs/core/service/Rpc.h"

namespace hf3fs::core {

struct HotUpdateConfigOperation : ServiceOperationWithMetric<"CoreService", "HotUpdateConfig", "op"> {
  HotUpdateConfigReq req;

  explicit HotUpdateConfigOperation(HotUpdateConfigReq r)
      : req(std::move(r)) {}

  String toStringImpl() const final { return "HotUpdateConfig"; }

  CoTryTask<HotUpdateConfigRsp> handle() {
    auto res = ApplicationBase::hotUpdateConfig(req.update, req.render);
    CO_RETURN_ON_ERROR(res);
    co_return HotUpdateConfigRsp::create();
  }
};

}  // namespace hf3fs::core

#pragma once

#include "common/kv/ITransaction.h"
#include "core/user/UserCache.h"
#include "fbs/core/user/User.h"

namespace hf3fs::core {
using flat::Gid;
using flat::Uid;
using flat::UserAttr;
using flat::UserInfo;
using hf3fs::kv::IReadOnlyTransaction;
using hf3fs::kv::IReadWriteTransaction;

class UserStore {
 public:
  UserStore() = default;

  CoTryTask<std::optional<UserAttr>> getUser(IReadOnlyTransaction &txn, Uid uid);

  CoTryTask<UserAttr> addUser(IReadWriteTransaction &txn,
                              Uid uid,
                              String name,
                              std::vector<Gid> groups,
                              bool isRootUser,
                              bool isAdmin = false,
                              std::string_view token = {});

  CoTryTask<void> removeUser(IReadWriteTransaction &txn, Uid uid);

  CoTryTask<UserAttr> setUserAttr(IReadWriteTransaction &txn,
                                  Uid uid,
                                  std::string_view newName,
                                  const std::vector<Gid> *gids,
                                  std::optional<bool> isRootUser,
                                  std::optional<bool> isAdmin);

  CoTryTask<UserAttr> setUserToken(IReadWriteTransaction &txn,
                                   Uid uid,
                                   std::string_view newToken,
                                   bool invalidateExistedToken,
                                   size_t maxTokens,
                                   Duration tokenLifetimeExtendLength);

  CoTryTask<std::vector<UserAttr>> listUsers(IReadOnlyTransaction &txn);
};
}  // namespace hf3fs::core

#pragma once

#include <folly/ThreadLocal.h>
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <mutex>

#include "common/utils/CPUExecutorGroup.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/FdWrapper.h"
#include "common/utils/Path.h"
#include "common/utils/Result.h"
#include "common/utils/Shards.h"

namespace hf3fs::storage {

struct FileDescriptor {
  FdWrapper normal_;
  FdWrapper direct_;
  std::optional<uint32_t> index_{};
};

class GlobalFileStore {
 public:
  Result<FileDescriptor *> open(const Path &filePath, bool createFile = false);

  void collect(std::vector<int> &fds);

  Result<Void> clear(CPUExecutorGroup &executor);

 private:
  constexpr static auto kShardsNum = 256u;
  using FdMap = std::unordered_map<Path, FileDescriptor>;
  Shards<FdMap, kShardsNum> shards_;
};

}  // namespace hf3fs::storage

#pragma once

#include "MgmtdTypes.h"
#include "common/serde/SerdeComparisons.h"
#include "common/serde/SerdeHelper.h"

namespace hf3fs::flat {
struct ChainTargetSetting : public serde::SerdeHelper<ChainTargetSetting> {
  SERDE_STRUCT_FIELD(targetId, TargetId(0));

  auto operator<=>(const ChainTargetSetting &other) const { return serde::compare(*this, other); }
};
}  // namespace hf3fs::flat

#pragma once

#include "LauncherUtils.h"
#include "client/mgmtd/MgmtdClient.h"
#include "common/net/Client.h"

namespace hf3fs::core::launcher {
struct MgmtdClientFetcher {
  MgmtdClientFetcher(String clusterId,
                     const net::Client::Config &clientCfg,
                     const client::MgmtdClient::Config &mgmtdClientCfg);

  template <typename ConfigT>
  MgmtdClientFetcher(const ConfigT &cfg)
      : MgmtdClientFetcher(cfg.cluster_id(), cfg.client(), cfg.mgmtd_client()) {}

  virtual ~MgmtdClientFetcher() { stopClient(); }

  Result<flat::ConfigInfo> loadConfigTemplate(flat::NodeType nodeType);
  Result<Void> ensureClientInited();
  void stopClient();

  virtual Result<Void> completeAppInfo(flat::AppInfo &appInfo) = 0;

  const String clusterId_;
  const net::Client::Config &clientCfg_;
  const client::MgmtdClient::Config &mgmtdClientCfg_;
  std::unique_ptr<net::Client> client_;
  std::shared_ptr<client::MgmtdClient> mgmtdClient_;
};
}  // namespace hf3fs::core::launcher

#pragma once

#include <functional>

#include "common/utils/String.h"

namespace hf3fs::client::cli {
class Printer {
 public:
  using OutputHandler = std::function<void(const String &)>;

  explicit Printer(OutputHandler handler)
      : stdout_(std::move(handler)),
        stderr_(stdout_) {}

  Printer(OutputHandler stdoutHandler, OutputHandler stderrHandler)
      : stdout_(std::move(stdoutHandler)),
        stderr_(stderrHandler) {}

  void print(const String &s) const { stdout_(s); }
  void printError(const String &s) const { stderr_(s); }
  void print(const std::vector<std::vector<String>> &table) const;

 private:
  OutputHandler stdout_;
  OutputHandler stderr_;
};
}  // namespace hf3fs::client::cli

#pragma once

#include <boost/filesystem.hpp>
#include <filesystem>
#include <fmt/format.h>

namespace hf3fs {

using Path = boost::filesystem::path;

}  // namespace hf3fs

template <>
struct std::hash<hf3fs::Path> {
  size_t operator()(const hf3fs::Path &path) const;
};

FMT_BEGIN_NAMESPACE

template <>
struct formatter<hf3fs::Path> : formatter<std::string_view> {
  template <typename FormatContext>
  auto format(const hf3fs::Path &path, FormatContext &ctx) const {
    return formatter<std::string_view>::format(path.string(), ctx);
  }
};

FMT_END_NAMESPACE

#pragma once

#include "common/app/ApplicationBase.h"
#include "core/utils/ServiceOperation.h"
#include "fbs/core/service/Rpc.h"

namespace hf3fs::core {

struct GetConfigOperation : ServiceOperationWithMetric<"CoreService", "GetConfig", "op"> {
  GetConfigReq req;

  explicit GetConfigOperation(GetConfigReq r)
      : req(std::move(r)) {}

  String toStringImpl() const final { return "GetConfig"; }

  CoTryTask<GetConfigRsp> handle() {
    auto cfg = ApplicationBase::getConfigString(req.configKey);
    CO_RETURN_ON_ERROR(cfg);
    co_return GetConfigRsp::create(std::move(*cfg));
  }
};

}  // namespace hf3fs::core

#pragma once

#include "common/app/ApplicationBase.h"
#include "core/utils/ServiceOperation.h"
#include "fbs/core/service/Rpc.h"

namespace hf3fs::core {

struct GetLastConfigUpdateRecordOperation
    : ServiceOperationWithMetric<"CoreService", "GetLastConfigUpdateRecord", "op"> {
  GetLastConfigUpdateRecordReq req;

  explicit GetLastConfigUpdateRecordOperation(GetLastConfigUpdateRecordReq r)
      : req(std::move(r)) {}

  String toStringImpl() const final { return "GetLastConfigUpdateRecord"; }

  CoTryTask<GetLastConfigUpdateRecordRsp> handle() {
    auto res = ApplicationBase::getLastConfigUpdateRecord();
    co_return GetLastConfigUpdateRecordRsp::create(std::move(res));
  }
};

}  // namespace hf3fs::core

#pragma once

#include "common/serde/ClientContext.h"
#include "fbs/meta/Service.h"
#include "stubs/MetaService/IMetaServiceStub.h"

namespace hf3fs::meta {

template <typename Ctx>
class MetaServiceStub : public IMetaServiceStub {
 public:
  explicit MetaServiceStub(Ctx ctx)
      : context_(std::move(ctx)) {}

#define META_STUB_METHOD(NAME, REQ, RESP)                                                                             \
  CoTryTask<RESP> NAME(const REQ &req, const net::UserRequestOptions &options, serde::Timestamp *timestamp = nullptr) \
      override

  META_STUB_METHOD(statFs, StatFsReq, StatFsRsp);
  META_STUB_METHOD(stat, StatReq, StatRsp);
  META_STUB_METHOD(create, CreateReq, CreateRsp);
  META_STUB_METHOD(mkdirs, MkdirsReq, MkdirsRsp);
  META_STUB_METHOD(symlink, SymlinkReq, SymlinkRsp);
  META_STUB_METHOD(hardLink, HardLinkReq, HardLinkRsp);
  META_STUB_METHOD(remove, RemoveReq, RemoveRsp);
  META_STUB_METHOD(open, OpenReq, OpenRsp);
  META_STUB_METHOD(sync, SyncReq, SyncRsp);
  META_STUB_METHOD(close, CloseReq, CloseRsp);
  META_STUB_METHOD(rename, RenameReq, RenameRsp);
  META_STUB_METHOD(list, ListReq, ListRsp);
  META_STUB_METHOD(truncate, TruncateReq, TruncateRsp);
  META_STUB_METHOD(getRealPath, GetRealPathReq, GetRealPathRsp);
  META_STUB_METHOD(setAttr, SetAttrReq, SetAttrRsp);
  META_STUB_METHOD(pruneSession, PruneSessionReq, PruneSessionRsp);
  META_STUB_METHOD(dropUserCache, DropUserCacheReq, DropUserCacheRsp);
  META_STUB_METHOD(authenticate, AuthReq, AuthRsp);
  META_STUB_METHOD(lockDirectory, LockDirectoryReq, LockDirectoryRsp);
  META_STUB_METHOD(testRpc, TestRpcReq, TestRpcRsp);
  META_STUB_METHOD(batchStat, BatchStatReq, BatchStatRsp);
  META_STUB_METHOD(batchStatByPath, BatchStatByPathReq, BatchStatByPathRsp);

#undef META_STUB_METHOD

 private:
  Ctx context_;
};

}  // namespace hf3fs::meta

#pragma once

#include "ICommonMgmtdClient.h"
#include "fbs/mgmtd/ChainSetting.h"
#include "fbs/mgmtd/ConfigInfo.h"

namespace hf3fs::client {
class IMgmtdClientForAdmin : public ICommonMgmtdClient {
 public:
  virtual CoTryTask<mgmtd::SetChainsRsp> setChains(const flat::UserInfo &userInfo,
                                                   const std::vector<flat::ChainSetting> &chains) = 0;

  virtual CoTryTask<mgmtd::SetChainTableRsp> setChainTable(const flat::UserInfo &userInfo,
                                                           flat::ChainTableId tableId,
                                                           const std::vector<flat::ChainId> &chains,
                                                           const String &desc) = 0;

  virtual CoTryTask<flat::ConfigVersion> setConfig(const flat::UserInfo &userInfo,
                                                   flat::NodeType nodeType,
                                                   const String &content,
                                                   const String &desc) = 0;

  virtual CoTryTask<void> enableNode(const flat::UserInfo &userInfo, flat::NodeId nodeId) = 0;

  virtual CoTryTask<void> disableNode(const flat::UserInfo &userInfo, flat::NodeId nodeId) = 0;

  virtual CoTryTask<void> registerNode(const flat::UserInfo &userInfo, flat::NodeId nodeId, flat::NodeType type) = 0;

  virtual CoTryTask<flat::NodeInfo> setNodeTags(const flat::UserInfo &userInfo,
                                                flat::NodeId nodeId,
                                                const std::vector<flat::TagPair> &tags,
                                                flat::SetTagMode mode) = 0;

  virtual CoTryTask<void> unregisterNode(const flat::UserInfo &userInfo, flat::NodeId nodeId) = 0;

  virtual CoTryTask<std::vector<flat::TagPair>> setUniversalTags(const flat::UserInfo &userInfo,
                                                                 const String &universalId,
                                                                 const std::vector<flat::TagPair> &tags,
                                                                 flat::SetTagMode mode) = 0;

  virtual CoTryTask<flat::ChainInfo> rotateLastSrv(const flat::UserInfo &userInfo, flat::ChainId chainId) = 0;

  virtual CoTryTask<flat::ChainInfo> rotateAsPreferredOrder(const flat::UserInfo &userInfo, flat::ChainId chainId) = 0;

  virtual CoTryTask<flat::ChainInfo> setPreferredTargetOrder(
      const flat::UserInfo &userInfo,
      flat::ChainId chainId,
      const std::vector<flat::TargetId> &preferredTargetOrder) = 0;

  virtual CoTryTask<mgmtd::ListOrphanTargetsRsp> listOrphanTargets() = 0;

  virtual CoTryTask<flat::ChainInfo> updateChain(const flat::UserInfo &userInfo,
                                                 flat::ChainId cid,
                                                 flat::TargetId tid,
                                                 mgmtd::UpdateChainReq::Mode mode) = 0;
};
}  // namespace hf3fs::client

#pragma once

#include "MgmtdTypes.h"
#include "common/serde/SerdeHelper.h"

namespace hf3fs::flat {
struct ConfigInfo : public serde::SerdeHelper<ConfigInfo> {
  SERDE_STRUCT_FIELD(configVersion, ConfigVersion(0));
  SERDE_STRUCT_FIELD(content, String{});
  SERDE_STRUCT_FIELD(desc, String{});

 public:
  String genUpdateDesc() const { return fmt::format("version: {} desc: {}", configVersion.toUnderType(), desc); }
};
}  // namespace hf3fs::flat

#pragma once

#include <atomic>
#include <condition_variable>
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <mutex>

#include "storage/service/TargetMap.h"

namespace hf3fs::storage {

struct Components;

class SyncMetaKvWorker {
 public:
  class Config : public ConfigBase<Config> {
    CONFIG_HOT_UPDATED_ITEM(sync_meta_kv_interval, 1_min);
  };

  SyncMetaKvWorker(const Config &config, Components &components);

  Result<Void> start();

  Result<Void> stopAndJoin();

 protected:
  void loop();

  void syncAllMetaKVs();

 private:
  ConstructLog<"storage::SyncMetaKvWorker"> constructLog_;
  const Config &config_;
  Components &components_;
  folly::CPUThreadPoolExecutor executors_;

  std::mutex mutex_;
  std::condition_variable cond_;
  std::atomic<bool> stopping_ = false;
  std::atomic<bool> started_ = false;
  std::atomic<bool> stopped_ = false;
};

}  // namespace hf3fs::storage

#pragma once

#include <sys/types.h>
#include <sys/vfs.h>

#include "common/serde/Serde.h"
#include "common/utils/Uuid.h"

namespace hf3fs::lib::agent {
struct ProcessInfo {
  SERDE_STRUCT_TYPED_FIELD(int, pid, 0);
  SERDE_STRUCT_TYPED_FIELD(int, ppid, 0);
};

struct TimeSpec {
  SERDE_STRUCT_TYPED_FIELD(time_t, sec, 0);
  SERDE_STRUCT_TYPED_FIELD(long, nsec, 0);
};

struct Stat {
  SERDE_STRUCT_TYPED_FIELD(dev_t, dev, 0);
  SERDE_STRUCT_TYPED_FIELD(ino_t, ino, 0);
  SERDE_STRUCT_TYPED_FIELD(mode_t, mode, 0);
  SERDE_STRUCT_TYPED_FIELD(nlink_t, nlink, 0);
  SERDE_STRUCT_TYPED_FIELD(uid_t, uid, 0);
  SERDE_STRUCT_TYPED_FIELD(gid_t, gid, 0);
  SERDE_STRUCT_TYPED_FIELD(dev_t, rdev, 0);
  SERDE_STRUCT_TYPED_FIELD(off_t, size, 0);
  SERDE_STRUCT_TYPED_FIELD(blksize_t, blksize, 0);
  SERDE_STRUCT_TYPED_FIELD(blkcnt_t, blocks, 0);
  SERDE_STRUCT_TYPED_FIELD(TimeSpec, atim, TimeSpec{});
  SERDE_STRUCT_TYPED_FIELD(TimeSpec, mtim, TimeSpec{});
  SERDE_STRUCT_TYPED_FIELD(TimeSpec, ctim, TimeSpec{});
  SERDE_STRUCT_TYPED_FIELD(std::optional<String>, ltarg, std::nullopt);
  SERDE_STRUCT_TYPED_FIELD(Path, path, Path{"/"});
};

struct StatFs {
  SERDE_STRUCT_TYPED_FIELD(__fsword_t, type, 0);
  SERDE_STRUCT_TYPED_FIELD(__fsword_t, bsize, 0);
  SERDE_STRUCT_TYPED_FIELD(fsblkcnt_t, blocks, 0);
  SERDE_STRUCT_TYPED_FIELD(fsblkcnt_t, bfree, 0);
  SERDE_STRUCT_TYPED_FIELD(fsblkcnt_t, bavail, 0);
  SERDE_STRUCT_TYPED_FIELD(fsfilcnt_t, files, 0);
  SERDE_STRUCT_TYPED_FIELD(fsfilcnt_t, ffree, 0);
  SERDE_STRUCT_TYPED_FIELD(long, fsid, 0);
  SERDE_STRUCT_TYPED_FIELD(__fsword_t, namelen, 0);
  SERDE_STRUCT_TYPED_FIELD(__fsword_t, frsize, 0);
  SERDE_STRUCT_TYPED_FIELD(__fsword_t, flags, 0);
};

struct DirEnt {
  SERDE_STRUCT_TYPED_FIELD(unsigned char, type, 0);
  SERDE_STRUCT_TYPED_FIELD(String, name, "");
};

struct DirEntList {
  SERDE_STRUCT_TYPED_FIELD(std::vector<DirEnt>, dents, std::vector<DirEnt>{});
  SERDE_STRUCT_TYPED_FIELD(bool, hasMore, false);
};

struct SharedFileHandles {
  SERDE_STRUCT_TYPED_FIELD(std::vector<String>, fhs, std::vector<String>{});
};

struct AllocatedIov {
  SERDE_STRUCT_TYPED_FIELD(int, iovDesc, 0);
  SERDE_STRUCT_TYPED_FIELD(Path, path, Path{});
  SERDE_STRUCT_TYPED_FIELD(long, offInBuf, 0);
  SERDE_STRUCT_TYPED_FIELD(ulong, bytes, 0);
  SERDE_STRUCT_TYPED_FIELD(Uuid, id, Uuid{});
  SERDE_STRUCT_TYPED_FIELD(ulong, blockSize, 0);
};

struct IoInfo {
  SERDE_STRUCT_TYPED_FIELD(int, iovDesc, 0);
  SERDE_STRUCT_TYPED_FIELD(Uuid, iovId, Uuid{});
  SERDE_STRUCT_TYPED_FIELD(long, iovOff, 0);
  SERDE_STRUCT_TYPED_FIELD(ulong, iovLen, 0);

  SERDE_STRUCT_TYPED_FIELD(int, fd, 0);
  SERDE_STRUCT_TYPED_FIELD(long, off, 0);
  SERDE_STRUCT_TYPED_FIELD(ulong, readahead, 0);

  SERDE_STRUCT_TYPED_FIELD(uint16_t, track, 0);
};

struct PioVRes {
  SERDE_STRUCT_TYPED_FIELD(std::vector<long>, res, std::vector<long>{});
  SERDE_STRUCT_TYPED_FIELD(std::vector<long>, offs, std::vector<long>{});
};
}  // namespace hf3fs::lib::agent

#pragma once

#include "common/serde/Service.h"
#include "fbs/storage/Common.h"

namespace hf3fs::storage {

SERDE_SERVICE(StorageSerde, 3) {
  SERDE_SERVICE_METHOD(batchRead, 1, BatchReadReq, BatchReadRsp);
  SERDE_SERVICE_METHOD(write, 2, WriteReq, WriteRsp);
  SERDE_SERVICE_METHOD(update, 3, UpdateReq, UpdateRsp);
  SERDE_SERVICE_METHOD(queryLastChunk, 5, QueryLastChunkReq, QueryLastChunkRsp);
  SERDE_SERVICE_METHOD(truncateChunks, 6, TruncateChunksReq, TruncateChunksRsp);
  SERDE_SERVICE_METHOD(removeChunks, 7, RemoveChunksReq, RemoveChunksRsp);
  SERDE_SERVICE_METHOD(syncStart, 8, SyncStartReq, TargetSyncInfo);
  SERDE_SERVICE_METHOD(syncDone, 9, SyncDoneReq, SyncDoneRsp);
  SERDE_SERVICE_METHOD(spaceInfo, 10, SpaceInfoReq, SpaceInfoRsp);
  SERDE_SERVICE_METHOD(createTarget, 11, CreateTargetReq, CreateTargetRsp);
  SERDE_SERVICE_METHOD(queryChunk, 12, QueryChunkReq, QueryChunkRsp);
  SERDE_SERVICE_METHOD(getAllChunkMetadata, 13, GetAllChunkMetadataReq, GetAllChunkMetadataRsp);
  SERDE_SERVICE_METHOD(offlineTarget, 16, OfflineTargetReq, OfflineTargetRsp);
  SERDE_SERVICE_METHOD(removeTarget, 17, RemoveTargetReq, RemoveTargetRsp);
};

}  // namespace hf3fs::storage

#pragma once

#include "PersistentNodeInfo.h"
#include "common/utils/UtcTimeSerde.h"

namespace hf3fs::flat {
struct MgmtdLeaseInfo : public serde::SerdeHelper<MgmtdLeaseInfo> {
  MgmtdLeaseInfo() = default;
  MgmtdLeaseInfo(PersistentNodeInfo primaryInfo, UtcTime leaseStart, UtcTime leaseEnd)
      : primary(std::move(primaryInfo)),
        leaseStart(leaseStart),
        leaseEnd(leaseEnd),
        releaseVersion(ReleaseVersion::fromVersionInfo()) {}
  MgmtdLeaseInfo(PersistentNodeInfo primaryInfo, UtcTime leaseStart, UtcTime leaseEnd, ReleaseVersion rv)
      : primary(std::move(primaryInfo)),
        leaseStart(leaseStart),
        leaseEnd(leaseEnd),
        releaseVersion(rv) {}

  SERDE_STRUCT_FIELD(primary, PersistentNodeInfo{});
  SERDE_STRUCT_FIELD(leaseStart, UtcTime{});
  SERDE_STRUCT_FIELD(leaseEnd, UtcTime{});
  SERDE_STRUCT_FIELD(releaseVersion, ReleaseVersion());
};
}  // namespace hf3fs::flat

#pragma once

#include "common/app/NodeId.h"
#include "common/net/ib/IBDevice.h"
#include "common/utils/ConfigBase.h"

namespace hf3fs::core {
struct ServerAppConfig : public ConfigBase<ServerAppConfig> {
  CONFIG_ITEM(node_id, 0);
  CONFIG_ITEM(allow_empty_node_id, true);

 public:
  using Base = ConfigBase<ServerAppConfig>;
  using Base::init;

  void init(const String &filePath, bool dump, const std::vector<config::KeyValue> &updates);

  flat::NodeId getNodeId() const { return flat::NodeId(node_id()); }
};
}  // namespace hf3fs::core

#pragma once

#include "MgmtdTypes.h"
#include "common/app/AppInfo.h"
#include "common/serde/SerdeComparisons.h"
#include "common/serde/SerdeHelper.h"

namespace hf3fs::flat {
struct PersistentNodeInfo : public serde::SerdeHelper<PersistentNodeInfo> {
  bool operator==(const PersistentNodeInfo &other) const { return serde::equals(*this, other); }

  SERDE_STRUCT_FIELD(nodeId, NodeId(0));
  SERDE_STRUCT_FIELD(type, NodeType(NodeType::MIN));
  SERDE_STRUCT_FIELD(serviceGroups, std::vector<ServiceGroupInfo>{});
  SERDE_STRUCT_FIELD(tags, std::vector<TagPair>{});
  SERDE_STRUCT_FIELD(hostname, String{});
};
}  // namespace hf3fs::flat

#pragma once

#include "common/serde/CallContext.h"
#include "common/serde/Serde.h"
#include "common/serde/Service.h"

namespace hf3fs::serde::echo {

struct Message {
  SERDE_STRUCT_FIELD(str, std::string{});
};

SERDE_SERVICE(Service, 10000) {
  // send back which received.
  SERDE_SERVICE_METHOD(echo, 1, Message, Message);
};

struct ServiceImpl : ServiceWrapper<ServiceImpl, Service> {
  CoTryTask<Message> echo(CallContext &, const Message &req) { co_return req; }
};

}  // namespace hf3fs::serde::echo

#pragma once

#include "ChainInfo.h"
#include "common/serde/SerdeComparisons.h"

namespace hf3fs::flat {
struct ChainTable : public serde::SerdeHelper<ChainTable> {
  bool operator==(const ChainTable &other) const { return serde::equals(*this, other); }

  SERDE_STRUCT_FIELD(chainTableId, ChainTableId(0));
  SERDE_STRUCT_FIELD(chainTableVersion, ChainTableVersion(0));
  SERDE_STRUCT_FIELD(chains, std::vector<ChainId>{});
  SERDE_STRUCT_FIELD(desc, String{});
};
}  // namespace hf3fs::flat

#pragma once

#include "MgmtdTypes.h"
#include "common/serde/SerdeComparisons.h"
#include "common/serde/SerdeHelper.h"

namespace hf3fs::flat {
struct ChainTargetInfo : public serde::SerdeHelper<ChainTargetInfo> {
  bool operator==(const ChainTargetInfo &other) const { return serde::equals(*this, other); }

  SERDE_STRUCT_FIELD(targetId, TargetId(0));
  SERDE_STRUCT_FIELD(publicState, PublicTargetState(PublicTargetState::INVALID));
};
}  // namespace hf3fs::flat

#include "common/utils/Coroutine.h"
#include "common/utils/Result.h"

#ifdef DEFINE_FBS_SERVICE
#undef DEFINE_FBS_SERVICE
#endif

#define DEFINE_FBS_SERVICE(name, fbsns)                   \
  template <typename Ctx>                                 \
  class name##ServiceStub : public I##name##ServiceStub { \
   public:                                                \
    using ContextType = Ctx;                              \
    using name##Client = ::fbsns::name##Client<Ctx>;      \
                                                          \
    explicit name##ServiceStub(Ctx ctx)                   \
        : client_(std::move(ctx)) {}

#ifdef FINISH_FBS_SERVICE
#undef FINISH_FBS_SERVICE
#endif

#define FINISH_FBS_SERVICE(name, fbsns) \
 private:                               \
  name##Client client_;                 \
  }

#ifdef DEFINE_FBS_SERVICE_METHOD
#undef DEFINE_FBS_SERVICE_METHOD
#endif

#define DEFINE_FBS_SERVICE_METHOD(svc, name, reqtype, rsptype, flatns) \
  CoTryTask<flatns::rsptype> name(const flatns::reqtype &req) override

#ifdef DEFINE_FBS_SERVICE_METHOD_VOID
#undef DEFINE_FBS_SERVICE_METHOD_VOID
#endif

#define DEFINE_FBS_SERVICE_METHOD_VOID(svc, name, reqtype, rsptype, flatns) \
  DEFINE_FBS_SERVICE_METHOD(svc, name, reqtype, rsptype, flatns)

#ifdef DEFINE_FBS_SERVICE_METHOD_RETURNS
#undef DEFINE_FBS_SERVICE_METHOD_RETURNS
#endif

#define DEFINE_FBS_SERVICE_METHOD_RETURNS(svc, name, reqtype, rsptype, flatns, returns, rtype) \
  DEFINE_FBS_SERVICE_METHOD(svc, name, reqtype, rsptype, flatns)

#pragma once

#include "ClientSessionData.h"
#include "common/app/ConfigStatus.h"
#include "common/utils/UtcTimeSerde.h"

namespace hf3fs::mgmtd {
struct ExtendClientSessionReq;
}

namespace hf3fs::flat {
struct ClientSession : public serde::SerdeHelper<ClientSession> {
  SERDE_STRUCT_FIELD(clientId, String{});
  SERDE_STRUCT_FIELD(configVersion, ConfigVersion(0));
  SERDE_STRUCT_FIELD(start, UtcTime{});
  SERDE_STRUCT_FIELD(lastExtend, UtcTime{});
  SERDE_STRUCT_FIELD(universalId, String{});
  SERDE_STRUCT_FIELD(description, String{});
  SERDE_STRUCT_FIELD(serviceGroups, std::vector<ServiceGroupInfo>{});
  SERDE_STRUCT_FIELD(releaseVersion, ReleaseVersion{});
  SERDE_STRUCT_FIELD(configStatus, ConfigStatus::NORMAL);
  SERDE_STRUCT_FIELD(type, flat::NodeType::CLIENT);
  SERDE_STRUCT_FIELD(clientStart, UtcTime{});

 public:
  ClientSession() = default;
  explicit ClientSession(const mgmtd::ExtendClientSessionReq &req, UtcTime now = UtcClock::now());
};
}  // namespace hf3fs::flat

#pragma once

#include "common/app/NodeId.h"
#include "common/net/ib/IBDevice.h"
#include "common/utils/ConfigBase.h"

namespace hf3fs::fuse {
struct FuseAppConfig : public ConfigBase<FuseAppConfig> {
 public:
  using Base = ConfigBase<FuseAppConfig>;
  using Base::init;

  void init(const String &filePath, bool dump, const std::vector<config::KeyValue> &updates);
  flat::NodeId getNodeId() const { return flat::NodeId(0); }
};
}  // namespace hf3fs::fuse

#pragma once

#include "common/app/ApplicationBase.h"
#include "common/utils/suicide.h"
#include "core/utils/ServiceOperation.h"
#include "fbs/core/service/Rpc.h"

namespace hf3fs::core {

struct ShutdownOperation : ServiceOperationWithMetric<"CoreService", "Shutdown", "op"> {
  ShutdownReq req_;

  explicit ShutdownOperation(ShutdownReq req)
      : req_(std::move(req)) {}

  String toStringImpl() const final { return "Shutdown"; }

  CoTryTask<ShutdownRsp> handle() {
    CO_RETURN_ON_ERROR(suicide(req_.graceful));
    co_return ShutdownRsp::create();
  }
};

}  // namespace hf3fs::core

#include "common/utils/Result.h"

#ifdef DEFINE_FBS_SERVICE
#undef DEFINE_FBS_SERVICE
#endif

#define DEFINE_FBS_SERVICE(name, fbsns)                   \
  template <typename Ctx>                                 \
  class name##ServiceStub : public I##name##ServiceStub { \
   public:                                                \
    using ContextType = Ctx;                              \
    using name##Client = ::fbsns::name##Client<Ctx>;      \
                                                          \
    explicit name##ServiceStub(Ctx ctx)                   \
        : client_(std::move(ctx)) {}

#ifdef FINISH_FBS_SERVICE
#undef FINISH_FBS_SERVICE
#endif

#define FINISH_FBS_SERVICE(name, fbsns) \
 private:                               \
  name##Client client_;                 \
  }

#ifdef DEFINE_FBS_SERVICE_METHOD
#undef DEFINE_FBS_SERVICE_METHOD
#endif

#define DEFINE_FBS_SERVICE_METHOD(svc, name, reqtype, rsptype, flatns) \
  hf3fs::Result<flatns::rsptype> name(const flatns::reqtype &req) override

#ifdef DEFINE_FBS_SERVICE_METHOD_VOID
#undef DEFINE_FBS_SERVICE_METHOD_VOID
#endif

#define DEFINE_FBS_SERVICE_METHOD_VOID(svc, name, reqtype, rsptype, flatns) \
  hf3fs::Result<Void> name(const flatns::reqtype &req) override

#ifdef DEFINE_FBS_SERVICE_METHOD_RETURNS
#undef DEFINE_FBS_SERVICE_METHOD_RETURNS
#endif

#define DEFINE_FBS_SERVICE_METHOD_RETURNS(svc, name, reqtype, rsptype, flatns, returns, rtype) \
  hf3fs::Result<rtype> name(const flatns::reqtype &req) override

#pragma once

#include "common/utils/Result.h"
#include "common/utils/Status.h"
#include "common/utils/StatusCodeDetails.h"

namespace hf3fs {

class StatusCodeConversion {
 public:
  static Status convertToStorageClientCode(Status status);

  template <typename T>
  static Result<T> convertToStorageClientCode(const Result<T> &result) {
    if (UNLIKELY(result.hasError())) {
      return makeError(convertToStorageClientCode(result.error()));
    }

    return result;
  }
};

}  // namespace hf3fs

#pragma once

#include <folly/Executor.h>
#include <variant>

#include "CPUExecutorGroup.h"
#include "Coroutine.h"
#include "CountDownLatch.h"
#include "Duration.h"

namespace hf3fs {
class BackgroundRunner {
 public:
  explicit BackgroundRunner(folly::Executor::KeepAlive<> executor);
  explicit BackgroundRunner(CPUExecutorGroup &executor);

  ~BackgroundRunner();

  using TaskGetter = std::function<CoTask<void>()>;
  using IntervalGetter = std::function<Duration()>;
  bool start(String taskName, TaskGetter taskGetter, IntervalGetter intervalGetter);
  bool startOnce(String taskName, TaskGetter taskGetter);
  CoTask<void> stopAll();

 private:
  folly::Executor::KeepAlive<> getExecutor();
  CoTask<void> run(String taskName, TaskGetter taskGetter, IntervalGetter intervalGetter);

  bool stopping_ = false;
  std::variant<folly::Executor::KeepAlive<>, CPUExecutorGroup *> executor_;
  CancellationSource cancel_;
  CountDownLatch<> latch_;
  std::mutex mutex_;
};
}  // namespace hf3fs

#pragma once

#include "MemKV.h"
#include "MemTransaction.h"
#include "common/kv/IKVEngine.h"

namespace hf3fs::kv {

class MemKVEngine : public IKVEngine {
 public:
  MemKVEngine() = default;

  std::unique_ptr<IReadOnlyTransaction> createReadonlyTransaction() override { return createReadWriteTransaction(); }

  std::unique_ptr<IReadWriteTransaction> createReadWriteTransaction() override {
    return std::make_unique<MemTransaction>(mem_);
  }

  mem::MemKV &getKV() { return mem_; }

 private:
  mem::MemKV mem_;
};

}  // namespace hf3fs::kv

#pragma once

#include <cstdint>
#include <string_view>

namespace hf3fs {

struct VersionInfo {
  static std::string_view fullV0();
  static std::string_view full();
  static uint8_t versionMajor();
  static uint8_t versionMinor();
  static uint8_t versionPatch();
  static uint64_t buildTimeInSeconds();
  static uint32_t buildPipelineId();

  static uint32_t commitHashShort();
  static std::string_view commitHashFull();

  static uint32_t gitTag();
  static uint32_t gitTagSeqNum();
  static bool isReleaseBranch();
};

}  // namespace hf3fs

#pragma once

#include "common/serde/CallContext.h"
#include "common/serde/Service.h"
#include "common/utils/Result.h"
#include "fbs/meta/Service.h"

namespace hf3fs::meta {

class MockMetaService : public serde::ServiceWrapper<MockMetaService, MetaSerde> {
 public:
  virtual ~MockMetaService() = default;

#define META_MOCK_SERVICE_METHOD(NAME, REQ, RESP)                         \
  virtual CoTryTask<RESP> NAME(serde::CallContext &ctx, const REQ &req) { \
    co_return makeError(StatusCode::kNotImplemented);                     \
  }

  META_MOCK_SERVICE_METHOD(statFs, StatFsReq, StatFsRsp);
  META_MOCK_SERVICE_METHOD(stat, StatReq, StatRsp);
  META_MOCK_SERVICE_METHOD(create, CreateReq, CreateRsp);
  META_MOCK_SERVICE_METHOD(mkdirs, MkdirsReq, MkdirsRsp);
  META_MOCK_SERVICE_METHOD(symlink, SymlinkReq, SymlinkRsp);
  META_MOCK_SERVICE_METHOD(hardLink, HardLinkReq, HardLinkRsp);
  META_MOCK_SERVICE_METHOD(remove, RemoveReq, RemoveRsp);
  META_MOCK_SERVICE_METHOD(open, OpenReq, OpenRsp);
  META_MOCK_SERVICE_METHOD(sync, SyncReq, SyncRsp);
  META_MOCK_SERVICE_METHOD(close, CloseReq, CloseRsp);
  META_MOCK_SERVICE_METHOD(rename, RenameReq, RenameRsp);
  META_MOCK_SERVICE_METHOD(list, ListReq, ListRsp);
  META_MOCK_SERVICE_METHOD(truncate, TruncateReq, TruncateRsp);
  META_MOCK_SERVICE_METHOD(getRealPath, GetRealPathReq, GetRealPathRsp);
  META_MOCK_SERVICE_METHOD(setAttr, SetAttrReq, SetAttrRsp);
  META_MOCK_SERVICE_METHOD(pruneSession, PruneSessionReq, PruneSessionRsp);
  META_MOCK_SERVICE_METHOD(dropUserCache, DropUserCacheReq, DropUserCacheRsp);
  META_MOCK_SERVICE_METHOD(authenticate, AuthReq, AuthRsp);
  META_MOCK_SERVICE_METHOD(lockDirectory, LockDirectoryReq, LockDirectoryRsp);
  META_MOCK_SERVICE_METHOD(testRpc, TestRpcReq, TestRpcRsp);
  META_MOCK_SERVICE_METHOD(batchStat, BatchStatReq, BatchStatRsp);
  META_MOCK_SERVICE_METHOD(batchStatByPath, BatchStatByPathReq, BatchStatByPathRsp);
};
}  // namespace hf3fs::meta

#pragma once

#include "MgmtdTypes.h"
#include "common/app/AppInfo.h"
#include "common/serde/SerdeComparisons.h"
#include "common/serde/SerdeHelper.h"

namespace hf3fs::flat {
struct ClientSessionData : public serde::SerdeHelper<ClientSessionData> {
  bool operator==(const ClientSessionData &other) const { return serde::equals(*this, other); }

  SERDE_STRUCT_FIELD(universalId, String{});
  SERDE_STRUCT_FIELD(description, String{});
  SERDE_STRUCT_FIELD(serviceGroups, std::vector<ServiceGroupInfo>{});
  SERDE_STRUCT_FIELD(releaseVersion, ReleaseVersion{});
};
}  // namespace hf3fs::flat

#pragma once

#include "ICommonMgmtdClient.h"
#include "fbs/mgmtd/Rpc.h"

namespace hf3fs::client {
class IMgmtdClientForServer : public ICommonMgmtdClient {
 public:
  // manual calls
  virtual CoTryTask<mgmtd::HeartbeatRsp> heartbeat() = 0;

  virtual Result<Void> triggerHeartbeat() = 0;

  // must set before any manual or auto heartbeat
  virtual void setAppInfoForHeartbeat(flat::AppInfo info) = 0;

  // return whether this configuration change is succeeded
  using ConfigListener = std::function<hf3fs::Result<hf3fs::Void>(const String &, const String &)>;
  virtual void setConfigListener(ConfigListener listener) = 0;

  using HeartbeatPayload = flat::HeartbeatInfo::Payload;
  virtual void updateHeartbeatPayload(HeartbeatPayload payload) = 0;
};
}  // namespace hf3fs::client

#pragma once

#include <type_traits>

namespace hf3fs::thief {
namespace detail {

template <typename Tag>
struct Bridge {
#if defined(__GNUC__) && !defined(__clang__)
// Silence unnecessary warning
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wnon-template-friend"
#endif
  friend consteval auto ADL(Bridge<Tag>);
#if defined(__GNUC__) && !defined(__clang__)
#pragma GCC diagnostic pop
#endif
};

template <typename Tag, typename Store>
struct StealType {
  friend consteval auto ADL(Bridge<Tag>) { return std::type_identity<Store>{}; }
};

}  // namespace detail

template <typename Tag, typename Store>
using steal = decltype(detail::StealType<Tag, Store>{});

template <typename Tag>
using retrieve = typename decltype(ADL(detail::Bridge<Tag>{}))::type;

}  // namespace hf3fs::thief

#pragma once

#include "core/utils/ServiceOperation.h"
#include "fbs/core/service/Rpc.h"

namespace hf3fs::core {

struct EchoOperation : ServiceOperationWithMetric<"CoreService", "Echo", "op"> {
  EchoMessage req;

  explicit EchoOperation(EchoMessage r)
      : req(std::move(r)) {}

  String toStringImpl() const final { return "Echo"; }

  CoTryTask<EchoMessage> handle() { co_return req; }
};

}  // namespace hf3fs::core

#pragma once

#include "common/serde/CallContext.h"
#include "fbs/meta/Service.h"
#include "meta/service/MetaOperator.h"

namespace hf3fs::meta::server {

class MetaSerdeService : public serde::ServiceWrapper<MetaSerdeService, MetaSerde> {
 public:
  MetaSerdeService(MetaOperator &meta)
      : meta_(meta) {}

#define META_SERVICE_METHOD(NAME, REQ, RESP) \
  CoTryTask<RESP> NAME(serde::CallContext &, const REQ &req) { return meta_.NAME(req); }

  META_SERVICE_METHOD(statFs, StatFsReq, StatFsRsp);
  META_SERVICE_METHOD(stat, StatReq, StatRsp);
  META_SERVICE_METHOD(create, CreateReq, CreateRsp);
  META_SERVICE_METHOD(mkdirs, MkdirsReq, MkdirsRsp);
  META_SERVICE_METHOD(symlink, SymlinkReq, SymlinkRsp);
  META_SERVICE_METHOD(hardLink, HardLinkReq, HardLinkRsp);
  META_SERVICE_METHOD(remove, RemoveReq, RemoveRsp);
  META_SERVICE_METHOD(open, OpenReq, OpenRsp);
  META_SERVICE_METHOD(sync, SyncReq, SyncRsp);
  META_SERVICE_METHOD(close, CloseReq, CloseRsp);
  META_SERVICE_METHOD(rename, RenameReq, RenameRsp);
  META_SERVICE_METHOD(list, ListReq, ListRsp);
  META_SERVICE_METHOD(truncate, TruncateReq, TruncateRsp);
  META_SERVICE_METHOD(getRealPath, GetRealPathReq, GetRealPathRsp);
  META_SERVICE_METHOD(setAttr, SetAttrReq, SetAttrRsp);
  META_SERVICE_METHOD(pruneSession, PruneSessionReq, PruneSessionRsp);
  META_SERVICE_METHOD(dropUserCache, DropUserCacheReq, DropUserCacheRsp);
  META_SERVICE_METHOD(authenticate, AuthReq, AuthRsp);
  META_SERVICE_METHOD(lockDirectory, LockDirectoryReq, LockDirectoryRsp);
  META_SERVICE_METHOD(testRpc, TestRpcReq, TestRpcRsp);
  META_SERVICE_METHOD(batchStat, BatchStatReq, BatchStatRsp);
  META_SERVICE_METHOD(batchStatByPath, BatchStatByPathReq, BatchStatByPathRsp);
#undef META_SERVICE_METHOD

 private:
  MetaOperator &meta_;
};

}  // namespace hf3fs::meta::server

#pragma once

#include "core/utils/ServiceOperation.h"
#include "fbs/mgmtd/Rpc.h"
#include "mgmtd/service/MgmtdState.h"

namespace hf3fs::mgmtd {
struct ListClientSessionsOperation : core::ServiceOperationWithMetric<"MgmtdService", "ListClientSessions", "op"> {
  ListClientSessionsReq req;

  explicit ListClientSessionsOperation(ListClientSessionsReq r)
      : req(std::move(r)) {}

  String toStringImpl() const final { return fmt::format("ListClientSessions"); }

  CoTryTask<ListClientSessionsRsp> handle(MgmtdState &state);
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "core/utils/ServiceOperation.h"
#include "fbs/mgmtd/Rpc.h"
#include "mgmtd/service/MgmtdState.h"

namespace hf3fs::mgmtd {
struct ListOrphanTargetsOperation : core::ServiceOperationWithMetric<"MgmtdService", "ListOrphanTargets", "op"> {
  ListOrphanTargetsReq req;

  explicit ListOrphanTargetsOperation(ListOrphanTargetsReq r)
      : req(std::move(r)) {}

  String toStringImpl() const final { return fmt::format("ListOrphanTargets"); }

  CoTryTask<ListOrphanTargetsRsp> handle(MgmtdState &state);
};
}  // namespace hf3fs::mgmtd

#pragma once

#include <cstdint>

namespace hf3fs {
enum class ConfigStatus : uint8_t { NORMAL = 0, DIRTY = 1, FAILED = 2, UNKNOWN = 3, STALE = 4 };
}

#pragma once

#include "ICommonMgmtdClient.h"
#include "fbs/mgmtd/ClientSessionData.h"

namespace hf3fs::client {
class IMgmtdClientForClient : public ICommonMgmtdClient {
 public:
  virtual CoTryTask<void> extendClientSession() = 0;

  using ConfigListener = std::function<hf3fs::Result<hf3fs::Void>(const String &, const String &)>;
  virtual void setConfigListener(ConfigListener listener) = 0;

  struct ClientSessionPayload {
    String clientId;
    flat::NodeType nodeType = flat::NodeType::CLIENT;
    flat::ClientSessionData data;
    flat::UserInfo userInfo;
  };

  virtual void setClientSessionPayload(ClientSessionPayload payload) = 0;
};
}  // namespace hf3fs::client

#pragma once

#include "WithTimestamp.h"
#include "fbs/mgmtd/TargetInfo.h"

namespace hf3fs::mgmtd {
class TargetInfo : public WithTimestamp<flat::TargetInfo> {
 public:
  using Base = WithTimestamp<flat::TargetInfo>;
  using Base::Base;

  bool locationInitLoaded = false;
  std::optional<flat::NodeId> persistedNodeId;
  std::optional<uint32_t> persistedDiskIndex;

  SteadyTime importantInfoChangedTime = SteadyClock::now();
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "common/kv/ITransaction.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Result.h"
#include "fbs/core/user/User.h"

namespace hf3fs::core {
String encodeUserToken(uint32_t uid, uint64_t timestamp);
CoTryTask<String> encodeUserToken(uint32_t uid, kv::IReadOnlyTransaction &txn);

Result<std::pair<uint32_t, uint64_t>> decodeUserToken(std::string_view token);
Result<flat::Uid> decodeUidFromUserToken(std::string_view token);
}  // namespace hf3fs::core

#pragma once

#include "common/serde/CallContext.h"
#include "fbs/migration/SerdeService.h"

namespace hf3fs::migration::server {

class MigrationService : public serde::ServiceWrapper<MigrationService, MigrationSerde> {
 public:
  MigrationService();

#define DECLARE_SERVICE_METHOD(METHOD, REQ, RESP) CoTryTask<RESP> METHOD(serde::CallContext &, const REQ &req)

  DECLARE_SERVICE_METHOD(start, StartJobReq, StartJobRsp);
  DECLARE_SERVICE_METHOD(stop, StopJobReq, StopJobRsp);
  DECLARE_SERVICE_METHOD(list, ListJobsReq, ListJobsRsp);

#undef DECLARE_SERVICE_METHOD

 private:
};

}  // namespace hf3fs::migration::server

#pragma once

#include "common/monitor/Reporter.h"
#include "common/monitor/Sample.h"
#include "common/utils/ConfigBase.h"

namespace clickhouse {
class Client;
}  // namespace clickhouse

namespace hf3fs::monitor {

class ClickHouseClient : public Reporter {
 public:
  struct Config : ConfigBase<Config> {
    CONFIG_ITEM(host, "");
    CONFIG_ITEM(port, "");
    CONFIG_ITEM(user, "");
    CONFIG_ITEM(passwd, "");
    CONFIG_ITEM(db, "");
  };

  ClickHouseClient(const Config &config);
  ~ClickHouseClient() override;

  Result<Void> init() final;

  Result<Void> commit(const std::vector<Sample> &samples) final;

 private:
  const Config &config_;
  std::unique_ptr<clickhouse::Client> client_;
  bool errorHappened_{false};
};

}  // namespace hf3fs::monitor

#pragma once

#include <optional>
#include <string>
#include <string_view>
#include <vector>

#include "common/kv/ITransaction.h"
#include "common/utils/Coroutine.h"
#include "common/utils/UtcTime.h"
#include "fbs/mgmtd/ChainInfo.h"
#include "fbs/mgmtd/ChainTable.h"
#include "fbs/mgmtd/ConfigInfo.h"
#include "fbs/mgmtd/MgmtdLeaseInfo.h"
#include "fbs/mgmtd/PersistentNodeInfo.h"
#include "fbs/mgmtd/RoutingInfo.h"

namespace hf3fs::mgmtd {
class MgmtdStore {
 public:
  MgmtdStore() = default;

  // return current lease
  CoTryTask<flat::MgmtdLeaseInfo> extendLease(kv::IReadWriteTransaction &txn,
                                              const flat::PersistentNodeInfo &nodeInfo,
                                              std::chrono::microseconds leaseLength,
                                              UtcTime now,
                                              flat::ReleaseVersion rv = flat::ReleaseVersion::fromVersionInfo(),
                                              bool checkReleaseVersion = true);
  CoTryTask<void> ensureLeaseValid(kv::IReadOnlyTransaction &txn, flat::NodeId nodeId, UtcTime now);

  CoTryTask<void> storeNodeInfo(kv::IReadWriteTransaction &txn, const flat::PersistentNodeInfo &info);
  CoTryTask<void> clearNodeInfo(kv::IReadWriteTransaction &txn, flat::NodeId nodeId);
  CoTryTask<std::optional<flat::PersistentNodeInfo>> loadNodeInfo(kv::IReadOnlyTransaction &txn,
                                                                  flat::NodeId nodeId,
                                                                  bool snapshotLoad = false);

  CoTryTask<flat::RoutingInfoVersion> loadRoutingInfoVersion(kv::IReadOnlyTransaction &txn);
  CoTryTask<void> storeRoutingInfoVersion(kv::IReadWriteTransaction &txn, flat::RoutingInfoVersion version);

  CoTryTask<std::vector<flat::PersistentNodeInfo>> loadAllNodes(kv::IReadOnlyTransaction &txn);

  CoTryTask<std::optional<flat::MgmtdLeaseInfo>> loadMgmtdLeaseInfo(kv::IReadOnlyTransaction &txn);

  CoTryTask<flat::MgmtdLeaseInfo> storeMgmtdLeaseInfo(kv::IReadWriteTransaction &txn,
                                                      const flat::MgmtdLeaseInfo &leaseInfo);

  CoTryTask<void> storeConfig(kv::IReadWriteTransaction &txn, flat::NodeType nodeType, const flat::ConfigInfo &info);

  CoTryTask<std::vector<std::pair<flat::NodeType, flat::ConfigInfo>>> loadAllConfigs(kv::IReadOnlyTransaction &txn);

  CoTryTask<std::optional<flat::ConfigInfo>> loadLatestConfig(kv::IReadOnlyTransaction &txn, flat::NodeType type);

  CoTryTask<std::optional<flat::ConfigInfo>> loadConfig(kv::IReadOnlyTransaction &txn,
                                                        flat::NodeType type,
                                                        flat::ConfigVersion version);

  CoTryTask<void> storeChainTable(kv::IReadWriteTransaction &txn, const flat::ChainTable &chainTable);

  CoTryTask<std::vector<flat::ChainTable>> loadAllChainTables(kv::IReadOnlyTransaction &txn);

  CoTryTask<void> storeChainInfo(kv::IReadWriteTransaction &txn, const flat::ChainInfo &chainInfo);

  CoTryTask<std::vector<flat::ChainInfo>> loadAllChains(kv::IReadOnlyTransaction &txn);

  // test only
  CoTryTask<void> storeRoutingInfo(kv::IReadWriteTransaction &txn, const flat::RoutingInfo &routingInfo);

  CoTryTask<void> storeUniversalTags(kv::IReadWriteTransaction &txn,
                                     std::string_view id,
                                     const std::vector<flat::TagPair> &tags);

  CoTryTask<std::vector<flat::TagPair>> loadUniversalTags(kv::IReadOnlyTransaction &txn, std::string_view id);

  CoTryTask<std::vector<std::pair<String, std::vector<flat::TagPair>>>> loadAllUniversalTags(
      kv::IReadOnlyTransaction &txn);

  CoTryTask<void> clearUniversalTags(kv::IReadWriteTransaction &txn, std::string_view id);

  CoTryTask<void> shutdownAllChains(kv::IReadWriteTransaction &txn);

  CoTryTask<std::optional<flat::TargetInfo>> loadTargetInfo(kv::IReadOnlyTransaction &txn, flat::TargetId tid);

  CoTryTask<void> storeTargetInfo(kv::IReadWriteTransaction &txn, const flat::TargetInfo &ti);

  CoTryTask<void> clearTargetInfo(kv::IReadWriteTransaction &txn, flat::TargetId tid);

  CoTryTask<std::vector<flat::TargetInfo>> loadTargetsFrom(kv::IReadOnlyTransaction &txn, flat::TargetId tid);
};
}  // namespace hf3fs::mgmtd

#pragma once

#include <atomic>
#include <condition_variable>
#include <folly/executors/CPUThreadPoolExecutor.h>
#include <mutex>

#include "storage/service/TargetMap.h"

namespace hf3fs::storage {

struct Components;

class PunchHoleWorker {
 public:
  PunchHoleWorker(Components &components);

  // start recycle worker.
  Result<Void> start();

  // stop recycle worker. End all sync tasks immediately.
  Result<Void> stopAndJoin();

 protected:
  void loop();

 private:
  ConstructLog<"storage::PunchHoleWorker"> constructLog_;
  Components &components_;
  folly::CPUThreadPoolExecutor executors_;

  std::mutex mutex_;
  std::condition_variable cond_;
  std::atomic<bool> stopping_ = false;
  std::atomic<bool> started_ = false;
  std::atomic<bool> stopped_ = false;
};

}  // namespace hf3fs::storage

#pragma once

#include <common/utils/Result.h>
#include <folly/Unit.h>
#include <folly/experimental/coro/Task.h>
#include <folly/experimental/coro/WithCancellation.h>
#include <type_traits>

namespace hf3fs {
template <typename T>
using CoTask = folly::coro::Task<T>;

template <typename T>
using CoTryTask = CoTask<std::conditional_t<std::is_void_v<T>, hf3fs::Result<Void>, hf3fs::Result<T>>>;

template <typename T>
struct IsCoTask : std::false_type {};

template <typename T>
struct IsCoTask<CoTask<T>> : std::true_type {};

template <typename T>
inline constexpr bool IsCoTaskV = IsCoTask<T>::value;

template <typename T>
struct RemoveCoroutine {
  using type = T;
};

template <typename T>
struct RemoveCoroutine<CoTask<T>> {
  using type = T;
};

using CancellationToken = folly::CancellationToken;
using CancellationSource = folly::CancellationSource;
using OperationCancelled = folly::OperationCancelled;

}  // namespace hf3fs

#pragma once

#include <algorithm>
#include <string_view>

namespace hf3fs {
template <size_t N>
struct NameWrapper {
  constexpr NameWrapper(const char (&str)[N]) { std::copy_n(str, N, string); }
  constexpr operator std::string_view() const { return {string, N - 1}; }
  char string[N];
};
}  // namespace hf3fs

#pragma once

#include "ChainSetting.h"
#include "ClientSession.h"
#include "ClientSessionData.h"
#include "ConfigInfo.h"
#include "HeartbeatInfo.h"
#include "NodeInfo.h"
#include "PersistentNodeInfo.h"
#include "RoutingInfo.h"
#include "common/app/ConfigStatus.h"
#include "fbs/core/user/User.h"

namespace hf3fs::mgmtd {
DEFINE_SERDE_HELPER_STRUCT(GetPrimaryMgmtdReq) {
  SERDE_STRUCT_FIELD(clusterId, String{});
  SERDE_STRUCT_FIELD(user, flat::UserInfo{});
};

DEFINE_SERDE_HELPER_STRUCT(GetPrimaryMgmtdRsp) {
  SERDE_STRUCT_FIELD(primary, std::optional<flat::PersistentNodeInfo>{});
};

DEFINE_SERDE_HELPER_STRUCT(HeartbeatReq) {
  SERDE_STRUCT_FIELD(clusterId, String{});
  SERDE_STRUCT_FIELD(info, flat::HeartbeatInfo{});
  SERDE_STRUCT_FIELD(timestamp, UtcTime{});
  SERDE_STRUCT_FIELD(user, flat::UserInfo{});
};

DEFINE_SERDE_HELPER_STRUCT(HeartbeatRsp) { SERDE_STRUCT_FIELD(config, std::optional<flat::ConfigInfo>{}); };

DEFINE_SERDE_HELPER_STRUCT(RegisterNodeReq) {
  SERDE_STRUCT_FIELD(clusterId, String{});
  SERDE_STRUCT_FIELD(nodeId, flat::NodeId(0));
  SERDE_STRUCT_FIELD(type, flat::NodeType(flat::NodeType::MIN));
  SERDE_STRUCT_FIELD(user, flat::UserInfo{});
};

DEFINE_SERDE_HELPER_STRUCT(RegisterNodeRsp) { SERDE_STRUCT_FIELD(dummy, Void{}); };

DEFINE_SERDE_HELPER_STRUCT(GetRoutingInfoReq) {
  SERDE_STRUCT_FIELD(clusterId, String{});
  SERDE_STRUCT_FIELD(routingInfoVersion, flat::RoutingInfoVersion(0));
  SERDE_STRUCT_FIELD(user, flat::UserInfo{});
};

DEFINE_SERDE_HELPER_STRUCT(GetRoutingInfoRsp) { SERDE_STRUCT_FIELD(info, std::optional<flat::RoutingInfo>{}); };

DEFINE_SERDE_HELPER_STRUCT(SetConfigReq) {
  SERDE_STRUCT_FIELD(clusterId, String{});
  SERDE_STRUCT_FIELD(nodeType, flat::NodeType(flat::NodeType::MIN));
  SERDE_STRUCT_FIELD(content, String{});
  SERDE_STRUCT_FIELD(desc, String{});
  SERDE_STRUCT_FIELD(user, flat::UserInfo{});
};

DEFINE_SERDE_HELPER_STRUCT(SetConfigRsp) { SERDE_STRUCT_FIELD(configVersion, flat::ConfigVersion(0)); };

DEFINE_SERDE_HELPER_STRUCT(GetConfigReq) {
  SERDE_STRUCT_FIELD(clusterId, String{});
  SERDE_STRUCT_FIELD(nodeType, flat::NodeType(flat::NodeType::MIN));
  SERDE_STRUCT_FIELD(configVersion, flat::ConfigVersion(0));
  SERDE_STRUCT_FIELD(exactVersion, bool{false});
  SERDE_STRUCT_FIELD(user, flat::UserInfo{});
};

DEFINE_SERDE_HELPER_STRUCT(GetConfigRsp) { SERDE_STRUCT_FIELD(info, std::optional<flat::ConfigInfo>{}); };

DEFINE_SERDE_HELPER_STRUCT(SetChainTableReq) {
  SERDE_STRUCT_FIELD(clusterId, String{});
  SERDE_STRUCT_FIELD(chainTableId, flat::ChainTableId(0));
  SERDE_STRUCT_FIELD(chains, std::vector<flat::ChainId>{});
  SERDE_STRUCT_FIELD(desc, String{});
  SERDE_STRUCT_FIELD(user, flat::UserInfo{});
};

DEFINE_SERDE_HELPER_STRUCT(SetChainTableRsp) { SERDE_STRUCT_FIELD(chainTableVersion, flat::ChainTableVersion(0)); };

DEFINE_SERDE_HELPER_STRUCT(EnableNodeReq) {
  SERDE_STRUCT_FIELD(clusterId, String{});
  SERDE_STRUCT_FIELD(nodeId, flat::NodeId(0));
  SERDE_STRUCT_FIELD(user, flat::UserInfo{});
};

DEFINE_SERDE_HELPER_STRUCT(EnableNodeRsp) { SERDE_STRUCT_FIELD(dummy, Void{}); };

DEFINE_SERDE_HELPER_STRUCT(DisableNodeReq) {
  SERDE_STRUCT_FIELD(clusterId, String{});
  SERDE_STRUCT_FIELD(nodeId, flat::NodeId(0));
  SERDE_STRUCT_FIELD(user, flat::UserInfo{});
};

DEFINE_SERDE_HELPER_STRUCT(DisableNodeRsp) { SERDE_STRUCT_FIELD(dummy, Void{}); };

DEFINE_SERDE_HELPER_STRUCT(ExtendClientSessionReq) {
  SERDE_STRUCT_FIELD(clusterId, String{});
  SERDE_STRUCT_FIELD(clientId, String{});
  SERDE_STRUCT_FIELD(clientSessionVersion, flat::ClientSessionVersion(1));
  SERDE_STRUCT_FIELD(configVersion, flat::ConfigVersion(0));
  SERDE_STRUCT_FIELD(data, flat::ClientSessionData{});
  SERDE_STRUCT_FIELD(configStatus, ConfigStatus::NORMAL);
  SERDE_STRUCT_FIELD(type, flat::NodeType::CLIENT);
  SERDE_STRUCT_FIELD(user, flat::UserInfo{});
  SERDE_STRUCT_FIELD(clientStart, UtcTime{});
};

DEFINE_SERDE_HELPER_STRUCT(ExtendClientSessionRsp) {
  SERDE_STRUCT_FIELD(config, std::optional<flat::ConfigInfo>{});
  SERDE_STRUCT_FIELD(tags, std::vector<flat::TagPair>{});
};

DEFINE_SERDE_HELPER_STRUCT(ListClientSessionsReq) {
  SERDE_STRUCT_FIELD(clusterId, String{});
  SERDE_STRUCT_FIELD(user, flat::UserInfo{});
};

DEFINE_SERDE_HELPER_STRUCT(ListClientSessionsRsp) {
  SERDE_STRUCT_FIELD(bootstrapping, bool(false));
  SERDE_STRUCT_FIELD(sessions, std::vector<flat::ClientSession>{});
  SERDE_STRUCT_FIELD(referencedTags, RHStringHashMap<std::vector<flat::TagPair>>{});
};

DEFINE_SERDE_HELPER_STRUCT(SetNodeTagsReq) {
  SERDE_STRUCT_FIELD(clusterId, String{});
  SERDE_STRUCT_FIELD(nodeId, flat::NodeId(0));
  SERDE_STRUCT_FIELD(tags, std::vector<flat::TagPair>{});
  SERDE_STRUCT_FIELD(mode, flat::SetTagMode(flat::SetTagMode::REPLACE));
  SERDE_STRUCT_FIELD(user, flat::UserInfo{});
};

DEFINE_SERDE_HELPER_STRUCT(SetNodeTagsRsp) { SERDE_STRUCT_FIELD(info, flat::NodeInfo{}); };

DEFINE_SERDE_HELPER_STRUCT(UnregisterNodeReq) {
  SERDE_STRUCT_FIELD(clusterId, String{});
  SERDE_STRUCT_FIELD(nodeId, flat::NodeId(0));
  SERDE_STRUCT_FIELD(user, flat::UserInfo{});
};

DEFINE_SERDE_HELPER_STRUCT(UnregisterNodeRsp) { SERDE_STRUCT_FIELD(dummy, Void{}); };

DEFINE_SERDE_HELPER_STRUCT(SetChainsReq) {
  SERDE_STRUCT_FIELD(clusterId, String{});
  SERDE_STRUCT_FIELD(chains, std::vector<flat::ChainSetting>{});
  SERDE_STRUCT_FIELD(user, flat::UserInfo{});
};

DEFINE_SERDE_HELPER_STRUCT(SetChainsRsp) { SERDE_STRUCT_FIELD(dummy, Void{}); };

DEFINE_SERDE_HELPER_STRUCT(SetUniversalTagsReq) {
  SERDE_STRUCT_FIELD(clusterId, String{});
  SERDE_STRUCT_FIELD(universalId, String{});
  SERDE_STRUCT_FIELD(tags, std::vector<flat::TagPair>{});
  SERDE_STRUCT_FIELD(mode, flat::SetTagMode(flat::SetTagMode::REPLACE));
  SERDE_STRUCT_FIELD(user, flat::UserInfo{});
};

DEFINE_SERDE_HELPER_STRUCT(SetUniversalTagsRsp) { SERDE_STRUCT_FIELD(tags, std::vector<flat::TagPair>{}); };

DEFINE_SERDE_HELPER_STRUCT(GetUniversalTagsReq) {
  SERDE_STRUCT_FIELD(clusterId, String{});
  SERDE_STRUCT_FIELD(universalId, String{});
  SERDE_STRUCT_FIELD(user, flat::UserInfo{});
};

DEFINE_SERDE_HELPER_STRUCT(GetUniversalTagsRsp) { SERDE_STRUCT_FIELD(tags, std::vector<flat::TagPair>{}); };

DEFINE_SERDE_HELPER_STRUCT(GetConfigVersionsReq) {
  SERDE_STRUCT_FIELD(clusterId, String{});
  SERDE_STRUCT_FIELD(user, flat::UserInfo{});
};

DEFINE_SERDE_HELPER_STRUCT(GetConfigVersionsRsp) {
  SERDE_STRUCT_FIELD(versions, RHStringHashMap<flat::ConfigVersion>{});
};

DEFINE_SERDE_HELPER_STRUCT(GetClientSessionReq) {
  SERDE_STRUCT_FIELD(clusterId, String{});
  SERDE_STRUCT_FIELD(clientId, String{});
  SERDE_STRUCT_FIELD(user, flat::UserInfo{});
};

DEFINE_SERDE_HELPER_STRUCT(GetClientSessionRsp) {
  SERDE_STRUCT_FIELD(bootstrapping, bool(false));
  SERDE_STRUCT_FIELD(session, std::optional<flat::ClientSession>{});
  SERDE_STRUCT_FIELD(referencedTags, std::vector<flat::TagPair>{});
};

DEFINE_SERDE_HELPER_STRUCT(RotateLastSrvReq) {
  SERDE_STRUCT_FIELD(clusterId, String{});
  SERDE_STRUCT_FIELD(chainId, flat::ChainId(0));
  SERDE_STRUCT_FIELD(user, flat::UserInfo{});
};

DEFINE_SERDE_HELPER_STRUCT(RotateLastSrvRsp) { SERDE_STRUCT_FIELD(chain, flat::ChainInfo{}); };

DEFINE_SERDE_HELPER_STRUCT(ListOrphanTargetsReq) {
  SERDE_STRUCT_FIELD(clusterId, String{});
  SERDE_STRUCT_FIELD(user, flat::UserInfo{});
};

DEFINE_SERDE_HELPER_STRUCT(ListOrphanTargetsRsp) { SERDE_STRUCT_FIELD(targets, std::vector<flat::TargetInfo>()); };

DEFINE_SERDE_HELPER_STRUCT(SetPreferredTargetOrderReq) {
  SERDE_STRUCT_FIELD(clusterId, String{});
  SERDE_STRUCT_FIELD(chainId, flat::ChainId{0});
  SERDE_STRUCT_FIELD(preferredTargetOrder, std::vector<flat::TargetId>{});
  SERDE_STRUCT_FIELD(user, flat::UserInfo{});
};

DEFINE_SERDE_HELPER_STRUCT(SetPreferredTargetOrderRsp) { SERDE_STRUCT_FIELD(chain, flat::ChainInfo{}); };

DEFINE_SERDE_HELPER_STRUCT(RotateAsPreferredOrderReq) {
  SERDE_STRUCT_FIELD(clusterId, String{});
  SERDE_STRUCT_FIELD(chainId, flat::ChainId(0));
  SERDE_STRUCT_FIELD(user, flat::UserInfo{});
};

DEFINE_SERDE_HELPER_STRUCT(RotateAsPreferredOrderRsp) { SERDE_STRUCT_FIELD(chain, flat::ChainInfo{}); };

DEFINE_SERDE_HELPER_STRUCT(UpdateChainReq) {
  enum class Mode : uint8_t {
    ADD = 0,
    REMOVE = 1,
  };
  SERDE_STRUCT_FIELD(clusterId, String{});
  SERDE_STRUCT_FIELD(user, flat::UserInfo{});
  SERDE_STRUCT_FIELD(chainId, flat::ChainId(0));
  SERDE_STRUCT_FIELD(targetId, flat::TargetId(0));
  SERDE_STRUCT_FIELD(mode, Mode::ADD);
};

DEFINE_SERDE_HELPER_STRUCT(UpdateChainRsp) { SERDE_STRUCT_FIELD(chain, flat::ChainInfo{}); };

}  // namespace hf3fs::mgmtd

#pragma once

#include "common/serde/CallContext.h"
#include "fbs/simple_example/SerdeService.h"

namespace hf3fs::simple_example::server {

class SimpleExampleService : public serde::ServiceWrapper<SimpleExampleService, SimpleExampleSerde> {
 public:
  SimpleExampleService();

#define DECLARE_SERVICE_METHOD(METHOD, REQ, RESP) CoTryTask<RESP> METHOD(serde::CallContext &, const REQ &req)

  DECLARE_SERVICE_METHOD(echo, SimpleExampleReq, SimpleExampleRsp);

#undef DECLARE_SERVICE_METHOD

 private:
};

}  // namespace hf3fs::simple_example::server

#pragma once

#include <folly/Range.h>
#include <folly/logging/LogHandler.h>
#include <folly/logging/LogHandlerFactory.h>
#include <memory>

namespace hf3fs::logging {
class FileHandlerFactory : public folly::LogHandlerFactory {
 public:
  folly::StringPiece getType() const override { return "file"; }

  std::shared_ptr<folly::LogHandler> createHandler(const Options &options) override;
};

class EventLogHandlerFactory : public folly::LogHandlerFactory {
 public:
  folly::StringPiece getType() const override { return "event"; }

  std::shared_ptr<folly::LogHandler> createHandler(const Options &options) override;
};

}  // namespace hf3fs::logging

#pragma once

#include <cassert>
#include <folly/Random.h>
#include <span>
#include <vector>

namespace hf3fs {

struct RandomUtils {
  template <typename T>
  static T randomSelect(const std::vector<T> &vec) {
    assert(!vec.empty());
    return vec[folly::Random::rand64(vec.size())];
  }
};

}  // namespace hf3fs

#pragma once

#include <optional>

#include "ConfigStatus.h"
#include "ConfigUpdateRecord.h"

namespace hf3fs::config {
struct IConfig;
}

namespace hf3fs::flat {
struct AppInfo;
}

namespace hf3fs::app {
struct ConfigManager {
  void updateConfigRecord(Status status, String desc);
  void updateConfigContent(config::IConfig &cfg,
                           const flat::AppInfo *appInfo,
                           std::string_view configContent,
                           bool checkDiff = true);
  void updateConfigStatus(config::IConfig &cfg, const flat::AppInfo *appInfo);

  Result<Void> updateConfig(const String &configContent,
                            std::string_view configDesc,
                            config::IConfig &cfg,
                            const flat::AppInfo *appInfo,
                            bool hotUpdate = true,
                            bool checkDiff = true);

  Result<Void> hotUpdateConfig(const String &update, bool render, config::IConfig &cfg, const flat::AppInfo *appInfo);

  std::optional<ConfigUpdateRecord> lastConfigUpdateRecord;
  String lastConfigContent;
  ConfigStatus configStatus = ConfigStatus::NORMAL;
};
}  // namespace hf3fs::app

#pragma once

#include "ICoreServiceStub.h"

namespace hf3fs::core {
template <typename Ctx>
class CoreServiceStub : public ICoreServiceStub {
 public:
  using ContextType = Ctx;
  explicit CoreServiceStub(ContextType ctx)
      : ctx_(std::move(ctx)) {}

#define DEFINE_SERDE_SERVICE_METHOD_FULL(svc, name, Name, id, reqtype, rsptype) \
  CoTryTask<rsptype> name(const reqtype &req) override;

#include "fbs/core/service/CoreServiceDef.h"

 private:
  ContextType ctx_;
};
}  // namespace hf3fs::core

#pragma once

#include "common/app/AppInfo.h"
#include "common/kv/IKVEngine.h"
#include "fbs/mgmtd/ConfigInfo.h"
#include "fdb/HybridKvEngineConfig.h"

namespace hf3fs::mgmtd {
class MgmtdServer;
struct MgmtdConfigFetcher {
  template <typename ConfigT>
  explicit MgmtdConfigFetcher(const ConfigT &cfg) {
    init(cfg.kv_engine(), cfg.use_memkv(), cfg.fdb());
  }

  Result<flat::ConfigInfo> loadConfigTemplate(flat::NodeType nodeType);
  Result<Void> completeAppInfo(flat::AppInfo &appInfo);

  Result<Void> startServer(MgmtdServer &server, const flat::AppInfo &appInfo);

 private:
  void init(const kv::HybridKvEngineConfig &config, bool useMemKV, const kv::fdb::FDBConfig &fdbConfig);

  std::shared_ptr<kv::IKVEngine> kvEngine_;
};
}  // namespace hf3fs::mgmtd

#pragma once

#include <folly/logging/StandardLogHandlerFactory.h>
#include <optional>

namespace hf3fs::logging {
class FileWriterFactory : public folly::StandardLogHandlerFactory::WriterFactory {
 public:
  bool processOption(folly::StringPiece name, folly::StringPiece value) override;

  std::shared_ptr<folly::LogWriter> createWriter() override;

 private:
  std::string path_;
  bool async_{true};
  bool rotate_{false};
  size_t maxBufferSize_{0};
  size_t maxFileSize_{0};
  size_t maxFiles_{0};
  std::optional<bool> rotateOnOpen_;
};

}  // namespace hf3fs::logging

#pragma once

#include "common/serde/Serde.h"
#include "common/serde/Service.h"

namespace hf3fs::simple_example {

struct SimpleExampleReq {
  SERDE_STRUCT_FIELD(message, String{});
};

struct SimpleExampleRsp {
  SERDE_STRUCT_FIELD(message, String{});
};

SERDE_SERVICE(SimpleExampleSerde, 0xF0) { SERDE_SERVICE_METHOD(echo, 1, SimpleExampleReq, SimpleExampleRsp); };

}  // namespace hf3fs::simple_example

#pragma once

#include <folly/Synchronized.h>

#include "ClientSession.h"
#include "LeaseInfo.h"
#include "RoutingInfo.h"
#include "common/utils/RobinHoodUtils.h"
#include "fbs/mgmtd/ConfigInfo.h"
#include "fbs/mgmtd/MgmtdTypes.h"
#include "fbs/mgmtd/RoutingInfo.h"

namespace hf3fs::core {
struct ServiceOperation;
}

namespace hf3fs::mgmtd {
using VersionedConfigMap = std::map<flat::ConfigVersion, flat::ConfigInfo>;
using ConfigMap = std::map<flat::NodeType, VersionedConfigMap>;
using UniversalTagsMap = RHStringHashMap<std::vector<flat::TagPair>>;

struct MgmtdConfig;

struct MgmtdData {
  SteadyTime leaseStartTs;
  RoutingInfo routingInfo;
  ConfigMap configMap;
  LeaseInfo lease;
  UniversalTagsMap universalTagsMap;

  mutable folly::Synchronized<std::optional<flat::RoutingInfo>> routingInfoCache;

  Result<Void> checkConfigVersion(core::ServiceOperation &ctx, flat::NodeType, flat::ConfigVersion version) const;
  std::optional<flat::ConfigInfo> getConfig(flat::NodeType, flat::ConfigVersion version, bool latest) const;
  flat::ConfigVersion getLatestConfigVersion(flat::NodeType) const;

  Result<Void> checkRoutingInfoVersion(core::ServiceOperation &ctx, flat::RoutingInfoVersion version) const;
  std::optional<flat::RoutingInfo> getRoutingInfo(flat::RoutingInfoVersion version, const MgmtdConfig &config) const;

  void appendChangedChains(flat::ChainId chainId,
                           std::vector<flat::ChainInfo> &changedChains,
                           bool tryAdjustTargetOrder) const;

  void reset(flat::RoutingInfoVersion routingInfoVersion,
             NodeMap allNodes,
             ConfigMap allConfigs,
             ChainTableMap allChainTables,
             ChainMap allChains,
             TargetMap allTargets,
             UniversalTagsMap allUniversalTags);

  bool bootstrapping(const MgmtdConfig &config) const;
};

}  // namespace hf3fs::mgmtd

#pragma once

#include <regex>

#include "common/monitor/Reporter.h"
#include "common/utils/ConfigBase.h"

namespace hf3fs::monitor {

class LogReporter : public Reporter {
 public:
  struct Config : ConfigBase<Config> {
    CONFIG_ITEM(ignore, std::string{});
  };

  LogReporter(const Config &config)
      : config_(config) {}

  Result<Void> init() final;

  Result<Void> commit(const std::vector<Sample> &samples) final;

 private:
  const Config &config_;
  std::optional<std::regex> ignore_;
};

}  // namespace hf3fs::monitor

#pragma once

#include "core/utils/ServiceOperation.h"
#include "fbs/mgmtd/Rpc.h"
#include "mgmtd/service/MgmtdState.h"

namespace hf3fs::mgmtd {
struct EnableNodeOperation : core::ServiceOperationWithMetric<"MgmtdService", "EnableNode", "op"> {
  EnableNodeReq req;

  explicit EnableNodeOperation(EnableNodeReq r)
      : req(std::move(r)) {}

  String toStringImpl() const final { return fmt::format("EnableNode {}", req.nodeId); }

  CoTryTask<EnableNodeRsp> handle(MgmtdState &state);
};

struct DisableNodeOperation : core::ServiceOperationWithMetric<"MgmtdService", "DisableNode", "op"> {
  DisableNodeReq req;

  explicit DisableNodeOperation(DisableNodeReq &&r)
      : req(std::move(r)) {}

  String toStringImpl() const final { return fmt::format("disableNode {}", req.nodeId); }

  CoTryTask<DisableNodeRsp> handle(MgmtdState &state);
};

}  // namespace hf3fs::mgmtd

#include "common/utils/Result.h"

#ifdef DEFINE_FBS_SERVICE
#undef DEFINE_FBS_SERVICE
#endif

#define DEFINE_FBS_SERVICE(name, fbsns)         \
  class I##name##ServiceStub {                  \
   public:                                      \
    using InterfaceType = I##name##ServiceStub; \
                                                \
    virtual ~I##name##ServiceStub() = default;

#ifdef FINISH_FBS_SERVICE
#undef FINISH_FBS_SERVICE
#endif

#define FINISH_FBS_SERVICE(name, fbsns) }

#ifdef DEFINE_FBS_SERVICE_METHOD
#undef DEFINE_FBS_SERVICE_METHOD
#endif

#define DEFINE_FBS_SERVICE_METHOD(svc, name, reqtype, rsptype, flatns) \
  virtual hf3fs::Result<flatns::rsptype> name(const flatns::reqtype &req) = 0

#ifdef DEFINE_FBS_SERVICE_METHOD_VOID
#undef DEFINE_FBS_SERVICE_METHOD_VOID
#endif

#define DEFINE_FBS_SERVICE_METHOD_VOID(svc, name, reqtype, rsptype, flatns) \
  virtual hf3fs::Result<Void> name(const flatns::reqtype &req) = 0

#ifdef DEFINE_FBS_SERVICE_METHOD_RETURNS
#undef DEFINE_FBS_SERVICE_METHOD_RETURNS
#endif

#define DEFINE_FBS_SERVICE_METHOD_RETURNS(svc, name, reqtype, rsptype, flatns, returns, rtype) \
  virtual hf3fs::Result<rtype> name(const flatns::reqtype &req) = 0

#pragma once

#include "IMgmtdServiceStub.h"

namespace hf3fs::mgmtd {
template <typename Ctx>
class MgmtdServiceStub : public IMgmtdServiceStub {
 public:
  using ContextType = Ctx;
  explicit MgmtdServiceStub(ContextType ctx)
      : ctx_(std::move(ctx)) {}

#define DEFINE_SERDE_SERVICE_METHOD_FULL(svc, name, Name, id, reqtype, rsptype) \
  CoTryTask<rsptype> name(const reqtype &req) override;

#include "fbs/mgmtd/MgmtdServiceDef.h"

 private:
  ContextType ctx_;
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "UserCache.h"
#include "UserStore.h"
#include "common/kv/IKVEngine.h"
#include "common/kv/TransactionRetry.h"

namespace hf3fs::core {

class UserStoreEx {
 public:
  UserStoreEx(kv::IKVEngine &kvEngine, const kv::TransactionRetry &transactionRetry, const UserCache::Config &cacheCfg);

  CoTryTask<void> authenticate(flat::UserInfo &userInfo);
  CoTryTask<flat::UserAttr> getUser(std::string_view token);
  CoTryTask<flat::UserAttr> getUser(flat::Uid uid);

  UserCache &cache() { return userCache_; }

 private:
  kv::IKVEngine &kvEngine_;
  const kv::TransactionRetry &transactionRetry_;
  UserStore userStore_;
  UserCache userCache_;
};
}  // namespace hf3fs::core

#pragma once

#include "core/utils/ServiceOperation.h"
#include "fbs/mgmtd/Rpc.h"
#include "mgmtd/service/MgmtdState.h"

namespace hf3fs::mgmtd {
struct ExtendClientSessionOperation : core::ServiceOperationWithMetric<"MgmtdService", "ExtendClientSession", "op"> {
  ExtendClientSessionReq req;

  explicit ExtendClientSessionOperation(ExtendClientSessionReq r)
      : req(std::move(r)) {}

  String toStringImpl() const final {
    return fmt::format("ExtendClientSession {}@{}", req.clientId, req.data.universalId);
  }

  CoTryTask<ExtendClientSessionRsp> handle(MgmtdState &state);
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "core/utils/ServiceOperation.h"
#include "fbs/mgmtd/Rpc.h"
#include "mgmtd/service/MgmtdState.h"

namespace hf3fs::mgmtd {
struct SetPreferredTargetOrderOperation
    : core::ServiceOperationWithMetric<"MgmtdService", "SetPreferredTargetOrder", "op"> {
  SetPreferredTargetOrderReq req;

  explicit SetPreferredTargetOrderOperation(SetPreferredTargetOrderReq r)
      : req(std::move(r)) {}

  String toStringImpl() const final { return fmt::format("SetPreferredTargetOrder {}", req.chainId); }

  CoTryTask<SetPreferredTargetOrderRsp> handle(MgmtdState &state);
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "core/utils/ServiceOperation.h"
#include "fbs/mgmtd/Rpc.h"
#include "mgmtd/service/MgmtdState.h"

namespace hf3fs::mgmtd {
struct RegisterNodeOperation : core::ServiceOperationWithMetric<"MgmtdService", "RegisterNode", "op"> {
  RegisterNodeReq req;

  explicit RegisterNodeOperation(RegisterNodeReq r)
      : req(std::move(r)) {}

  String toStringImpl() const final {
    return fmt::format("RegisterNode {} as {}", req.nodeId, magic_enum::enum_name(req.type));
  }

  CoTryTask<RegisterNodeRsp> handle(MgmtdState &state);
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "fbs/mgmtd/LocalTargetInfo.h"

namespace hf3fs::mgmtd {
struct LocalTargetInfoWithNodeId {
  flat::TargetId targetId{0};
  flat::NodeId nodeId{0};
  flat::LocalTargetState localState{flat::LocalTargetState::INVALID};

  LocalTargetInfoWithNodeId() = default;
  LocalTargetInfoWithNodeId(flat::NodeId nodeId, flat::LocalTargetInfo info)
      : targetId(info.targetId),
        nodeId(nodeId),
        localState(info.localState) {}
  LocalTargetInfoWithNodeId(flat::TargetId i, flat::NodeId ni, flat::LocalTargetState state)
      : targetId(i),
        nodeId(ni),
        localState(state) {}
};
}  // namespace hf3fs::mgmtd

#include "common/utils/Coroutine.h"

#ifdef DEFINE_FBS_SERVICE
#undef DEFINE_FBS_SERVICE
#endif

#define DEFINE_FBS_SERVICE(name, fbsns)         \
  class I##name##ServiceStub {                  \
   public:                                      \
    using InterfaceType = I##name##ServiceStub; \
                                                \
    virtual ~I##name##ServiceStub() = default;

#ifdef FINISH_FBS_SERVICE
#undef FINISH_FBS_SERVICE
#endif

#define FINISH_FBS_SERVICE(name, fbsns) }

#ifdef DEFINE_FBS_SERVICE_METHOD
#undef DEFINE_FBS_SERVICE_METHOD
#endif

#define DEFINE_FBS_SERVICE_METHOD(svc, name, reqtype, rsptype, flatns) \
  virtual CoTryTask<flatns::rsptype> name(const flatns::reqtype &req) = 0

#ifdef DEFINE_FBS_SERVICE_METHOD_VOID
#undef DEFINE_FBS_SERVICE_METHOD_VOID
#endif

#define DEFINE_FBS_SERVICE_METHOD_VOID(svc, name, reqtype, rsptype, flatns) \
  DEFINE_FBS_SERVICE_METHOD(svc, name, reqtype, rsptype, flatns)

#ifdef DEFINE_FBS_SERVICE_METHOD_RETURNS
#undef DEFINE_FBS_SERVICE_METHOD_RETURNS
#endif

#define DEFINE_FBS_SERVICE_METHOD_RETURNS(svc, name, reqtype, rsptype, flatns, returns, rtype) \
  DEFINE_FBS_SERVICE_METHOD(svc, name, reqtype, rsptype, flatns)

#pragma once

#include <memory>

#include "common/utils/Address.h"

namespace hf3fs::stubs {
template <typename IStub>
class IStubFactory {
 public:
  using Stub = IStub;

  virtual ~IStubFactory() = default;
  virtual std::unique_ptr<IStub> create(net::Address addr) = 0;
};
}  // namespace hf3fs::stubs

#pragma once

#include "MgmtdState.h"
#include "common/net/PeerInfo.h"
#include "fbs/mgmtd/Rpc.h"
#include "mgmtd/background/MgmtdBackgroundRunner.h"

namespace hf3fs::mgmtd {
namespace testing {
class MgmtdTestHelper;
}

class MgmtdOperator : public folly::NonCopyableNonMovable {
 public:
  MgmtdOperator(std::shared_ptr<core::ServerEnv> env, const MgmtdConfig &config);

  ~MgmtdOperator();

  // start/stop background tasks
  void start();
  CoTask<void> stop();

#define DEFINE_SERDE_SERVICE_METHOD_FULL(Service, method, Method, Id, Req, Rsp) \
  CoTryTask<Rsp> method(Req req, const net::PeerInfo &peer);

#include "fbs/mgmtd/MgmtdServiceDef.h"

 private:
  MgmtdState state_;
  MgmtdBackgroundRunner backgroundRunner_;

  friend class testing::MgmtdTestHelper;
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "core/utils/ServiceOperation.h"
#include "fbs/mgmtd/Rpc.h"
#include "mgmtd/service/MgmtdState.h"

namespace hf3fs::mgmtd {
struct HeartbeatOperation : core::ServiceOperationWithMetric<"MgmtdService", "Heartbeat", "op"> {
  HeartbeatReq req;

  explicit HeartbeatOperation(HeartbeatReq r)
      : req(std::move(r)) {}

  String toStringImpl() const final {
    return fmt::format("Heartbeat from {}@{} type {}",
                       req.info.app.nodeId,
                       req.info.app.hostname,
                       magic_enum::enum_name(req.info.type()));
  }

  CoTryTask<HeartbeatRsp> handle(MgmtdState &state);
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "common/serde/CallContext.h"
#include "fbs/mgmtd/MgmtdServiceBase.h"

namespace hf3fs::mgmtd {
class MgmtdOperator;
class MgmtdService : public serde::ServiceWrapper<MgmtdService, MgmtdServiceBase> {
 public:
  MgmtdService(MgmtdOperator &opr)
      : operator_(opr) {}

#define DEFINE_SERDE_SERVICE_METHOD_FULL(svc, name, Name, id, reqtype, rsptype) \
  CoTryTask<rsptype> name(serde::CallContext &ctx, const reqtype &req);

#include "fbs/mgmtd/MgmtdServiceDef.h"

 private:
  MgmtdOperator &operator_;
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "core/utils/ServiceOperation.h"
#include "fbs/mgmtd/Rpc.h"
#include "mgmtd/service/MgmtdState.h"

namespace hf3fs::mgmtd {
struct RotateAsPreferredOrderOperation
    : core::ServiceOperationWithMetric<"MgmtdService", "RotateAsPreferredOrder", "op"> {
  RotateAsPreferredOrderReq req;

  explicit RotateAsPreferredOrderOperation(RotateAsPreferredOrderReq r)
      : req(std::move(r)) {}

  String toStringImpl() const final { return fmt::format("RotateAsPreferredOrder {}", req.chainId); }

  CoTryTask<RotateAsPreferredOrderRsp> handle(MgmtdState &state);
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "core/utils/ServiceOperation.h"
#include "fbs/mgmtd/Rpc.h"
#include "mgmtd/service/MgmtdState.h"

namespace hf3fs::mgmtd {
struct GetConfigOperation : core::ServiceOperationWithMetric<"MgmtdService", "GetConfig", "op"> {
  GetConfigReq req;

  explicit GetConfigOperation(GetConfigReq r)
      : req(std::move(r)) {}

  String toStringImpl() const final {
    return fmt::format("GetConfig {}@{}", magic_enum::enum_name(req.nodeType), req.configVersion);
  }

  CoTryTask<GetConfigRsp> handle(MgmtdState &state);
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "common/kv/TransactionRetry.h"
#include "common/utils/ConfigBase.h"
#include "common/utils/Duration.h"
#include "core/user/UserCache.h"

namespace hf3fs::mgmtd {
struct MgmtdConfig : ConfigBase<MgmtdConfig> {
  CONFIG_HOT_UPDATED_ITEM(lease_length, 60_s);
  CONFIG_HOT_UPDATED_ITEM(extend_lease_interval, 10_s);
  CONFIG_HOT_UPDATED_ITEM(suspicious_lease_interval, 20_s);
  CONFIG_HOT_UPDATED_ITEM(heartbeat_timestamp_valid_window, 30_s);
  CONFIG_HOT_UPDATED_ITEM(allow_heartbeat_from_unregistered, false);
  CONFIG_HOT_UPDATED_ITEM(check_status_interval, 10_s);
  CONFIG_HOT_UPDATED_ITEM(heartbeat_fail_interval, 60_s);
  CONFIG_HOT_UPDATED_ITEM(new_chain_bootstrap_interval, 2_min);
  CONFIG_HOT_UPDATED_ITEM(send_heartbeat, true);
  CONFIG_HOT_UPDATED_ITEM(send_heartbeat_interval, 10_s);
  CONFIG_HOT_UPDATED_ITEM(client_session_timeout, 20_min);
  CONFIG_HOT_UPDATED_ITEM(bootstrapping_length, 2_min);
  CONFIG_HOT_UPDATED_ITEM(update_chains_interval, 1_s);
  CONFIG_HOT_UPDATED_ITEM(validate_lease_on_write, true);
  CONFIG_HOT_UPDATED_ITEM(bump_routing_info_version_interval, 5_s);
  // deprecated
  CONFIG_HOT_UPDATED_ITEM(heartbeat_ignore_unknown_targets, false);
  CONFIG_HOT_UPDATED_ITEM(heartbeat_ignore_stale_targets, true);
  CONFIG_HOT_UPDATED_ITEM(retry_times_on_txn_errors, -1);  // -1 means infitely retry, 0 means no retry
  CONFIG_HOT_UPDATED_ITEM(update_metrics_interval, 1_s);
  CONFIG_HOT_UPDATED_ITEM(target_info_persist_interval, 1_s);
  CONFIG_HOT_UPDATED_ITEM(target_info_persist_batch, 1000);
  CONFIG_HOT_UPDATED_ITEM(target_info_load_interval, 1_s);
  CONFIG_HOT_UPDATED_ITEM(try_adjust_target_order_as_preferred, false);
  CONFIG_HOT_UPDATED_ITEM(extend_lease_check_release_version, true);
  CONFIG_HOT_UPDATED_ITEM(authenticate, false);
  CONFIG_OBJ(retry_transaction, kv::TransactionRetry);
  CONFIG_OBJ(user_cache, core::UserCache::Config);
  CONFIG_HOT_UPDATED_ITEM(enable_routinginfo_cache, true);
  CONFIG_HOT_UPDATED_ITEM(only_accept_client_uuid, false);
};
}  // namespace hf3fs::mgmtd

#pragma once

#include <vector>

#include "common/monitor/Sample.h"
#include "common/utils/Result.h"

namespace hf3fs::monitor {

class Reporter {
 public:
  virtual ~Reporter() = default;
  virtual Result<Void> init() = 0;
  virtual Result<Void> commit(const std::vector<Sample> &samples) = 0;
};

}  // namespace hf3fs::monitor

#pragma once

#include <condition_variable>
#include <folly/MPMCQueue.h>
#include <mutex>
#include <thread>

#include "common/monitor/Monitor.h"
#include "monitor_collector/service/MonitorCollectorService.h"

namespace hf3fs::monitor {

class MonitorCollectorOperator {
 public:
  MonitorCollectorOperator(const MonitorCollectorService::Config &cfg);
  ~MonitorCollectorOperator();

  CoTryTask<void> write(std::vector<Sample> &&samples);

 private:
  void connThreadFunc(std::stop_token stoken);
  void monitorThreadFunc(std::stop_token stoken);

  const MonitorCollectorService::Config &cfg_;
  std::vector<std::jthread> threads_;
  std::mutex m_;
  std::condition_variable_any cv_;
  folly::MPMCQueue<std::vector<Sample>> sampleQueue_;
};

}  // namespace hf3fs::monitor

#pragma once

#include "MgmtdTypes.h"
#include "common/serde/SerdeHelper.h"
#include "common/utils/UtcTimeSerde.h"

namespace hf3fs::flat {
struct NodeStatusChange : public serde::SerdeHelper<NodeStatusChange> {
  SERDE_STRUCT_FIELD(status, NodeStatus(NodeStatus::MIN));
  SERDE_STRUCT_FIELD(time, UtcTime{});
};
}  // namespace hf3fs::flat

#pragma once

#include "common/app/AppInfo.h"

namespace hf3fs {
Result<String> renderConfig(const String &configTemplate,
                            const flat::AppInfo *app,
                            const std::map<String, String> *envs = nullptr);
Result<flat::ReleaseVersion> parseReleaseVersion(const String &str, const flat::ReleaseVersion &actual);
}  // namespace hf3fs

#pragma once

#include "core/utils/ServiceOperation.h"
#include "fbs/mgmtd/Rpc.h"
#include "mgmtd/service/MgmtdState.h"

namespace hf3fs::mgmtd {

struct GetPrimaryMgmtdOperation : core::ServiceOperationWithMetric<"MgmtdService", "GetPrimaryMgmtd", "op"> {
  GetPrimaryMgmtdReq req;

  explicit GetPrimaryMgmtdOperation(GetPrimaryMgmtdReq r)
      : req(std::move(r)) {}

  String toStringImpl() const final { return "GetPrimaryMgmtd"; }

  CoTryTask<GetPrimaryMgmtdRsp> handle(MgmtdState &state) const;
};
}  // namespace hf3fs::mgmtd

#pragma once

#include <memory>

#include "common/kv/ITransaction.h"

namespace hf3fs::kv {
class IKVEngine {
 public:
  virtual ~IKVEngine() = default;

  virtual std::unique_ptr<IReadOnlyTransaction> createReadonlyTransaction() = 0;
  virtual std::unique_ptr<IReadWriteTransaction> createReadWriteTransaction() = 0;
};
}  // namespace hf3fs::kv

#pragma once

#include "client/mgmtd/MgmtdClient.h"
#include "common/app/NodeId.h"
#include "common/net/Client.h"
#include "common/utils/ConfigBase.h"

namespace hf3fs::core {
struct ServerLauncherConfig : public ConfigBase<ServerLauncherConfig> {
  CONFIG_ITEM(cluster_id, "");
  CONFIG_OBJ(ib_devices, net::IBDevice::Config);
  CONFIG_OBJ(client, net::Client::Config);
  CONFIG_OBJ(mgmtd_client, client::MgmtdClient::Config);
  CONFIG_ITEM(allow_dev_version, true);

 public:
  using Base = ConfigBase<ServerLauncherConfig>;
  using Base::init;

  void init(const String &filePath, bool dump, const std::vector<config::KeyValue> &updates);
};
}  // namespace hf3fs::core

#pragma once

#include "fbs/core/service/Rpc.h"

namespace hf3fs::core {

class ICoreServiceStub {
 public:
  using InterfaceType = ICoreServiceStub;

  virtual ~ICoreServiceStub() = default;

#define DEFINE_SERDE_SERVICE_METHOD_FULL(svc, name, Name, id, reqtype, rsptype) \
  virtual CoTryTask<rsptype> name(const reqtype &req) = 0;
#include "fbs/core/service/CoreServiceDef.h"
};
}  // namespace hf3fs::core

#pragma once

#include "FDBConfig.h"

namespace hf3fs::kv {
struct HybridKvEngineConfig : public ConfigBase<HybridKvEngineConfig> {
  bool operator==(const HybridKvEngineConfig &other) const { return static_cast<const ConfigBase &>(*this) == other; }

  CONFIG_ITEM(use_memkv, false);
  CONFIG_OBJ(fdb, kv::fdb::FDBConfig);
};
}  // namespace hf3fs::kv

#pragma once

#include <folly/logging/xlog.h>

namespace hf3fs::logging {

// set logging level to ERR if the specified condition is evaluated to be true
#define ERRLOGF_IF(level, cond, fmt, ...)                     \
  do {                                                        \
    XLOGF_IF(ERR, cond, fmt __VA_OPT__(, ) __VA_ARGS__);      \
    XLOGF_IF(level, !(cond), fmt __VA_OPT__(, ) __VA_ARGS__); \
  } while (false)

// set logging level to WARN if the specified condition is evaluated to be true
#define WARNLOGF_IF(level, cond, fmt, ...)                    \
  do {                                                        \
    XLOGF_IF(WARN, cond, fmt __VA_OPT__(, ) __VA_ARGS__);     \
    XLOGF_IF(level, !(cond), fmt __VA_OPT__(, ) __VA_ARGS__); \
  } while (false)

}  // namespace hf3fs::logging

#pragma once

#include "fbs/mgmtd/Rpc.h"

namespace hf3fs::mgmtd {

class IMgmtdServiceStub {
 public:
  using InterfaceType = IMgmtdServiceStub;

  virtual ~IMgmtdServiceStub() = default;

#define DEFINE_SERDE_SERVICE_METHOD_FULL(svc, name, Name, id, reqtype, rsptype) \
  virtual CoTryTask<rsptype> name(const reqtype &req) = 0;
#include "fbs/mgmtd/MgmtdServiceDef.h"
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "common/net/ib/IBDevice.h"
#include "fdb/HybridKvEngineConfig.h"

namespace hf3fs::mgmtd {
struct MgmtdLauncherConfig : public ConfigBase<MgmtdLauncherConfig> {
  CONFIG_ITEM(cluster_id, "");
  CONFIG_ITEM(use_memkv, false);        // deprecated
  CONFIG_OBJ(fdb, kv::fdb::FDBConfig);  // deprecated
  CONFIG_OBJ(ib_devices, net::IBDevice::Config);
  CONFIG_OBJ(kv_engine, kv::HybridKvEngineConfig);
  CONFIG_ITEM(allow_dev_version, true);

 public:
  using ConfigBase<MgmtdLauncherConfig>::init;
  void init(const String &filePath, bool dump, const std::vector<config::KeyValue> &updates);
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "client/mgmtd/MgmtdClientForClient.h"
#include "common/app/NodeId.h"
#include "common/net/Client.h"
#include "common/utils/ConfigBase.h"

namespace hf3fs::fuse {
struct FuseLauncherConfig : public ConfigBase<FuseLauncherConfig> {
  CONFIG_ITEM(cluster_id, "");
  CONFIG_OBJ(ib_devices, net::IBDevice::Config);
  CONFIG_OBJ(client, net::Client::Config);
  CONFIG_OBJ(mgmtd_client, client::MgmtdClientForClient::Config);
  CONFIG_ITEM(mountpoint, "");
  CONFIG_ITEM(allow_other, true);
  CONFIG_ITEM(token_file, "");

 public:
  using Base = ConfigBase<FuseLauncherConfig>;
  using Base::init;

  void init(const String &filePath, bool dump, const std::vector<config::KeyValue> &updates);
};
}  // namespace hf3fs::fuse

#pragma once

#include "core/utils/ServiceOperation.h"
#include "fbs/mgmtd/Rpc.h"
#include "mgmtd/service/MgmtdState.h"

namespace hf3fs::mgmtd {
struct UpdateChainOperation : core::ServiceOperationWithMetric<"MgmtdService", "UpdateChain", "op"> {
  UpdateChainReq req;

  explicit UpdateChainOperation(UpdateChainReq r)
      : req(std::move(r)) {}

  String toStringImpl() const final { return fmt::format("UpdateChain {}", req.chainId); }

  CoTryTask<UpdateChainRsp> handle(MgmtdState &state);
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "common/utils/Status.h"

#define ENSURE_USAGE(condition, ...)                                                                       \
  do {                                                                                \
    if (!UNLIKELY(static_cast<bool>(condition)))                                                           \
      throw hf3fs::StatusException(hf3fs::Status(hf3fs::CliCode::kWrongUsage __VA_OPT__(, ) __VA_ARGS__)); \
  } while (false)

#pragma once

#include "common/monitor/Sample.h"
#include "common/serde/Serde.h"
#include "common/serde/Service.h"

namespace hf3fs::monitor {

struct MonitorCollectorRsp {
  SERDE_STRUCT_FIELD(dummy, Void{});
};

SERDE_SERVICE(MonitorCollector, 194) { SERDE_SERVICE_METHOD(write, 1, std::vector<Sample>, MonitorCollectorRsp); };

}  // namespace hf3fs::monitor

#pragma once

#include "ApplicationBase.h"

namespace hf3fs::app_detail {
Result<Void> parseFlags(int *argc,
                        char ***argv,
                        ApplicationBase::ConfigFlags &appConfigFlags,
                        ApplicationBase::ConfigFlags &launcherConfigFlags,
                        ApplicationBase::ConfigFlags &configFlags);

void initLogging(const logging::LogConfig &cfg, const String &serverName);
void initCommonComponents(const ApplicationBase::Config &cfg, const String &serverName, flat::NodeId nodeId);
void persistConfig(const config::IConfig &cfg);

std::unique_ptr<ConfigCallbackGuard> makeLogConfigUpdateCallback(logging::LogConfig &cfg, String serverName);

std::unique_ptr<ConfigCallbackGuard> makeMemConfigUpdateCallback(memory::MemoryAllocatorConfig &cfg, String hostname);

void loadAppInfo(std::function<Result<flat::AppInfo>()> launcher, flat::AppInfo &baseInfo);

void initConfig(config::IConfig &cfg,
                const std::vector<config::KeyValue> &updates,
                const flat::AppInfo &appInfo,
                std::function<Result<std::pair<String, String>>()> launcher);

void initConfigFromFile(config::IConfig &cfg,
                        const String &filePath,
                        bool dump,
                        const std::vector<config::KeyValue> &updates);

}  // namespace hf3fs::app_detail

#pragma once

#include "core/utils/ServiceOperation.h"
#include "fbs/mgmtd/Rpc.h"
#include "mgmtd/service/MgmtdState.h"

namespace hf3fs::mgmtd {
struct SetConfigOperation : core::ServiceOperationWithMetric<"MgmtdService", "SetConfig", "op"> {
  SetConfigReq req;

  explicit SetConfigOperation(SetConfigReq r)
      : req(std::move(r)) {}

  String toStringImpl() const final { return fmt::format("SetConfig {}", magic_enum::enum_name(req.nodeType)); }

  CoTryTask<SetConfigRsp> handle(MgmtdState &state);
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "core/utils/ServiceOperation.h"
#include "fbs/mgmtd/Rpc.h"
#include "mgmtd/service/MgmtdState.h"

namespace hf3fs::mgmtd {
struct GetClientSessionOperation : core::ServiceOperationWithMetric<"MgmtdService", "GetClientSession", "op"> {
  GetClientSessionReq req;

  explicit GetClientSessionOperation(GetClientSessionReq r)
      : req(std::move(r)) {}

  String toStringImpl() const final { return fmt::format("GetClientSession {}", req.clientId); }

  CoTryTask<GetClientSessionRsp> handle(MgmtdState &state);
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "core/utils/ServiceOperation.h"
#include "fbs/mgmtd/Rpc.h"
#include "mgmtd/service/MgmtdState.h"

namespace hf3fs::mgmtd {
struct SetNodeTagsOperation : core::ServiceOperationWithMetric<"MgmtdService", "SetNodeTags", "op"> {
  SetNodeTagsReq req;

  explicit SetNodeTagsOperation(SetNodeTagsReq r)
      : req(std::move(r)) {}

  String toStringImpl() const final { return fmt::format("SetNodeTags {}", req.nodeId); }

  CoTryTask<SetNodeTagsRsp> handle(MgmtdState &state);
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "core/utils/ServiceOperation.h"
#include "fbs/mgmtd/Rpc.h"
#include "mgmtd/service/MgmtdState.h"

namespace hf3fs::mgmtd {
struct SetChainTableOperation : core::ServiceOperationWithMetric<"MgmtdService", "SetChainTable", "op"> {
  SetChainTableReq req;

  explicit SetChainTableOperation(SetChainTableReq r)
      : req(std::move(r)) {}

  String toStringImpl() const final { return fmt::format("SetChainTable {}", req.chainTableId); }

  CoTryTask<SetChainTableRsp> handle(MgmtdState &state);
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "core/utils/ServiceOperation.h"
#include "fbs/mgmtd/Rpc.h"
#include "mgmtd/service/MgmtdState.h"

namespace hf3fs::mgmtd {
struct GetUniversalTagsOperation : core::ServiceOperationWithMetric<"MgmtdService", "GetUniversalTags", "op"> {
  GetUniversalTagsReq req;

  explicit GetUniversalTagsOperation(GetUniversalTagsReq r)
      : req(std::move(r)) {}

  String toStringImpl() const final { return fmt::format("GetUniversalTags {}", req.universalId); }

  CoTryTask<GetUniversalTagsRsp> handle(MgmtdState &state);
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "core/utils/ServiceOperation.h"
#include "fbs/mgmtd/Rpc.h"
#include "mgmtd/service/MgmtdState.h"

namespace hf3fs::mgmtd {
struct SetUniversalTagsOperation : core::ServiceOperationWithMetric<"MgmtdService", "SetUniversalTags", "op"> {
  SetUniversalTagsReq req;

  explicit SetUniversalTagsOperation(SetUniversalTagsReq r)
      : req(std::move(r)) {}

  String toStringImpl() const final { return fmt::format("SetUniversalTags {}", req.universalId); }

  CoTryTask<SetUniversalTagsRsp> handle(MgmtdState &state);
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "ChainTargetInfo.h"

namespace hf3fs::flat {
struct ChainInfo : public serde::SerdeHelper<ChainInfo> {
  SERDE_STRUCT_FIELD(chainId, ChainId(0));
  SERDE_STRUCT_FIELD(chainVersion, ChainVersion(0));
  SERDE_STRUCT_FIELD(targets, std::vector<ChainTargetInfo>{});
  SERDE_STRUCT_FIELD(preferredTargetOrder, std::vector<TargetId>{});

  bool operator==(const ChainInfo &other) const;
};
}  // namespace hf3fs::flat

#pragma once

#include "core/utils/ServiceOperation.h"
#include "fbs/mgmtd/Rpc.h"
#include "mgmtd/service/MgmtdState.h"

namespace hf3fs::mgmtd {

struct GetRoutingInfoOperation : core::ServiceOperationWithMetric<"MgmtdService", "GetRoutingInfo", "op"> {
  GetRoutingInfoReq req;

  explicit GetRoutingInfoOperation(GetRoutingInfoReq r)
      : req(std::move(r)) {}

  String toStringImpl() const final { return fmt::format("GetRoutingInfo for {}", req.routingInfoVersion); }

  CoTryTask<GetRoutingInfoRsp> handle(MgmtdState &state);
};

}  // namespace hf3fs::mgmtd

#pragma once

#include "core/utils/ServiceOperation.h"
#include "fbs/mgmtd/Rpc.h"
#include "mgmtd/service/MgmtdState.h"

namespace hf3fs::mgmtd {
struct UnregisterNodeOperation : core::ServiceOperationWithMetric<"MgmtdService", "UnregisterNode", "op"> {
  UnregisterNodeReq req;

  explicit UnregisterNodeOperation(UnregisterNodeReq r)
      : req(std::move(r)) {}

  String toStringImpl() const final { return fmt::format("UnregisterNode {}", req.nodeId); }

  CoTryTask<UnregisterNodeRsp> handle(MgmtdState &state);
};

}  // namespace hf3fs::mgmtd

#pragma once

#include "core/utils/ServiceOperation.h"
#include "fbs/mgmtd/Rpc.h"
#include "mgmtd/service/MgmtdState.h"

namespace hf3fs::mgmtd {
struct SetChainsOperation : core::ServiceOperationWithMetric<"MgmtdService", "SetChains", "op"> {
  SetChainsReq req;

  explicit SetChainsOperation(SetChainsReq r)
      : req(std::move(r)) {}

  String toStringImpl() const final { return "SetChains"; }

  CoTryTask<SetChainsRsp> handle(MgmtdState &state);
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "core/utils/ServiceOperation.h"
#include "fbs/mgmtd/Rpc.h"
#include "mgmtd/service/MgmtdState.h"

namespace hf3fs::mgmtd {
struct GetConfigVersionsOperation : core::ServiceOperationWithMetric<"MgmtdService", "GetConfigVersions", "op"> {
  GetConfigVersionsReq req;

  explicit GetConfigVersionsOperation(GetConfigVersionsReq r)
      : req(std::move(r)) {}

  String toStringImpl() const final { return "GetConfigVersions"; }

  CoTryTask<GetConfigVersionsRsp> handle(MgmtdState &state);
};
}  // namespace hf3fs::mgmtd

#include "common/app/OnePhaseApplication.h"
#include "memory/common/OverrideCppNewDelete.h"
#include "migration/service/Server.h"

int main(int argc, char *argv[]) {
  return hf3fs::OnePhaseApplication<hf3fs::migration::server::MigrationServer>::instance().run(argc, argv);
}

#pragma once

#include "core/utils/ServiceOperation.h"
#include "fbs/mgmtd/Rpc.h"
#include "mgmtd/service/MgmtdState.h"

namespace hf3fs::mgmtd {
struct RotateLastSrvOperation : core::ServiceOperationWithMetric<"MgmtdService", "RotateLastSrv", "op"> {
  RotateLastSrvReq req;

  explicit RotateLastSrvOperation(RotateLastSrvReq r)
      : req(std::move(r)) {}

  String toStringImpl() const final { return fmt::format("RotateLastSrv {}", req.chainId); }

  CoTryTask<RotateLastSrvRsp> handle(MgmtdState &state);
};
}  // namespace hf3fs::mgmtd

#ifndef DEFINE_PREFIX
#define DEFINE_PREFIX(...)
#endif

// clang-format off
DEFINE_PREFIX(Inode,            "INOD")
DEFINE_PREFIX(Dentry,           "DENT")
DEFINE_PREFIX(MetaDistributor,  "META")
DEFINE_PREFIX(User,             "USER")
DEFINE_PREFIX(NodeTable,        "NODE")
// Single is used for some single keys (not tables)
DEFINE_PREFIX(Single,           "SING")
DEFINE_PREFIX(ChainTable,       "CHIT")
DEFINE_PREFIX(ChainInfo,        "CHIF")
DEFINE_PREFIX(InodeSession,     "INOS")
DEFINE_PREFIX(ClientSession,    "CLIS") // deperated
DEFINE_PREFIX(MetaIdempotent,   "IDEM")
DEFINE_PREFIX(UniversalTags,    "UTGS")
DEFINE_PREFIX(Config,           "CONF")
DEFINE_PREFIX(TargetInfo,       "TGIF")
DEFINE_PREFIX(KvTable,          "KVTB")
DEFINE_PREFIX(KvNamespace,      "KVNS")
DEFINE_PREFIX(KvWorkerGroup,    "KVWG")
// clang-format on

#undef DEFINE_PREFIX

#include "fbs/macros/SerdeDef.h"

DEFINE_SERDE_SERVICE_METHOD(Mgmtd, getPrimaryMgmtd, GetPrimaryMgmtd, 1)
// 2 is deprecated
DEFINE_SERDE_SERVICE_METHOD(Mgmtd, heartbeat, Heartbeat, 3)
DEFINE_SERDE_SERVICE_METHOD(Mgmtd, registerNode, RegisterNode, 4)
DEFINE_SERDE_SERVICE_METHOD(Mgmtd, getRoutingInfo, GetRoutingInfo, 5)
DEFINE_SERDE_SERVICE_METHOD(Mgmtd, setConfig, SetConfig, 6)
DEFINE_SERDE_SERVICE_METHOD(Mgmtd, getConfig, GetConfig, 7)
DEFINE_SERDE_SERVICE_METHOD(Mgmtd, setChainTable, SetChainTable, 8)
DEFINE_SERDE_SERVICE_METHOD(Mgmtd, enableNode, EnableNode, 9)
DEFINE_SERDE_SERVICE_METHOD(Mgmtd, disableNode, DisableNode, 10)
DEFINE_SERDE_SERVICE_METHOD(Mgmtd, extendClientSession, ExtendClientSession, 11)
DEFINE_SERDE_SERVICE_METHOD(Mgmtd, listClientSessions, ListClientSessions, 12)
DEFINE_SERDE_SERVICE_METHOD(Mgmtd, setNodeTags, SetNodeTags, 13)
DEFINE_SERDE_SERVICE_METHOD(Mgmtd, unregisterNode, UnregisterNode, 14)
DEFINE_SERDE_SERVICE_METHOD(Mgmtd, setChains, SetChains, 15)
DEFINE_SERDE_SERVICE_METHOD(Mgmtd, setUniversalTags, SetUniversalTags, 16)
DEFINE_SERDE_SERVICE_METHOD(Mgmtd, getUniversalTags, GetUniversalTags, 17)
DEFINE_SERDE_SERVICE_METHOD(Mgmtd, getConfigVersions, GetConfigVersions, 18)
DEFINE_SERDE_SERVICE_METHOD(Mgmtd, getClientSession, GetClientSession, 19)
DEFINE_SERDE_SERVICE_METHOD(Mgmtd, rotateLastSrv, RotateLastSrv, 20)
DEFINE_SERDE_SERVICE_METHOD(Mgmtd, listOrphanTargets, ListOrphanTargets, 21)
DEFINE_SERDE_SERVICE_METHOD(Mgmtd, setPreferredTargetOrder, SetPreferredTargetOrder, 22)
DEFINE_SERDE_SERVICE_METHOD(Mgmtd, rotateAsPreferredOrder, RotateAsPreferredOrder, 23)
DEFINE_SERDE_SERVICE_METHOD(Mgmtd, updateChain, UpdateChain, 24)

#include "fbs/macros/Undef.h"

#include "common/app/OnePhaseApplication.h"
#include "memory/common/OverrideCppNewDelete.h"
#include "monitor_collector/service/MonitorCollectorServer.h"

int main(int argc, char *argv[]) {
  return hf3fs::OnePhaseApplication<hf3fs::monitor::MonitorCollectorServer>::instance().run(argc, argv);
}

#pragma once

#include "client/cli/common/IEnv.h"
#include "client/core/CoreClient.h"
#include "client/meta/MetaClient.h"
#include "client/mgmtd/IMgmtdClientForAdmin.h"
#include "client/storage/StorageClient.h"
#include "common/kv/IKVEngine.h"
#include "fbs/core/user/User.h"

namespace hf3fs::client::cli {
struct AdminEnv : IEnv {
  flat::UserInfo userInfo{flat::Uid{0}, flat::Gid{0}, ""};
  meta::InodeId currentDirId = meta::InodeId::root();
  String currentDir = "/";

  std::function<std::shared_ptr<net::Client>()> clientGetter;
  std::function<std::shared_ptr<meta::client::MetaClient>()> metaClientGetter;
  std::function<std::shared_ptr<kv::IKVEngine>()> kvEngineGetter;
  std::function<std::shared_ptr<IMgmtdClientForAdmin>()> mgmtdClientGetter;
  std::function<std::shared_ptr<IMgmtdClientForAdmin>()> unsafeMgmtdClientGetter;
  std::function<std::shared_ptr<storage::client::StorageClient>()> storageClientGetter;
  std::function<std::shared_ptr<CoreClient>()> coreClientGetter;
};
}  // namespace hf3fs::client::cli

#pragma once

#include <optional>
#include <string>
#include <string_view>
#include <vector>

#include "common/utils/Result.h"

namespace hf3fs::client::cli {
class ArgsParser {
 public:
  explicit ArgsParser(std::string_view s)
      : s_(s) {}

  Result<std::string> parseOne();
  Result<std::vector<std::string>> parseAll();

 private:
  std::string_view s_;
};
}  // namespace hf3fs::client::cli

#pragma once

#include "Rpc.h"
#include "common/serde/Service.h"

namespace hf3fs::core {
SERDE_SERVICE_2(CoreServiceBase, Core, 10001) {
#define DEFINE_SERDE_SERVICE_METHOD_FULL(svc, name, Name, id, reqtype, rsptype) \
  SERDE_SERVICE_METHOD(name, id, reqtype, rsptype);
#include "CoreServiceDef.h"
};
}  // namespace hf3fs::core

#pragma once

#include "common/serde/CallContext.h"
#include "fbs/core/service/CoreServiceBase.h"

namespace hf3fs::core {
class CoreService : public serde::ServiceWrapper<CoreService, CoreServiceBase> {
 public:
#define DEFINE_SERDE_SERVICE_METHOD_FULL(svc, name, Name, id, reqtype, rsptype) \
  CoTryTask<rsptype> name(serde::CallContext &ctx, const reqtype &req);

#include "fbs/core/service/CoreServiceDef.h"
};
}  // namespace hf3fs::core

#pragma once

#include <folly/logging/xlog.h>

#include "common/utils/NameWrapper.h"

namespace hf3fs {

template <NameWrapper Name>
struct ConstructLog {
  ConstructLog() { XLOGF(INFO, "Construct {}", std::string_view{Name}); }
  ~ConstructLog() { XLOGF(INFO, "Destruct {}", std::string_view{Name}); }
};

}  // namespace hf3fs

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs {
class BackgroundRunner;
namespace mgmtd {
class MgmtdHeartbeater;
class MgmtdLeaseExtender;
struct MgmtdState;
class MgmtdChainsUpdater;
class MgmtdClientSessionsChecker;
class MgmtdHeartbeatChecker;
class MgmtdNewBornChainsChecker;
class MgmtdRoutingInfoVersionUpdater;
class MgmtdMetricsUpdater;
class MgmtdTargetInfoPersister;
class MgmtdTargetInfoLoader;
namespace testing {
class MgmtdTestHelper;
}

class MgmtdBackgroundRunner {
 public:
  explicit MgmtdBackgroundRunner(MgmtdState &state);
  ~MgmtdBackgroundRunner();

  void start();
  CoTask<void> stop();

 private:
  MgmtdState &state_;
  std::unique_ptr<BackgroundRunner> backgroundRunner_;
  std::unique_ptr<MgmtdHeartbeater> heartbeater_;
  std::unique_ptr<MgmtdLeaseExtender> leaseExtender_;
  std::unique_ptr<MgmtdChainsUpdater> chainsUpdater_;
  std::unique_ptr<MgmtdClientSessionsChecker> clientSessionsChecker_;
  std::unique_ptr<MgmtdHeartbeatChecker> heartbeatChecker_;
  std::unique_ptr<MgmtdNewBornChainsChecker> newBornChainsChecker_;
  std::unique_ptr<MgmtdRoutingInfoVersionUpdater> routingInfoVersionUpdater_;
  std::unique_ptr<MgmtdMetricsUpdater> metricsUpdater_;
  std::unique_ptr<MgmtdTargetInfoPersister> targetInfoPersister_;
  std::unique_ptr<MgmtdTargetInfoLoader> targetInfoLoader_;

  friend class testing::MgmtdTestHelper;
};
}  // namespace mgmtd
}  // namespace hf3fs

#pragma once

namespace hf3fs::client::cli {
// hide implementations, user should dynamic cast IEnv to a known concrete Env class.
struct IEnv {
  virtual ~IEnv() = default;
};
}  // namespace hf3fs::client::cli

#pragma once

#include "ChainTargetSetting.h"

namespace hf3fs::flat {
struct ChainSetting : public serde::SerdeHelper<ChainSetting> {
  SERDE_STRUCT_FIELD(chainId, ChainId(0));
  SERDE_STRUCT_FIELD(targets, std::vector<ChainTargetSetting>{});
  SERDE_STRUCT_FIELD(setPreferredTargetOrder, false);
};
}  // namespace hf3fs::flat

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerListGcHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "Rpc.h"
#include "common/serde/Service.h"

namespace hf3fs::mgmtd {
SERDE_SERVICE_2(MgmtdServiceBase, Mgmtd, 217) {
#define DEFINE_SERDE_SERVICE_METHOD_FULL(svc, name, Name, id, reqtype, rsptype) \
  SERDE_SERVICE_METHOD_REFL(name, id, reqtype, rsptype);
#include "MgmtdServiceDef.h"
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "common/utils/Coroutine.h"
#include "common/utils/UtcTime.h"

namespace hf3fs::mgmtd {
struct MgmtdState;
class MgmtdChainsUpdater {
 public:
  explicit MgmtdChainsUpdater(MgmtdState &state);

  CoTask<void> update(SteadyTime &lastUpdateTs);

 private:
  bool lastAdjustTargetOrderFlag = false;
  MgmtdState &state_;
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerListHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerListNodesHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerListClientsHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerListTargetsHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerListChainsHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerListChainTablesHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#include "common/app/TwoPhaseApplication.h"
#include "memory/common/OverrideCppNewDelete.h"
#include "simple_example/service/Server.h"

int main(int argc, char *argv[]) {
  using namespace hf3fs;
  return TwoPhaseApplication<simple_example::server::SimpleExampleServer>().run(argc, argv);
}

#pragma once

#include "common/utils/String.h"

namespace hf3fs::fuse {
int fuseMainLoop(const String &programName,
                 bool allowOther,
                 const String &mountpoint,
                 size_t maxbufsize,
                 const String &clusterId);
}

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::mgmtd {
struct MgmtdState;
class MgmtdHeartbeater {
 public:
  explicit MgmtdHeartbeater(MgmtdState &state);
  ~MgmtdHeartbeater();

  CoTask<void> send();

  struct SendHeartbeatContext;

 private:
  MgmtdState &state_;

  std::unique_ptr<SendHeartbeatContext> sendHeartbeatCtx_;
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "common/serde/Serde.h"
#include "common/utils/UtcTime.h"

namespace hf3fs::app {
struct ConfigUpdateRecord {
  SERDE_STRUCT_FIELD(updateTime, UtcTime{});
  SERDE_STRUCT_FIELD(result, Status(StatusCode::kOK));
  SERDE_STRUCT_FIELD(description, String{});
};
}  // namespace hf3fs::app

#include "common/app/TwoPhaseApplication.h"
#include "memory/common/OverrideCppNewDelete.h"
#include "storage/service/StorageServer.h"

int main(int argc, char *argv[]) {
  using namespace hf3fs;
  return TwoPhaseApplication<storage::StorageServer>().run(argc, argv);
}

#include "common/app/TwoPhaseApplication.h"
#include "memory/common/OverrideCppNewDelete.h"
#include "mgmtd/MgmtdServer.h"

int main(int argc, char *argv[]) {
  using namespace hf3fs;
  return TwoPhaseApplication<mgmtd::MgmtdServer>().run(argc, argv);
}

#pragma once

#include "common/monitor/Recorder.h"

namespace hf3fs {

template <class T>
class ExecutorStatsReporter {
 public:
  ExecutorStatsReporter(const T &executor);

  void report();

 private:
  const T &executor_;
  monitor::Recorder::TagRef<monitor::CountRecorder> threadCountRecorder_;
  monitor::Recorder::TagRef<monitor::CountRecorder> idleThreadCountRecorder_;
  monitor::Recorder::TagRef<monitor::CountRecorder> activeThreadCountRecorder_;
  monitor::Recorder::TagRef<monitor::CountRecorder> pendingTaskCountRecorder_;
  monitor::Recorder::TagRef<monitor::CountRecorder> totalTaskCountRecorder_;
};

}  // namespace hf3fs

#pragma once

#include "WithTimestamp.h"
#include "fbs/mgmtd/ClientSession.h"

namespace hf3fs::mgmtd {
class ClientSession : public WithTimestamp<flat::ClientSession> {
 public:
  using Base = WithTimestamp<flat::ClientSession>;
  using Base::Base;

  flat::ClientSessionVersion clientSessionVersion{0};
};
}  // namespace hf3fs::mgmtd

#include "common/app/TwoPhaseApplication.h"
#include "memory/common/OverrideCppNewDelete.h"
#include "meta/service/MetaServer.h"

int main(int argc, char *argv[]) {
  using namespace hf3fs;
  return TwoPhaseApplication<meta::server::MetaServer>().run(argc, argv);
}

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {

class Dispatcher;
CoTryTask<void> registerWriteFileHandler(Dispatcher &dispatcher);

}  // namespace hf3fs::client::cli

#include "fbs/macros/SerdeDef.h"

DEFINE_SERDE_SERVICE_METHOD(Core, echo, Echo, 1)
DEFINE_SERDE_SERVICE_METHOD(Core, getConfig, GetConfig, 2)
DEFINE_SERDE_SERVICE_METHOD(Core, renderConfig, RenderConfig, 3)
DEFINE_SERDE_SERVICE_METHOD(Core, hotUpdateConfig, HotUpdateConfig, 4)
DEFINE_SERDE_SERVICE_METHOD(Core, getLastConfigUpdateRecord, GetLastConfigUpdateRecord, 5)
DEFINE_SERDE_SERVICE_METHOD(Core, shutdown, Shutdown, 6)

#include "fbs/macros/Undef.h"

#pragma once

#include <map>
#include <string>

#include "common/utils/ConfigBase.h"

namespace hf3fs::kv::fdb {
struct FDBConfig : public ConfigBase<FDBConfig> {
  CONFIG_ITEM(clusterFile, "");
  CONFIG_ITEM(enableMultipleClient, false);
  CONFIG_ITEM(externalClientDir, "");
  CONFIG_ITEM(externalClientPath, "");
  CONFIG_ITEM(multipleClientThreadNum, 4L, ConfigCheckers::checkPositive);
  CONFIG_ITEM(trace_file, "");
  CONFIG_ITEM(trace_format, "json");
  CONFIG_ITEM(casual_read_risky, false);
  CONFIG_ITEM(default_backoff, 0);
  CONFIG_ITEM(readonly, false);
};

}  // namespace hf3fs::kv::fdb

#pragma once

namespace hf3fs::lib::agent {
static const char *varTmpPath = "/var/tmp/hf3fs_client_agent";
}

#pragma once

#include "common/utils/Coroutine.h"
#include "common/utils/UtcTime.h"

namespace hf3fs::mgmtd {
struct MgmtdState;
class MgmtdTargetInfoLoader {
 public:
  explicit MgmtdTargetInfoLoader(MgmtdState &state);

  CoTask<void> run();

 private:
  SteadyTime loadedLeaseStart;
  MgmtdState &state_;
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "common/utils/ConfigBase.h"
#include "common/utils/Duration.h"

namespace hf3fs::kv {

struct TransactionRetry : ConfigBase<TransactionRetry> {
  CONFIG_HOT_UPDATED_ITEM(max_backoff, 1_s);
  CONFIG_HOT_UPDATED_ITEM(max_retry_count, 10u);
};

}  // namespace hf3fs::kv

#pragma once
#include "common/utils/ConfigBase.h"

namespace hf3fs::memory {

struct MemoryAllocatorConfig : public ConfigBase<MemoryAllocatorConfig> {
  CONFIG_HOT_UPDATED_ITEM(prof_prefix, "");
  CONFIG_HOT_UPDATED_ITEM(prof_active, bool{});
};

}  // namespace hf3fs::memory

#pragma once

#include "common/utils/StrongType.h"

namespace hf3fs::flat {
STRONG_TYPEDEF(uint32_t, NodeId);
}

#pragma once

#include "fbs/mgmtd/MgmtdLeaseInfo.h"

namespace hf3fs::mgmtd {
struct LeaseInfo {
  std::optional<flat::MgmtdLeaseInfo> lease;
  bool bootstrapping = false;
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "Path.h"
#include "Result.h"

namespace hf3fs {
Result<bool> boostFilesystemExists(const Path &path);

Result<bool> boostFilesystemIsDir(const Path &path);

Result<boost::filesystem::directory_iterator> boostFilesystemDirIter(const Path &path);

Result<bool> boostFilesystemCreateDir(const Path &path);

Result<bool> boostFilesystemCreateDirs(const Path &path);

Result<bool> boostFilesystemRemove(const Path &path);

Result<boost::uintmax_t> boostFilesystemRemoveAll(const Path &path);

Result<bool> boostFilesystemIsEmpty(const Path &path);

Result<Void> boostFilesystemCopyFile(const Path &src, const Path &dst);

Result<Void> boostFilesystemCreateSymlink(const Path &src, const Path &dst);

Result<Void> boostFilesystemCreateHardLink(const Path &src, const Path &dst);

Result<Void> boostFilesystemRename(const Path &oldPath, const Path &newPath);

}  // namespace hf3fs

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::mgmtd {
struct MgmtdState;
class MgmtdMetricsUpdater {
 public:
  explicit MgmtdMetricsUpdater(MgmtdState &state);

  CoTask<void> update();

 private:
  MgmtdState &state_;
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::mgmtd {
struct MgmtdState;
class MgmtdClientSessionsChecker {
 public:
  explicit MgmtdClientSessionsChecker(MgmtdState &state);

  CoTask<void> check();

 private:
  MgmtdState &state_;
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::mgmtd {
struct MgmtdState;
class MgmtdHeartbeatChecker {
 public:
  explicit MgmtdHeartbeatChecker(MgmtdState &state);

  CoTask<void> check();

 private:
  MgmtdState &state_;
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::mgmtd {
struct MgmtdState;
class MgmtdRoutingInfoVersionUpdater {
 public:
  explicit MgmtdRoutingInfoVersionUpdater(MgmtdState &state);

  CoTask<void> update();

 private:
  MgmtdState &state_;
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::mgmtd {
struct MgmtdState;
class MgmtdTargetInfoPersister {
 public:
  explicit MgmtdTargetInfoPersister(MgmtdState &state);

  CoTask<void> run();

 private:
  MgmtdState &state_;
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::mgmtd {
struct MgmtdState;
class MgmtdLeaseExtender {
 public:
  explicit MgmtdLeaseExtender(MgmtdState &state);

  CoTask<void> extend();

 private:
  MgmtdState &state_;
};
}  // namespace hf3fs::mgmtd

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::mgmtd {
struct MgmtdState;
class MgmtdNewBornChainsChecker {
 public:
  explicit MgmtdNewBornChainsChecker(MgmtdState &state);

  CoTask<void> check();

 private:
  MgmtdState &state_;
};
}  // namespace hf3fs::mgmtd

#pragma once

#include <algorithm>
#include <ranges>
#include <string>
#include <utility>

namespace hf3fs {

using String = std::string;

}  // namespace hf3fs

#pragma once

#include <string>

#include "common/utils/Path.h"
#include "common/utils/Result.h"

namespace hf3fs {
Result<std::string> loadFile(const Path &path);

Result<Void> storeToFile(const Path &path, const std::string &content);

}  // namespace hf3fs

#ifndef EVENT
#define EVENT(...)
#endif

#ifndef PAIR_EVENT
#define PAIR_EVENT(...)
#endif

PAIR_EVENT(Fdb, NewTransaction, 0);
PAIR_EVENT(Fdb, CommitTransaction, 1);
PAIR_EVENT(Fdb, CancelTransaction, 2);

PAIR_EVENT(Meta, LoadInode, 0);
PAIR_EVENT(Meta, SnapshotLoadInode, 1);
PAIR_EVENT(Meta, LoadDirEntry, 2);
PAIR_EVENT(Meta, SnapshotLoadDirEntry, 3);

#pragma once

#include "client/cli/common/Dispatcher.h"
#include "fbs/mgmtd/ChainInfo.h"
#include "fbs/mgmtd/NodeInfo.h"

namespace hf3fs::client::cli {
void statNode(Dispatcher::OutputTable &table, const flat::NodeInfo &node);
void statChainInfo(Dispatcher::OutputTable &table, const flat::ChainInfo &chain);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"
#include "common/utils/Path.h"
#include "fbs/storage/Common.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerFindOrphanedChunksHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "client/meta/MetaClient.h"

namespace hf3fs::tools {
void setDirLayout(meta::client::MetaClient &metaClient, const flat::UserInfo &ui);
void createWithLayout(meta::client::MetaClient &metaClient, const flat::UserInfo &ui);
}  // namespace hf3fs::tools

#pragma once

#include "MgmtdClientFetcher.h"

namespace hf3fs::core::launcher {
struct ServerMgmtdClientFetcher : public MgmtdClientFetcher {
  using MgmtdClientFetcher::MgmtdClientFetcher;
  Result<Void> completeAppInfo(flat::AppInfo &appInfo) final;
};
}  // namespace hf3fs::core::launcher

#pragma once

#include "core/app/MgmtdClientFetcher.h"

namespace hf3fs::fuse {
struct FuseConfigFetcher : public core::launcher::MgmtdClientFetcher {
  using core::launcher::MgmtdClientFetcher::MgmtdClientFetcher;
  Result<Void> completeAppInfo(flat::AppInfo &appInfo) final;
};
}  // namespace hf3fs::fuse

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerCreateHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerStatHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerRemoveHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerRenameHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerRemoveRangeHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerChdirHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerRemoteCallHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerGetConfigHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerInitClusterHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerMkdirHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerOpenRangeHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerUpdateChainHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerRemoveTargetHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerVerifyConfigHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerSetPermissionHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerOfflineTargetHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerCreateRangeHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerSetLayoutHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerRenderConfigHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerRegisterNodesHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerCreateTargetHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerSymlinkHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerSetConfigHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerScanTree(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerSetNodeTagsHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerDumpChainsHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerHotUpdateConfigHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerUploadChainsHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerUnregisterNodesHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerRecursiveChownHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerGetRealPathHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerDropUserCacheHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerUploadChainTableHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerDumpChainTableHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerDecodeUserTokenHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerRefreshRoutingInfoHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerAdminUserCtrlHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerRemoveChunksHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerDumpSessionHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerSetPreferredTargetOrderHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerRotateAsPreferredOrderHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerShutdownAllChainsHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerRotateLastSrvHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerPruneSessionHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {
class Dispatcher;
CoTryTask<void> registerGetLastConfigUpdateRecordHandler(Dispatcher &dispatcher);
}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {

class Dispatcher;
CoTryTask<void> registerBenchHandler(Dispatcher &dispatcher);

}  // namespace hf3fs::client::cli

#pragma once
#define ZSTD_STATIC_LINKING_ONLY
#include "third_party/zstd/lib/zstd.h"

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {

class Dispatcher;
CoTryTask<void> registerReadBenchHandler(Dispatcher &dispatcher);

}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {

class Dispatcher;
CoTryTask<void> registerChecksumHandler(Dispatcher &dispatcher);

}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {

class Dispatcher;
CoTryTask<void> registerQueryChunkHandler(Dispatcher &dispatcher);

}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {

class Dispatcher;
CoTryTask<void> registerReadFileHandler(Dispatcher &dispatcher);

}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {

class Dispatcher;
CoTryTask<void> registerCreateTargetsHandler(Dispatcher &dispatcher);

}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {

class Dispatcher;
CoTryTask<void> registerFillZeroHandler(Dispatcher &dispatcher);

}  // namespace hf3fs::client::cli

#pragma once

#include "common/utils/Coroutine.h"

namespace hf3fs::client::cli {

class Dispatcher;
CoTryTask<void> registerParseTargetMetaHandler(Dispatcher &dispatcher);

}  // namespace hf3fs::client::cli

#pragma once

#include "common/app/AppInfo.h"

namespace hf3fs::core::launcher {
flat::AppInfo buildBasicAppInfo(flat::NodeId nodeId, const String &clusterId);
}  // namespace hf3fs::core::launcher

#pragma once

#include "common/utils/String.h"

namespace hf3fs::net {
struct PeerInfo {
  String str;
};
}  // namespace hf3fs::net

#pragma once

namespace hf3fs {

class Thread {
 public:
  static bool blockInterruptSignals();
  static bool unblockInterruptSignals();
};

}  // namespace hf3fs

#pragma once

#include "common/utils/String.h"

namespace hf3fs::logging {
void initLogHandlers();
bool init(const String &config);
void initOrDie(const String &config);
}  // namespace hf3fs::logging

#pragma once

#include "NodeInfo.h"
#include "PersistentNodeInfo.h"

namespace hf3fs::flat {
NodeInfo toNode(const PersistentNodeInfo &pn);
PersistentNodeInfo toPersistentNode(const NodeInfo &node);
}  // namespace hf3fs::flat

#pragma once

#include "CoreServiceBase.h"

namespace hf3fs::core {
SERDE_SERVICE_CLIENT(CoreServiceClient, CoreServiceBase);
}  // namespace hf3fs::core

#pragma once

#include <folly/String.h>

#include "fbs/meta/Schema.h"

namespace hf3fs::tools {
meta::Layout layoutFromFlags();
}  // namespace hf3fs::tools

#pragma once

namespace hf3fs {

inline void HandleException() {
  //
}

}  // namespace hf3fs

#ifndef DEFINE_SERDE_SERVICE_METHOD_FULL
#define DEFINE_SERDE_SERVICE_METHOD_FULL(ServiceName, methodName, MethodName, MethodId, RequestType, ResponseType)
#endif

#ifndef DEFINE_SERDE_SERVICE_METHOD
#define DEFINE_SERDE_SERVICE_METHOD(ServiceName, methodName, MethodName, MethodId) \
  DEFINE_SERDE_SERVICE_METHOD_FULL(ServiceName, methodName, MethodName, MethodId, MethodName##Req, MethodName##Rsp)
#endif

#pragma once

#include "client/cli/common/Dispatcher.h"

namespace hf3fs::client::cli {
CoTryTask<void> registerAdminCommands(Dispatcher &dispatcher);
}

#pragma once

#include "FuseClients.h"

namespace hf3fs::fuse {
FuseClients &getFuseClientsInstance();
const fuse_lowlevel_ops &getFuseOps();
}  // namespace hf3fs::fuse

#pragma once

#include "MgmtdServiceBase.h"

namespace hf3fs::mgmtd {
SERDE_SERVICE_CLIENT(MgmtdServiceClient, MgmtdServiceBase);
}  // namespace hf3fs::mgmtd

#pragma once

#include "Result.h"

namespace hf3fs {
Result<Void> suicide(bool graceful);
}

#pragma once

#include "mgmtd/ops/EnableDisableNodeOperation.h"
#include "mgmtd/ops/ExtendClientSessionOperation.h"
#include "mgmtd/ops/GetClientSessionOperation.h"
#include "mgmtd/ops/GetConfigOperation.h"
#include "mgmtd/ops/GetConfigVersionsOperation.h"
#include "mgmtd/ops/GetPrimaryMgmtdOperation.h"
#include "mgmtd/ops/GetRoutingInfoOperation.h"
#include "mgmtd/ops/GetUniversalTagsOperation.h"
#include "mgmtd/ops/HeartbeatOperation.h"
#include "mgmtd/ops/ListClientSessionsOperation.h"
#include "mgmtd/ops/ListOrphanTargetsOperation.h"
#include "mgmtd/ops/RegisterNodeOperation.h"
#include "mgmtd/ops/RotateAsPreferredOrderOperation.h"
#include "mgmtd/ops/RotateLastSrvOperation.h"
#include "mgmtd/ops/SetChainTableOperation.h"
#include "mgmtd/ops/SetChainsOperation.h"
#include "mgmtd/ops/SetConfigOperation.h"
#include "mgmtd/ops/SetNodeTagsOperation.h"
#include "mgmtd/ops/SetPreferredTargetOrderOperation.h"
#include "mgmtd/ops/SetUniversalTagsOperation.h"
#include "mgmtd/ops/UnregisterNodeOperation.h"
#include "mgmtd/ops/UpdateChainOperation.h"

#pragma once

#include "EchoOperation.h"
#include "GetConfigOperation.h"
#include "GetLastConfigUpdateRecordOperation.h"
#include "HotUpdateConfigOperation.h"
#include "RenderConfigOperation.h"
#include "ShutdownOperation.h"

#ifdef DEFINE_SERDE_SERVICE_METHOD
#undef DEFINE_SERDE_SERVICE_METHOD
#endif

#ifdef DEFINE_SERDE_SERVICE_METHOD_FULL
#undef DEFINE_SERDE_SERVICE_METHOD_FULL
#endif

class
SerdeBuffer
{
public:
uint8_t *buff() {}
const uint8_t *buff() const {}
auto &header() {}
auto &header() const {}
uint8_t *rawMsg() {}
const uint8_t *rawMsg() const {}
uint32_t length() const {}
size_t capacity() const {}
protected:
SerdeBuffer() {}
uint8_t *self() {}
const uint8_t *self() const {}
private:
uint32_t headroom_;
uint32_t capacity_;
};

This code segment functions to: Serialize structured data into a memory buffer with header information, apply optional compression based on configuration parameters, and manage memory allocation/deallocation for network transmission while ensuring data integrity through checksum verification.

```cpp
[TASK_BEGIN]
class SerdeBuffer {
public:
    uint8_t *buff() { return self() + sizeof(SerdeBuffer); }
    const uint8_t *buff() const { return self() + sizeof(SerdeBuffer); }
    auto &header() { return *reinterpret_cast<MessageHeader*>(buff() + headroom_); }
    auto &header() const { return *reinterpret_cast<const MessageHeader*>(buff() + headroom_); }
    uint8_t *rawMsg() { return buff() + headroom_ + sizeof(MessageHeader); }
    const uint8_t *rawMsg() const { return buff() + headroom_ + sizeof(MessageHeader); }
    uint32_t length() const { return sizeof(MessageHeader) + header().size; }
    size_t capacity() const { return capacity_; }
protected:
    SerdeBuffer() = default;
    uint8_t *self() { return reinterpret_cast<uint8_t*>(this); }
    const uint8_t *self() const { return reinterpret_cast<const uint8_t*>(this); }
private:
    uint32_t headroom_;
    uint32_t capacity_;
};
[TASK_END]
```

class SerdeBuffer {
 public:
  uint8_t *buff() { return self() + headroom_; }
  const uint8_t *buff() const { return self() + headroom_; }
  auto &header() { return *reinterpret_cast<MessageHeader *>(buff()); }
  auto &header() const { return *reinterpret_cast<const MessageHeader *>(buff()); }
  uint8_t *rawMsg() { return buff() + kMessageHeaderSize; }
  const uint8_t *rawMsg() const { return buff() + kMessageHeaderSize; }
  uint32_t length() const { return kMessageHeaderSize + header().size; }
  size_t capacity() const { return capacity_; }
  template <class Allocator>
  struct Deleter {
    void operator()(SerdeBuffer *buf) { Allocator::deallocate(buf->self(), buf->capacity()); }
  };
  template <serde::SerdeType T, class Allocator = net::Allocator<4_KB, 1024, 64 * 1024>>
  static auto create(const T &packet, const CoreRequestOptions &options) {
    serde::Out<DownwardBytes<Allocator>> out;
    serde::serialize(packet, out);
    MessageHeader header;
    header.size = out.bytes().size();
    out.bytes().append(&header, sizeof(header));
    out.bytes().reserve(header.size + kMessageHeaderSize + sizeof(SerdeBuffer));
    uint32_t offset;
    uint32_t capacity;
    std::unique_ptr<SerdeBuffer, Deleter<Allocator>> ptr{
        reinterpret_cast<SerdeBuffer *>(out.bytes().release(offset, capacity))};
    ptr->headroom_ = offset;
    ptr->capacity_ = capacity;
    if (packet.timestamp.has_value()) {
      auto timestamp = reinterpret_cast<hf3fs::serde::Timestamp *>(ptr->rawMsg() + ptr->header().size -
                                                                   sizeof(hf3fs::serde::Timestamp));
      if (packet.isRequest()) {
        timestamp->clientSerialized = UtcClock::now().time_since_epoch().count();
      } else {
        timestamp->serverSerialized = UtcClock::now().time_since_epoch().count();
      }
    }
    auto compress = options.compression.enable(capacity);
    if (compress) {
      auto originalSize = ptr->header().size;
      auto compressedOffset = sizeof(SerdeBuffer) + sizeof(MessageHeader);
      auto compressedBound = ZSTD_compressBound(originalSize);
      auto compressedCapacity = compressedOffset + compressedBound;
      std::unique_ptr<SerdeBuffer, Deleter<Allocator>> compressed{
          reinterpret_cast<SerdeBuffer *>(Allocator::allocate(compressedCapacity))};
      compressed->headroom_ = sizeof(SerdeBuffer);
      compressed->capacity_ = compressedCapacity;
      int ret = ZSTD_compress(compressed->rawMsg(),
                              compressedBound,
                              ptr->rawMsg(),
                              ptr->header().size,
                              options.compression.level);
      if (LIKELY(!ZSTD_isError(ret))) {
        compressed->header().size = ret;
        ptr = std::move(compressed);
      } else {
        compress = false;
      }
    }
    ptr->header().checksum = Checksum::calcSerde(ptr->rawMsg(), ptr->header().size, compress);
    return ptr;
  }
 protected:
  SerdeBuffer() = default;
  uint8_t *self() { return reinterpret_cast<uint8_t *>(this); }
  const uint8_t *self() const { return reinterpret_cast<const uint8_t *>(this); }
 private:
  uint32_t headroom_;
  uint32_t capacity_;
}

class SerdeBuffer {
public:
    uint8_t *buff() { return self() + sizeof(SerdeBuffer); }
    const uint8_t *buff() const { return self() + sizeof(SerdeBuffer); }
    auto &header() { return *reinterpret_cast<MessageHeader*>(buff() + headroom_); }
    auto &header() const { return *reinterpret_cast<const MessageHeader*>(buff() + headroom_); }
    uint8_t *rawMsg() { return buff() + headroom_ + sizeof(MessageHeader); }
    const uint8_t *rawMsg() const { return buff() + headroom_ + sizeof(MessageHeader); }
    uint32_t length() const { return sizeof(MessageHeader) + header().size; }
    size_t capacity() const { return capacity_; }
protected:
    SerdeBuffer() = default;
    uint8_t *self() { return reinterpret_cast<uint8_t*>(this); }
    const uint8_t *self() const { return reinterpret_cast<const uint8_t*>(this); }
private:
    uint32_t headroom_;
    uint32_t capacity_;
};