Delete folder asian_ai with huggingface_hub

asian_ai/asian_ai_integration.py

@@ -1,213 +0,0 @@
-"""
-MYTHOS AI - Asian AI Integration Hub
-=====================================
-Integracao completa com IAs Chinesas, Japonesas e Coreanas.
-DeepSeek, Qwen, GLM, RAGFlow, MetaGPT, CuPy, Optuna, vLLM, SGLang, EXAONE, Solar.
-"""
-
-import os
-import sys
-import json
-import logging
-import subprocess
-from pathlib import Path
-from dataclasses import dataclass, field
-from typing import Dict, List, Optional, Any
-
-logger = logging.getLogger(__name__)
-
-BASE_DIR = Path(__file__).parent
-CHINESE_DIR = BASE_DIR / "chinese"
-JAPANESE_DIR = BASE_DIR / "japanese"
-KOREAN_DIR = BASE_DIR / "korean"
-PERFORMANCE_DIR = BASE_DIR / "performance"
-
-
-@dataclass
-class RepoInfo:
-    name: str
-    path: Path
-    country: str
-    category: str
-    stars: int = 0
-    license: str = ""
-    description: str = ""
-    installed: bool = False
-    pip_installed: bool = False
-
-
-class AsianAIHub:
-    """Hub central para todas as IAs asiaticas integradas ao MYTHOS."""
-
-    def __init__(self):
-        self.repos: Dict[str, RepoInfo] = {}
-        self._discover_repos()
-        self._check_pip_packages()
-
-    def _discover_repos(self):
-        """Descobre todos os repos clonados."""
-        repo_configs = {
-            # CHINESES - DeepSeek
-            "DeepSeek-V3": {"country": "china", "category": "llm", "stars": 104000,
-                           "license": "MIT", "description": "MoE 671B, MLA attention, 14.8T tokens"},
-            "DeepSeek-R1": {"country": "china", "category": "reasoning", "stars": 92000,
-                           "license": "MIT", "description": "RL-based reasoning, self-verification"},
-            "DeepEP": {"country": "china", "category": "infrastructure", "stars": 9800,
-                      "license": "MIT", "description": "Expert-parallel communication library"},
-            "DeepGEMM": {"country": "china", "category": "infrastructure", "stars": 7500,
-                        "license": "MIT", "description": "Efficient GPU BLAS kernels"},
-            "DeepSpec": {"country": "china", "category": "inference", "stars": 5900,
-                        "license": "MIT", "description": "Speculative decoding full-stack"},
-            "3FS": {"country": "china", "category": "infrastructure", "stars": 8000,
-                   "license": "MIT", "description": "High-performance distributed file system"},
-
-            # CHINESES - Qwen/Alibaba
-            "Qwen3": {"country": "china", "category": "llm", "stars": 25000,
-                     "license": "Apache-2.0", "description": "MoE 235B, thinking modes, 100+ languages"},
-            "Qwen-Agent": {"country": "china", "category": "agent", "stars": 12000,
-                          "license": "Apache-2.0", "description": "Agent framework with planning, tools"},
-
-            # CHINESES - Plataformas
-            "RAGFlow": {"country": "china", "category": "rag", "stars": 84000,
-                       "license": "Apache-2.0", "description": "RAG engine with deep document parsing"},
-            "FastGPT": {"country": "china", "category": "rag", "stars": 29000,
-                       "license": "Apache-2.0", "description": "Knowledge base + visual workflow"},
-            "Dify": {"country": "china", "category": "platform", "stars": 135000,
-                    "license": "Apache-2.0", "description": "LLM app builder with RAG, agents, workflows"},
-            "MetaGPT": {"country": "china", "category": "agent", "stars": 59000,
-                       "license": "MIT", "description": "Multi-agent software company simulation"},
-            "PaddleOCR": {"country": "china", "category": "ocr", "stars": 84500,
-                         "license": "Apache-2.0", "description": "OCR 100+ languages, PDF to structured data"},
-
-            # CHINESES - GLM/Zhipu
-            "GLM-4": {"country": "china", "category": "llm", "stars": 10000,
-                     "license": "Apache-2.0", "description": "355B MoE, visual reasoning"},
-
-            # JAPONESAS
-            "CuPy": {"country": "japan", "category": "gpu", "stars": 9500,
-                    "license": "MIT", "description": "NumPy-compatible GPU matrix library"},
-            "Optuna": {"country": "japan", "category": "optimization", "stars": 12000,
-                      "license": "MIT", "description": "Automatic hyperparameter optimization"},
-            "PFRL": {"country": "japan", "category": "rl", "stars": 1300,
-                    "license": "MIT", "description": "PyTorch reinforcement learning library"},
-
-            # COREANAS
-            "K-EXAONE": {"country": "korea", "category": "llm", "stars": 1000,
-                        "license": "Mixed", "description": "LG AI MoE multilingual model"},
-        }
-
-        for category_dir, repos in [
-            (CHINESE_DIR, repo_configs),
-            (JAPANESE_DIR, repo_configs),
-            (KOREAN_DIR, repo_configs),
-        ]:
-            if not category_dir.exists():
-                continue
-            for repo_dir in category_dir.iterdir():
-                if repo_dir.is_dir() and repo_dir.name in repos:
-                    config = repos[repo_dir.name]
-                    self.repos[repo_dir.name] = RepoInfo(
-                        name=repo_dir.name,
-                        path=repo_dir,
-                        country=config["country"],
-                        category=config["category"],
-                        stars=config["stars"],
-                        license=config["license"],
-                        description=config["description"],
-                        installed=True,
-                    )
-
-        # Performance repos
-        if PERFORMANCE_DIR.exists():
-            perf_configs = {
-                "vLLM": {"category": "inference", "stars": 45000, "license": "Apache-2.0",
-                         "description": "High-throughput LLM serving with PagedAttention"},
-                "SGLang": {"category": "inference", "stars": 12000, "license": "Apache-2.0",
-                          "description": "Runtime for LLM with structured generation"},
-            }
-            for repo_dir in PERFORMANCE_DIR.iterdir():
-                if repo_dir.is_dir() and repo_dir.name in perf_configs:
-                    config = perf_configs[repo_dir.name]
-                    self.repos[repo_dir.name] = RepoInfo(
-                        name=repo_dir.name,
-                        path=repo_dir,
-                        country="performance",
-                        category=config["category"],
-                        stars=config["stars"],
-                        license=config["license"],
-                        description=config["description"],
-                        installed=True,
-                    )
-
-    def _check_pip_packages(self):
-        """Verifica quais pacotes estao instalados via pip."""
-        pip_packages = {
-            "optuna": "Optuna",
-            "cupy": "CuPy",
-            "cupyx": "CuPy",
-            "transformers": "Transformers",
-            "accelerate": "Accelerate",
-            "bitsandbytes": "BitsAndBytes",
-            "sentencepiece": "SentencePiece",
-            "huggingface_hub": "HuggingFace Hub",
-            "paddleocr": "PaddleOCR",
-            "paddlepaddle": "PaddlePaddle",
-        }
-        for pkg, repo_name in pip_packages.items():
-            try:
-                __import__(pkg)
-                if repo_name in self.repos:
-                    self.repos[repo_name].pip_installed = True
-            except ImportError:
-                pass
-
-    def get_summary(self) -> Dict[str, Any]:
-        """Retorna resumo de todos os repos."""
-        by_country = {"china": [], "japan": [], "korea": [], "performance": []}
-        for repo in self.repos.values():
-            by_country.setdefault(repo.country, []).append({
-                "name": repo.name,
-                "category": repo.category,
-                "stars": repo.stars,
-                "license": repo.license,
-                "description": repo.description,
-                "installed": repo.installed,
-                "pip_installed": repo.pip_installed,
-                "path": str(repo.path),
-            })
-        return {
-            "total_repos": len(self.repos),
-            "total_stars": sum(r.stars for r in self.repos.values()),
-            "by_country": by_country,
-            "pip_packages_installed": sum(1 for r in self.repos.values() if r.pip_installed),
-        }
-
-    def print_status(self):
-        """Imprime status formatado."""
-        summary = self.get_summary()
-        print(f"\n{'='*60}")
-        print(f"  MYTHOS AI - Asian AI Hub")
-        print(f"  {summary['total_repos']} repos | {summary['total_stars']:,} stars total")
-        print(f"{'='*60}")
-
-        for country, flag in [("china", "CN"), ("japan", "JP"), ("korea", "KR"), ("performance", "PERF")]:
-            repos = summary["by_country"].get(country, [])
-            if repos:
-                print(f"\n  [{flag}] {country.upper()} ({len(repos)} repos)")
-                for r in repos:
-                    status = "[pip]" if r["pip_installed"] else "[clone]"
-                    print(f"    {status} {r['name']:20s} | {r['category']:15s} | {r['stars']:>6,} stars | {r['license']}")
-
-        print(f"\n{'='*60}\n")
-
-
-def get_asian_ai_hub() -> AsianAIHub:
-    """Singleton para o Asian AI Hub."""
-    if not hasattr(get_asian_ai_hub, "_instance"):
-        get_asian_ai_hub._instance = AsianAIHub()
-    return get_asian_ai_hub._instance
-
-
-if __name__ == "__main__":
-    hub = get_asian_ai_hub()
-    hub.print_status()

asian_ai/chinese/3FS/.clang-format

@@ -1,29 +0,0 @@
-BasedOnStyle: Google
-Language: Cpp
-Standard: c++20
-AlignAfterOpenBracket: Align
-AlignEscapedNewlines: Left
-AlignTrailingComments: true
-AllowAllArgumentsOnNextLine: false 
-AllowAllParametersOfDeclarationOnNextLine: false
-AlwaysBreakAfterReturnType: None
-AlwaysBreakTemplateDeclarations: true
-BinPackArguments: false
-BinPackParameters: false
-BitFieldColonSpacing: Both
-BreakConstructorInitializers: BeforeColon
-BreakInheritanceList: BeforeColon
-IndentWidth: 2
-PointerAlignment: Right
-DerivePointerAlignment: false
-ColumnLimit: 120
-PackConstructorInitializers: Never
-SpaceAfterTemplateKeyword: true
-
-IncludeCategories:
-  - Regex: 'Flat_generated\.h("|>)'
-    Priority: 1000
-    SortPriority: 1000
-  - Regex: '"fbs/macros/'
-    Priority: 1001
-    SortPriority: 1001

asian_ai/chinese/3FS/.clang-tidy

@@ -1,117 +0,0 @@
-Checks: >
-  -*,
-  bugprone-undelegated-constructor,
-  bugprone-argument-comment,
-  bugprone-bad-signal-to-kill-thread,
-  bugprone-bool-pointer-implicit-conversion,
-  bugprone-copy-constructor-init,
-  bugprone-dangling-handle,
-  bugprone-forward-declaration-namespace,
-  bugprone-fold-init-type,
-  bugprone-inaccurate-erase,
-  bugprone-incorrect-roundings,
-  bugprone-infinite-loop,
-  bugprone-integer-division,
-  bugprone-macro-repeated-side-effects,
-  bugprone-misplaced-operator-in-strlen-in-alloc,
-  bugprone-misplaced-pointer-arithmetic-in-alloc,
-  bugprone-misplaced-widening-cast,
-  bugprone-move-forwarding-reference,
-  bugprone-multiple-statement-macro,
-  bugprone-parent-virtual-call,
-  bugprone-posix-return,
-  bugprone-reserved-identifier,
-  bugprone-signed-char-misuse,
-  bugprone-sizeof-container,
-  bugprone-sizeof-expression,
-  bugprone-string-constructor,
-  bugprone-string-integer-assignment,
-  bugprone-string-literal-with-embedded-nul,
-  bugprone-suspicious-enum-usage,
-  bugprone-suspicious-include,
-  bugprone-suspicious-memset-usage,
-  bugprone-suspicious-missing-comma,
-  bugprone-suspicious-string-compare,
-  bugprone-swapped-arguments,
-  bugprone-terminating-continue,
-  bugprone-throw-keyword-missing,
-  bugprone-too-small-loop-variable,
-  bugprone-undefined-memory-manipulation,
-  bugprone-unhandled-self-assignment,
-  bugprone-unused-raii,
-  bugprone-unused-return-value,
-  bugprone-use-after-move,
-  bugprone-virtual-near-miss,
-  # bugprone-macro-parentheses,
-  # bugprone-narrowing-conversions,
-  # bugprone-exception-escape,
-
-  performance-faster-string-find,
-  performance-for-range-copy,
-  performance-implicit-conversion-in-loop,
-  performance-inefficient-algorithm,
-  performance-inefficient-vector-operation,
-  performance-move-constructor-init,
-  performance-no-automatic-move,
-  performance-trivially-destructible,
-  performance-unnecessary-copy-initialization,
-  performance-move-const-arg,
-
-  modernize-avoid-bind,
-  modernize-loop-convert,
-  modernize-make-shared,
-  modernize-make-unique,
-  modernize-raw-string-literal,
-  modernize-redundant-void-arg,
-  modernize-replace-auto-ptr,
-  modernize-replace-random-shuffle,
-  modernize-use-auto,
-  modernize-use-bool-literals,
-  modernize-use-nullptr,
-  modernize-use-using,
-  modernize-use-override,
-  modernize-use-equals-default,
-  modernize-use-equals-delete,
-
-  misc-throw-by-value-catch-by-reference,
-  misc-misplaced-const,
-  misc-unconventional-assign-operator,
-  misc-redundant-expression,
-  misc-static-assert,
-  misc-unconventional-assign-operator,
-  misc-uniqueptr-reset-release,
-  misc-unused-alias-decls,
-  misc-unused-parameters,
-  misc-unused-using-decls,
-
-  readability-avoid-const-params-in-decls
-  readability-const-return-type,
-  readability-container-size-empty,
-  readability-delete-null-pointer,
-  readability-deleted-default,
-  readability-misplaced-array-index,
-  readability-non-const-parameter,
-  readability-redundant-control-flow,
-  readability-redundant-function-ptr-dereference,
-  readability-redundant-smartptr-get,
-  readability-redundant-string-cstr,
-  readability-redundant-string-init,
-  readability-static-definition-in-anonymous-namespace,
-  readability-string-compare,
-  readability-uniqueptr-delete-release,
-  readability-simplify-subscript-expr,
-  readability-simplify-boolean-expr,
-  readability-inconsistent-declaration-parameter-name,
-  # readability-qualified-auto,
-
-  cert-flp30-c,
-  cert-mem57-cpp,
-  cert-oop58-cpp,
-  google-build-explicit-make-pair,
-  google-runtime-operator,
-  hicpp-exception-baseclass,
-  cppcoreguidelines-virtual-class-destructor,
-WarningsAsErrors: ""
-CheckOptions:
-  - key: performance-move-const-arg.CheckTriviallyCopyableMove
-    value: false

asian_ai/chinese/3FS/.clangd

@@ -1,3 +0,0 @@
-CompileFlags:
-  Add: -fcoroutines-ts
-  Remove: [-fcoroutines, -DNO_INTELLISENSE]

asian_ai/chinese/3FS/.dockerignore

@@ -1,3 +0,0 @@
-clang
-build
-deploy/*/Dockerfile

asian_ai/chinese/3FS/.github/workflows/build-dev-docker-image.yml

@@ -1,52 +0,0 @@
-name: Build development Docker image
-
-on:
-  workflow_dispatch:
-
-env:
-  REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}-dev
-
-jobs:
-  push_to_registry:
-    runs-on: ubuntu-latest
-
-    permissions:
-      packages: write
-      contents: read
-      attestations: write
-      id-token: write
-
-    steps:
-    - name: Check out the repository
-      uses: actions/checkout@v4
-
-    - name: Log in to the Container registry
-      uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
-      with:
-        registry: ${{ env.REGISTRY }}
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
-
-    - name: Extract metadata (tags, labels) for Docker
-      id: meta
-      uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
-      with:
-        images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-
-    - name: Build and push Docker image
-      id: push
-      uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
-      with:
-        context: .
-        file: ./dockerfile/dev.dockerfile
-        push: true
-        tags: ${{ steps.meta.outputs.tags }}
-        labels: ${{ steps.meta.outputs.labels }}
-
-    - name: Generate artifact attestation
-      uses: actions/attest-build-provenance@v2
-      with:
-        subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
-        subject-digest: ${{ steps.push.outputs.digest }}
-        push-to-registry: true

asian_ai/chinese/3FS/.github/workflows/build.yml

@@ -1,30 +0,0 @@
-name: Build
-
-on:
-  push:
-    branches: [ "main" ]
-  pull_request:
-    branches: [ "main" ]
-
-jobs:
-  build:
-    runs-on: self-hosted  # or `ubuntu-22.04`
-
-    steps:
-    - uses: actions/checkout@v4
-
-    - name: Prepare
-      run: |
-        sudo apt install -y cmake libuv1-dev liblz4-dev liblzma-dev libdouble-conversion-dev libdwarf-dev libunwind-dev
-        sudo apt install -y libaio-dev libgflags-dev libgoogle-glog-dev libgtest-dev libgmock-dev clang-format-14 clang-14 clang-tidy-14 lld-14
-        sudo apt install -y libgoogle-perftools-dev google-perftools libssl-dev gcc-12 g++-12 libboost-all-dev meson rustc cargo
-        wget https://github.com/apple/foundationdb/releases/download/7.1.61/foundationdb-clients_7.1.61-1_amd64.deb && sudo dpkg -i foundationdb-clients_7.1.61-1_amd64.deb
-        git clone https://github.com/libfuse/libfuse.git libfuse -b fuse-3.16.2 --depth 1 && mkdir libfuse/build && cd libfuse/build && meson setup .. && ninja && sudo ninja install && cd ../.. && rm -rf libfuse
-        git submodule update --init --recursive
-        ./patches/apply.sh
-
-    - name: Build
-      run: |
-        cargo build --release
-        cmake -S . -B build -DCMAKE_CXX_COMPILER=clang++-14 -DCMAKE_C_COMPILER=clang-14 -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DSHUFFLE_METHOD=g++11
-        cmake --build build -j 32

asian_ai/chinese/3FS/.gitignore

@@ -1,17 +0,0 @@
-/.cache
-/.vscode
-/build
-/debug
-/release
-/clang
-/clangdbg
-/packages
-*.o
-*.a
-*.so
-*.log
-compile_commands.json
-*_generated.h
-*~
-__pycache__
-/target

asian_ai/chinese/3FS/.gitmodules

@@ -1,42 +0,0 @@
-[submodule "third_party/googletest"]
-	path = third_party/googletest
-	url = https://github.com/google/googletest.git
-[submodule "third_party/folly"]
-	path = third_party/folly
-	url = https://github.com/facebook/folly.git
-[submodule "third_party/leveldb"]
-	path = third_party/leveldb
-	url = https://github.com/google/leveldb.git
-[submodule "third_party/rocksdb"]
-	path = third_party/rocksdb
-	url = https://github.com/facebook/rocksdb.git
-[submodule "third_party/scnlib"]
-	path = third_party/scnlib
-	url = https://github.com/eliaskosunen/scnlib.git
-[submodule "third_party/pybind11"]
-	path = third_party/pybind11
-	url = https://github.com/pybind/pybind11.git
-[submodule "third_party/clickhouse-cpp"]
-	path = third_party/clickhouse-cpp
-	url = https://github.com/ClickHouse/clickhouse-cpp.git
-[submodule "third_party/fmt"]
-	path = third_party/fmt
-	url = https://github.com/fmtlib/fmt.git
-[submodule "third_party/toml11"]
-	path = third_party/toml11
-	url = https://github.com/ToruNiina/toml11.git
-[submodule "third_party/jemalloc"]
-	path = third_party/jemalloc
-	url = https://github.com/jemalloc/jemalloc.git
-[submodule "third_party/mimalloc"]
-	path = third_party/mimalloc
-	url = https://github.com/microsoft/mimalloc.git
-[submodule "third_party/zstd"]
-	path = third_party/zstd
-	url = https://github.com/facebook/zstd.git
-[submodule "third_party/liburing"]
-	path = third_party/liburing
-	url = https://github.com/axboe/liburing.git
-[submodule "third_party/gtest-parallel"]
-	path = third_party/gtest-parallel
-	url = https://github.com/google/gtest-parallel.git

asian_ai/chinese/3FS/CMakeLists.txt

@@ -1,208 +0,0 @@
-cmake_minimum_required(VERSION 3.12)
-project(3FS VERSION 0.1.5 LANGUAGES C CXX)
-
-set(CMAKE_CONFIGURATION_TYPES "RelWithDebInfo;Debug;Release;MinSizeRel" CACHE STRING "" FORCE)
-if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None")
-    set (CMAKE_BUILD_TYPE "RelWithDebInfo")
-    message (STATUS "CMAKE_BUILD_TYPE is not set, set to default = ${CMAKE_BUILD_TYPE}")
-endif ()
-message (STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
-
-string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC)
-
-if(CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG" )
-    option(ENABLE_ASSERTIONS "Enable assertions" ON)
-else()
-    option(ENABLE_ASSERTIONS "Enable assertions" OFF)
-endif()
-message (STATUS "ENABLE_ASSERTIONS: ${ENABLE_ASSERTIONS}")
-
-if(ENABLE_ASSERTIONS)
-    add_definitions(-D_DEBUG)
-    # On non-Debug builds cmake automatically defines NDEBUG, so we explicitly undefine it:
-    if(NOT CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
-        # NOTE: use `add_compile_options` rather than `add_definitions` since
-        # `add_definitions` does not support generator expressions.
-        add_compile_options($<$<OR:$<COMPILE_LANGUAGE:C>,$<COMPILE_LANGUAGE:CXX>>:-UNDEBUG>)
-    endif()
-endif()
-
-set(SHUFFLE_METHODS "g++10" "g++11" "stdshuffle")
-set(SHUFFLE_METHOD "" CACHE STRING "Choose the shuffle method (REQUIRED)")
-set_property(CACHE SHUFFLE_METHOD PROPERTY STRINGS ${SHUFFLE_METHODS})
-
-if("${SHUFFLE_METHOD}" STREQUAL "")
-    message(FATAL_ERROR
-        "\n [ERROR]: SHUFFLE_METHOD is not set!\n"
-        " You must specify it using: cmake -DSHUFFLE_METHOD=<value>\n"
-        " Valid options are: ${SHUFFLE_METHODS}"
-    )
-elseif(NOT SHUFFLE_METHOD IN_LIST SHUFFLE_METHODS)
-    message(FATAL_ERROR
-        "\n [ERROR]: Invalid SHUFFLE_METHOD: '${SHUFFLE_METHOD}'\n"
-        " Valid options are: ${SHUFFLE_METHODS}"
-    )
-endif()
-
-message(STATUS "Use Shuffle Method: ${SHUFFLE_METHOD}")
-if(SHUFFLE_METHOD STREQUAL "stdshuffle")
-    add_compile_definitions(USE_STD_SHUFFLE)
-elseif(SHUFFLE_METHOD STREQUAL "g++10")
-    add_compile_definitions(USE_GCC10_SHUFFLE)
-elseif(SHUFFLE_METHOD STREQUAL "g++11")
-    add_compile_definitions(USE_GCC11_SHUFFLE)
-endif()
-
-option(OVERRIDE_CXX_NEW_DELETE "Override C++ new/delete operator" OFF)
-option(SAVE_ALLOCATE_SIZE "Use more memory to save allocate size" OFF)
-
-option(ENABLE_FUSE_APPLICATION "" ON)
-
-if (DEFINED SANITIZER AND SANITIZER)
-    set(OVERRIDE_CXX_NEW_DELETE OFF)
-endif()
-message (STATUS "OVERRIDE_CXX_NEW_DELETE: ${OVERRIDE_CXX_NEW_DELETE}")
-
-if (OVERRIDE_CXX_NEW_DELETE)
-    add_definitions(-DOVERRIDE_CXX_NEW_DELETE)
-    if (SAVE_ALLOCATE_SIZE)
-        add_definitions(-DSAVE_ALLOCATE_SIZE)
-    endif()
-endif()
-message (STATUS "SAVE_ALLOCATE_SIZE: ${SAVE_ALLOCATE_SIZE}")
-
-set(CMAKE_C_STANDARD 11)
-set(CMAKE_C_STANDARD_REQUIRED OFF)
-set(CMAKE_C_EXTENSIONS ON)
-set(CMAKE_CXX_STANDARD 20)
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
-set(CMAKE_CXX_EXTENSIONS OFF)
-set(CMAKE_POSITION_INDEPENDENT_CODE ON)
-set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
-
-if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
-    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fcoroutines-ts")
-    set(CMAKE_CXX_LINK_FLAGS "${CMAKE_CXX_LINK_FLAGS} -latomic")
-    add_link_options(-fuse-ld=lld)
-    # Do not build with libc++ (LLVM's implementation of the C++ standard library) in fdb
-    set(USE_LIBCXX OFF)
-elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fcoroutines")
-endif()
-
-# Remove project root from the __FILE__ macro variable
-add_compile_options(-fmacro-prefix-map=${CMAKE_SOURCE_DIR}=.)
-
-if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
-    add_compile_options(-msse4.2 -mavx2)
-elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
-    # ARM architecture detected:
-    # Clang on ARM uses built-in runtime library (compiler-rt) to provide symbols like '__muloti4'.
-    # Without explicitly specifying '-rtlib=compiler-rt', linking may fail due to missing '__muloti4'.
-    # '-unwindlib=libgcc' ensures proper exception unwinding compatibility.
-    add_compile_options(-march=armv8-a+crc)
-    message(STATUS "ARM architecture detected, linking with compiler-rt and libgcc.")
-    add_link_options("-rtlib=compiler-rt" "-unwindlib=libgcc")
-endif()
-add_compile_definitions(ROCKSDB_NAMESPACE=rocksdb_internal)
-include(cmake/Sanitizers.cmake)
-include(cmake/CompileFlags.cmake)
-
-# folly can't work normally under -Werror
-store_compile_flags()
-add_subdirectory("third_party/fmt" EXCLUDE_FROM_ALL)
-restore_compile_flags()
-set(ZSTD_BUILD_STATIC ON)
-add_subdirectory("third_party/zstd/build/cmake" EXCLUDE_FROM_ALL)
-set(zstd_INCLUDE_DIRS "${PROJECT_SOURCE_DIR}/third_party/zstd/lib")
-set(ZSTD_INCLUDE_DIR "${PROJECT_SOURCE_DIR}/third_party/zstd/lib")
-set(zstd_FOUND ON)
-set(ZSTD_FOUND ON)
-set(zstd_LIBRARIES "${PROJECT_BINARY_DIR}/third_party/zstd/build/cmake/lib/libzstd.a")
-set(ZSTD_LIBRARY "${PROJECT_BINARY_DIR}/third_party/zstd/build/cmake/lib/libzstd.a")
-restore_compile_flags()
-add_subdirectory("third_party/googletest" EXCLUDE_FROM_ALL)
-restore_compile_flags()
-set(FOLLY_NO_EXCEPTION_TRACER ON)
-add_subdirectory("third_party/folly" EXCLUDE_FROM_ALL)
-restore_compile_flags()
-set(LEVELDB_BUILD_TESTS OFF CACHE BOOL "Disable LevelDB tests")
-set(LEVELDB_BUILD_BENCHMARKS OFF CACHE BOOL "Disable LevelDB benchmarks")
-set(LEVELDB_INSTALL OFF CACHE BOOL "Disable LevelDB install")
-add_subdirectory("third_party/leveldb" EXCLUDE_FROM_ALL)
-restore_compile_flags()
-set(CMAKE_POLICY_DEFAULT_CMP0077 NEW)
-set(WITH_LZ4 ON)
-set(WITH_ZSTD ON)
-set(USE_RTTI ON)
-set(WITH_TESTS OFF)
-set(WITH_BENCHMARK_TOOLS OFF)
-set(WITH_TOOLS OFF)
-set(WITH_ALL_TESTS OFF)
-
-add_subdirectory("third_party/rocksdb" EXCLUDE_FROM_ALL)
-restore_compile_flags()
-set(SCN_TESTS OFF)
-set(SCN_EXAMPLES OFF)
-set(SCN_BENCHMARKS OFF)
-set(SCN_DOCS OFF)
-set(SCN_INSTALL OFF)
-set(SCN_PEDANTIC OFF)
-add_subdirectory("third_party/scnlib" EXCLUDE_FROM_ALL)
-restore_compile_flags()
-add_subdirectory("third_party/pybind11" EXCLUDE_FROM_ALL)
-restore_compile_flags()
-add_subdirectory("third_party/toml11" EXCLUDE_FROM_ALL)
-restore_compile_flags()
-set (MI_OVERRIDE OFF)
-add_subdirectory("third_party/mimalloc" EXCLUDE_FROM_ALL)
-restore_compile_flags()
-add_subdirectory("third_party/clickhouse-cpp" EXCLUDE_FROM_ALL)
-TARGET_INCLUDE_DIRECTORIES(clickhouse-cpp-lib
-    PUBLIC ${PROJECT_SOURCE_DIR}/third_party/clickhouse-cpp
-)
-TARGET_INCLUDE_DIRECTORIES(clickhouse-cpp-lib-static
-    PUBLIC ${PROJECT_SOURCE_DIR}/third_party/clickhouse-cpp
-)
-TARGET_INCLUDE_DIRECTORIES (absl-lib
-    PUBLIC ${PROJECT_SOURCE_DIR}/third_party/clickhouse-cpp/contrib
-)
-restore_compile_flags()
-add_subdirectory("third_party/liburing-cmake" EXCLUDE_FROM_ALL)
-restore_compile_flags()
-
-set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Werror -Wpedantic")
-
-set (CMAKE_CXX_FLAGS                     "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS}")
-set (CMAKE_CXX_FLAGS_RELWITHDEBINFO      "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_CXX_FLAGS_ADD}")
-set (CMAKE_CXX_FLAGS_DEBUG               "${CMAKE_CXX_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} -fno-inline ${CMAKE_CXX_FLAGS_ADD}")
-
-set (CMAKE_C_FLAGS                       "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} ${CMAKE_C_FLAGS_ADD}")
-set (CMAKE_C_FLAGS_RELWITHDEBINFO        "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_C_FLAGS_ADD}")
-set (CMAKE_C_FLAGS_DEBUG                 "${CMAKE_C_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} -fno-inline ${CMAKE_C_FLAGS_ADD}")
-
-set (CMAKE_ASM_FLAGS                     "${CMAKE_ASM_FLAGS} ${COMPILER_FLAGS} ${CMAKE_ASM_FLAGS_ADD}")
-set (CMAKE_ASM_FLAGS_RELWITHDEBINFO      "${CMAKE_ASM_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_ASM_FLAGS_ADD}")
-set (CMAKE_ASM_FLAGS_DEBUG               "${CMAKE_ASM_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} -fno-inline ${CMAKE_ASM_FLAGS_ADD}")
-
-set(Boost_USE_STATIC_LIBS ON)
-set(FDB_VERSION 7.1.5-ibe)
-
-find_package(Threads REQUIRED)
-find_package(Boost REQUIRED COMPONENTS filesystem system program_options)
-find_library(LIBUV_LIBRARY NAMES libuv1)
-enable_testing()
-
-include(cmake/CodeCoverage.cmake)
-include(cmake/CLangFormat.cmake)
-include(cmake/CLangTidy.cmake)
-include(cmake/Target.cmake)
-include(cmake/DumpConfig.cmake)
-include(cmake/Jemalloc.cmake)
-include(cmake/ApacheArrow.cmake)
-include(cmake/AddCrate.cmake)
-configure_file(cmake/CTestCustom.cmake ${CMAKE_BINARY_DIR} @ONLY)
-
-add_subdirectory(src)
-add_subdirectory(tests)
-add_subdirectory(benchmarks)

asian_ai/chinese/3FS/Cargo.lock

@@ -1,2139 +0,0 @@
-# This file is automatically @generated by Cargo.
-# It is not intended for manual editing.
-version = 4
-
-[[package]]
-name = "aho-corasick"
-version = "1.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
-dependencies = [
- "memchr",
-]
-
-[[package]]
-name = "android-tzdata"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
-
-[[package]]
-name = "android_system_properties"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
-dependencies = [
- "libc",
-]
-
-[[package]]
-name = "anes"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
-
-[[package]]
-name = "ansi_term"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
-dependencies = [
- "winapi",
-]
-
-[[package]]
-name = "anstream"
-version = "0.6.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b"
-dependencies = [
- "anstyle",
- "anstyle-parse",
- "anstyle-query",
- "anstyle-wincon",
- "colorchoice",
- "is_terminal_polyfill",
- "utf8parse",
-]
-
-[[package]]
-name = "anstyle"
-version = "1.0.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9"
-
-[[package]]
-name = "anstyle-parse"
-version = "0.2.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9"
-dependencies = [
- "utf8parse",
-]
-
-[[package]]
-name = "anstyle-query"
-version = "1.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c"
-dependencies = [
- "windows-sys 0.59.0",
-]
-
-[[package]]
-name = "anstyle-wincon"
-version = "3.0.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e"
-dependencies = [
- "anstyle",
- "once_cell",
- "windows-sys 0.59.0",
-]
-
-[[package]]
-name = "anyhow"
-version = "1.0.94"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7"
-
-[[package]]
-name = "atomic-wait"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a55b94919229f2c42292fd71ffa4b75e83193bffdd77b1e858cd55fd2d0b0ea8"
-dependencies = [
- "libc",
- "windows-sys 0.42.0",
-]
-
-[[package]]
-name = "atty"
-version = "0.2.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
-dependencies = [
- "hermit-abi 0.1.19",
- "libc",
- "winapi",
-]
-
-[[package]]
-name = "autocfg"
-version = "1.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
-
-[[package]]
-name = "bindgen"
-version = "0.69.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088"
-dependencies = [
- "bitflags 2.6.0",
- "cexpr",
- "clang-sys",
- "itertools 0.12.1",
- "lazy_static",
- "lazycell",
- "proc-macro2",
- "quote",
- "regex",
- "rustc-hash 1.1.0",
- "shlex",
- "syn 2.0.100",
-]
-
-[[package]]
-name = "bindgen"
-version = "0.71.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3"
-dependencies = [
- "bitflags 2.6.0",
- "cexpr",
- "clang-sys",
- "itertools 0.12.1",
- "log",
- "prettyplease",
- "proc-macro2",
- "quote",
- "regex",
- "rustc-hash 2.1.1",
- "shlex",
- "syn 2.0.100",
-]
-
-[[package]]
-name = "bitflags"
-version = "1.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
-
-[[package]]
-name = "bitflags"
-version = "2.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
-
-[[package]]
-name = "bumpalo"
-version = "3.16.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c"
-
-[[package]]
-name = "byteorder"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
-
-[[package]]
-name = "bzip2-sys"
-version = "0.1.11+1.0.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc"
-dependencies = [
- "cc",
- "libc",
- "pkg-config",
-]
-
-[[package]]
-name = "cast"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
-
-[[package]]
-name = "cc"
-version = "1.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc"
-dependencies = [
- "jobserver",
- "libc",
- "shlex",
-]
-
-[[package]]
-name = "cexpr"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
-dependencies = [
- "nom",
-]
-
-[[package]]
-name = "cfg-if"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
-
-[[package]]
-name = "cfg_aliases"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
-
-[[package]]
-name = "chrono"
-version = "0.4.38"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401"
-dependencies = [
- "android-tzdata",
- "iana-time-zone",
- "js-sys",
- "num-traits",
- "wasm-bindgen",
- "windows-targets",
-]
-
-[[package]]
-name = "chunk_engine"
-version = "0.1.11"
-dependencies = [
- "anyhow",
- "byteorder",
- "clap 4.5.22",
- "crc32c",
- "criterion",
- "cxx",
- "cxx-build",
- "dashmap",
- "derse",
- "lazy_static",
- "libc",
- "lockmap",
- "rand",
- "rocksdb",
- "rolling-file",
- "serde",
- "static_assertions",
- "tempfile",
- "tinyvec",
- "toml",
- "tracing",
- "tracing-appender",
- "tracing-subscriber",
-]
-
-[[package]]
-name = "ciborium"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e"
-dependencies = [
- "ciborium-io",
- "ciborium-ll",
- "serde",
-]
-
-[[package]]
-name = "ciborium-io"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757"
-
-[[package]]
-name = "ciborium-ll"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9"
-dependencies = [
- "ciborium-io",
- "half",
-]
-
-[[package]]
-name = "clang-sys"
-version = "1.8.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
-dependencies = [
- "glob",
- "libc",
- "libloading",
-]
-
-[[package]]
-name = "clap"
-version = "2.34.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
-dependencies = [
- "ansi_term",
- "atty",
- "bitflags 1.3.2",
- "strsim 0.8.0",
- "textwrap",
- "unicode-width",
- "vec_map",
-]
-
-[[package]]
-name = "clap"
-version = "4.5.22"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69371e34337c4c984bbe322360c2547210bf632eb2814bbe78a6e87a2935bd2b"
-dependencies = [
- "clap_builder",
- "clap_derive",
-]
-
-[[package]]
-name = "clap_builder"
-version = "4.5.22"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e24c1b4099818523236a8ca881d2b45db98dadfb4625cf6608c12069fcbbde1"
-dependencies = [
- "anstream",
- "anstyle",
- "clap_lex",
- "strsim 0.11.1",
-]
-
-[[package]]
-name = "clap_derive"
-version = "4.5.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab"
-dependencies = [
- "heck 0.5.0",
- "proc-macro2",
- "quote",
- "syn 2.0.100",
-]
-
-[[package]]
-name = "clap_lex"
-version = "0.7.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7"
-
-[[package]]
-name = "codespan-reporting"
-version = "0.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e"
-dependencies = [
- "termcolor",
- "unicode-width",
-]
-
-[[package]]
-name = "colorchoice"
-version = "1.0.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
-
-[[package]]
-name = "core-foundation-sys"
-version = "0.8.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
-
-[[package]]
-name = "crc32c"
-version = "0.6.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47"
-dependencies = [
- "rustc_version",
-]
-
-[[package]]
-name = "criterion"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f"
-dependencies = [
- "anes",
- "cast",
- "ciborium",
- "clap 4.5.22",
- "criterion-plot",
- "is-terminal",
- "itertools 0.10.5",
- "num-traits",
- "once_cell",
- "oorandom",
- "plotters",
- "rayon",
- "regex",
- "serde",
- "serde_derive",
- "serde_json",
- "tinytemplate",
- "walkdir",
-]
-
-[[package]]
-name = "criterion-plot"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
-dependencies = [
- "cast",
- "itertools 0.10.5",
-]
-
-[[package]]
-name = "crossbeam-channel"
-version = "0.5.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2"
-dependencies = [
- "crossbeam-utils",
-]
-
-[[package]]
-name = "crossbeam-deque"
-version = "0.8.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d"
-dependencies = [
- "crossbeam-epoch",
- "crossbeam-utils",
-]
-
-[[package]]
-name = "crossbeam-epoch"
-version = "0.9.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
-dependencies = [
- "crossbeam-utils",
-]
-
-[[package]]
-name = "crossbeam-utils"
-version = "0.8.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
-
-[[package]]
-name = "crunchy"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
-
-[[package]]
-name = "cxx"
-version = "1.0.133"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05e1ec88093d2abd9cf1b09ffd979136b8e922bf31cad966a8fe0d73233112ef"
-dependencies = [
- "cc",
- "cxxbridge-cmd",
- "cxxbridge-flags",
- "cxxbridge-macro",
- "foldhash",
- "link-cplusplus",
-]
-
-[[package]]
-name = "cxx-build"
-version = "1.0.133"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9afa390d956ee7ccb41aeed7ed7856ab3ffb4fc587e7216be7e0f83e949b4e6c"
-dependencies = [
- "cc",
- "codespan-reporting",
- "proc-macro2",
- "quote",
- "scratch",
- "syn 2.0.100",
-]
-
-[[package]]
-name = "cxxbridge-cmd"
-version = "1.0.133"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c23bfff654d6227cbc83de8e059d2f8678ede5fc3a6c5a35d5c379983cc61e6"
-dependencies = [
- "clap 4.5.22",
- "codespan-reporting",
- "proc-macro2",
- "quote",
- "syn 2.0.100",
-]
-
-[[package]]
-name = "cxxbridge-flags"
-version = "1.0.133"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7c01b36e22051bc6928a78583f1621abaaf7621561c2ada1b00f7878fbe2caa"
-
-[[package]]
-name = "cxxbridge-macro"
-version = "1.0.133"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6e14013136fac689345d17b9a6df55977251f11d333c0a571e8d963b55e1f95"
-dependencies = [
- "proc-macro2",
- "quote",
- "rustversion",
- "syn 2.0.100",
-]
-
-[[package]]
-name = "dashmap"
-version = "6.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
-dependencies = [
- "cfg-if",
- "crossbeam-utils",
- "hashbrown 0.14.5",
- "lock_api",
- "once_cell",
- "parking_lot_core",
-]
-
-[[package]]
-name = "deranged"
-version = "0.3.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4"
-dependencies = [
- "powerfmt",
-]
-
-[[package]]
-name = "derse"
-version = "0.1.32"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddcfe91e406c110568015f9cd5934396bcfdf745c3c265c00a6e5c381f0fcde5"
-dependencies = [
- "derse-derive",
- "thiserror",
- "tinyvec",
-]
-
-[[package]]
-name = "derse-derive"
-version = "0.1.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "effbaf2997637f96a3b52502d1b4d8ca7a7f4a55a5206ef6d4d71712fa2368be"
-dependencies = [
- "proc-macro-crate",
- "proc-macro2",
- "quote",
- "syn 2.0.100",
-]
-
-[[package]]
-name = "either"
-version = "1.13.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
-
-[[package]]
-name = "equivalent"
-version = "1.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
-
-[[package]]
-name = "errno"
-version = "0.3.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d"
-dependencies = [
- "libc",
- "windows-sys 0.59.0",
-]
-
-[[package]]
-name = "fastrand"
-version = "2.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4"
-
-[[package]]
-name = "foldhash"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2"
-
-[[package]]
-name = "getrandom"
-version = "0.2.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
-dependencies = [
- "cfg-if",
- "libc",
- "wasi 0.11.0+wasi-snapshot-preview1",
-]
-
-[[package]]
-name = "getrandom"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8"
-dependencies = [
- "cfg-if",
- "libc",
- "wasi 0.13.3+wasi-0.2.2",
- "windows-targets",
-]
-
-[[package]]
-name = "glob"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
-
-[[package]]
-name = "half"
-version = "2.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888"
-dependencies = [
- "cfg-if",
- "crunchy",
-]
-
-[[package]]
-name = "hashbrown"
-version = "0.14.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
-
-[[package]]
-name = "hashbrown"
-version = "0.15.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
-
-[[package]]
-name = "heck"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"
-dependencies = [
- "unicode-segmentation",
-]
-
-[[package]]
-name = "heck"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
-
-[[package]]
-name = "hermit-abi"
-version = "0.1.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
-dependencies = [
- "libc",
-]
-
-[[package]]
-name = "hermit-abi"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc"
-
-[[package]]
-name = "hf3fs-usrbio-sys"
-version = "0.1.0"
-dependencies = [
- "bindgen 0.71.1",
- "shared_memory",
- "uuid",
-]
-
-[[package]]
-name = "iana-time-zone"
-version = "0.1.61"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220"
-dependencies = [
- "android_system_properties",
- "core-foundation-sys",
- "iana-time-zone-haiku",
- "js-sys",
- "wasm-bindgen",
- "windows-core",
-]
-
-[[package]]
-name = "iana-time-zone-haiku"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
-dependencies = [
- "cc",
-]
-
-[[package]]
-name = "indexmap"
-version = "2.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f"
-dependencies = [
- "equivalent",
- "hashbrown 0.15.2",
-]
-
-[[package]]
-name = "is-terminal"
-version = "0.4.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b"
-dependencies = [
- "hermit-abi 0.4.0",
- "libc",
- "windows-sys 0.52.0",
-]
-
-[[package]]
-name = "is_terminal_polyfill"
-version = "1.70.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
-
-[[package]]
-name = "itertools"
-version = "0.10.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
-dependencies = [
- "either",
-]
-
-[[package]]
-name = "itertools"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
-dependencies = [
- "either",
-]
-
-[[package]]
-name = "itoa"
-version = "1.0.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
-
-[[package]]
-name = "jobserver"
-version = "0.1.32"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0"
-dependencies = [
- "libc",
-]
-
-[[package]]
-name = "js-sys"
-version = "0.3.74"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a865e038f7f6ed956f788f0d7d60c541fff74c7bd74272c5d4cf15c63743e705"
-dependencies = [
- "once_cell",
- "wasm-bindgen",
-]
-
-[[package]]
-name = "lazy_static"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
-
-[[package]]
-name = "lazycell"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
-
-[[package]]
-name = "libc"
-version = "0.2.167"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc"
-
-[[package]]
-name = "libloading"
-version = "0.8.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34"
-dependencies = [
- "cfg-if",
- "windows-targets",
-]
-
-[[package]]
-name = "librocksdb-sys"
-version = "0.16.0+8.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce3d60bc059831dc1c83903fb45c103f75db65c5a7bf22272764d9cc683e348c"
-dependencies = [
- "bindgen 0.69.5",
- "bzip2-sys",
- "cc",
- "glob",
- "libc",
- "libz-sys",
- "lz4-sys",
- "zstd-sys",
-]
-
-[[package]]
-name = "libz-sys"
-version = "1.1.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472"
-dependencies = [
- "cc",
- "pkg-config",
- "vcpkg",
-]
-
-[[package]]
-name = "link-cplusplus"
-version = "1.0.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d240c6f7e1ba3a28b0249f774e6a9dd0175054b52dfbb61b16eb8505c3785c9"
-dependencies = [
- "cc",
-]
-
-[[package]]
-name = "linux-raw-sys"
-version = "0.4.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"
-
-[[package]]
-name = "lock_api"
-version = "0.4.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
-dependencies = [
- "autocfg",
- "scopeguard 1.2.0",
-]
-
-[[package]]
-name = "lockmap"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8aaaea9efca9e51113b978e9c485d2d16603693b8c82fff6ebe585ebe253f451"
-dependencies = [
- "atomic-wait",
-]
-
-[[package]]
-name = "log"
-version = "0.4.22"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
-
-[[package]]
-name = "lz4-sys"
-version = "1.11.1+lz4-1.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6"
-dependencies = [
- "cc",
- "libc",
-]
-
-[[package]]
-name = "memchr"
-version = "2.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
-
-[[package]]
-name = "memoffset"
-version = "0.6.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
-dependencies = [
- "autocfg",
-]
-
-[[package]]
-name = "minimal-lexical"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
-
-[[package]]
-name = "nix"
-version = "0.23.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f3790c00a0150112de0f4cd161e3d7fc4b2d8a5542ffc35f099a2562aecb35c"
-dependencies = [
- "bitflags 1.3.2",
- "cc",
- "cfg-if",
- "libc",
- "memoffset",
-]
-
-[[package]]
-name = "nix"
-version = "0.29.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
-dependencies = [
- "bitflags 2.6.0",
- "cfg-if",
- "cfg_aliases",
- "libc",
-]
-
-[[package]]
-name = "nom"
-version = "7.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
-dependencies = [
- "memchr",
- "minimal-lexical",
-]
-
-[[package]]
-name = "nu-ansi-term"
-version = "0.46.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84"
-dependencies = [
- "overload",
- "winapi",
-]
-
-[[package]]
-name = "num-conv"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
-
-[[package]]
-name = "num-traits"
-version = "0.2.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
-dependencies = [
- "autocfg",
-]
-
-[[package]]
-name = "once_cell"
-version = "1.20.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
-
-[[package]]
-name = "oorandom"
-version = "11.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9"
-
-[[package]]
-name = "overload"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
-
-[[package]]
-name = "parking_lot_core"
-version = "0.9.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
-dependencies = [
- "cfg-if",
- "libc",
- "redox_syscall",
- "smallvec",
- "windows-targets",
-]
-
-[[package]]
-name = "pin-project-lite"
-version = "0.2.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff"
-
-[[package]]
-name = "pkg-config"
-version = "0.3.31"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2"
-
-[[package]]
-name = "plotters"
-version = "0.3.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747"
-dependencies = [
- "num-traits",
- "plotters-backend",
- "plotters-svg",
- "wasm-bindgen",
- "web-sys",
-]
-
-[[package]]
-name = "plotters-backend"
-version = "0.3.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a"
-
-[[package]]
-name = "plotters-svg"
-version = "0.3.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670"
-dependencies = [
- "plotters-backend",
-]
-
-[[package]]
-name = "powerfmt"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
-
-[[package]]
-name = "ppv-lite86"
-version = "0.2.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04"
-dependencies = [
- "zerocopy",
-]
-
-[[package]]
-name = "prettyplease"
-version = "0.2.30"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1ccf34da56fc294e7d4ccf69a85992b7dfb826b7cf57bac6a70bba3494cc08a"
-dependencies = [
- "proc-macro2",
- "syn 2.0.100",
-]
-
-[[package]]
-name = "proc-macro-crate"
-version = "3.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b"
-dependencies = [
- "toml_edit",
-]
-
-[[package]]
-name = "proc-macro-error"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
-dependencies = [
- "proc-macro-error-attr",
- "proc-macro2",
- "quote",
- "syn 1.0.109",
- "version_check",
-]
-
-[[package]]
-name = "proc-macro-error-attr"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
-dependencies = [
- "proc-macro2",
- "quote",
- "version_check",
-]
-
-[[package]]
-name = "proc-macro2"
-version = "1.0.92"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0"
-dependencies = [
- "unicode-ident",
-]
-
-[[package]]
-name = "quote"
-version = "1.0.37"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
-dependencies = [
- "proc-macro2",
-]
-
-[[package]]
-name = "rand"
-version = "0.8.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
-dependencies = [
- "libc",
- "rand_chacha",
- "rand_core",
-]
-
-[[package]]
-name = "rand_chacha"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
-dependencies = [
- "ppv-lite86",
- "rand_core",
-]
-
-[[package]]
-name = "rand_core"
-version = "0.6.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
-dependencies = [
- "getrandom 0.2.15",
-]
-
-[[package]]
-name = "rayon"
-version = "1.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
-dependencies = [
- "either",
- "rayon-core",
-]
-
-[[package]]
-name = "rayon-core"
-version = "1.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
-dependencies = [
- "crossbeam-deque",
- "crossbeam-utils",
-]
-
-[[package]]
-name = "redox_syscall"
-version = "0.5.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f"
-dependencies = [
- "bitflags 2.6.0",
-]
-
-[[package]]
-name = "regex"
-version = "1.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
-dependencies = [
- "aho-corasick",
- "memchr",
- "regex-automata",
- "regex-syntax",
-]
-
-[[package]]
-name = "regex-automata"
-version = "0.4.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
-dependencies = [
- "aho-corasick",
- "memchr",
- "regex-syntax",
-]
-
-[[package]]
-name = "regex-syntax"
-version = "0.8.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
-
-[[package]]
-name = "rocksdb"
-version = "0.22.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6bd13e55d6d7b8cd0ea569161127567cd587676c99f4472f779a0279aa60a7a7"
-dependencies = [
- "libc",
- "librocksdb-sys",
-]
-
-[[package]]
-name = "rolling-file"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8395b4f860856b740f20a296ea2cd4d823e81a2658cf05ef61be22916026a906"
-dependencies = [
- "chrono",
-]
-
-[[package]]
-name = "rustc-hash"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
-
-[[package]]
-name = "rustc-hash"
-version = "2.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
-
-[[package]]
-name = "rustc_version"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
-dependencies = [
- "semver",
-]
-
-[[package]]
-name = "rustix"
-version = "0.38.41"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6"
-dependencies = [
- "bitflags 2.6.0",
- "errno",
- "libc",
- "linux-raw-sys",
- "windows-sys 0.52.0",
-]
-
-[[package]]
-name = "rustversion"
-version = "1.0.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248"
-
-[[package]]
-name = "ryu"
-version = "1.0.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
-
-[[package]]
-name = "same-file"
-version = "1.0.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
-dependencies = [
- "winapi-util",
-]
-
-[[package]]
-name = "scopeguard"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27"
-
-[[package]]
-name = "scopeguard"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
-
-[[package]]
-name = "scratch"
-version = "1.0.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3cf7c11c38cb994f3d40e8a8cde3bbd1f72a435e4c49e85d6553d8312306152"
-
-[[package]]
-name = "semver"
-version = "1.0.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b"
-
-[[package]]
-name = "serde"
-version = "1.0.215"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f"
-dependencies = [
- "serde_derive",
-]
-
-[[package]]
-name = "serde_derive"
-version = "1.0.215"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.100",
-]
-
-[[package]]
-name = "serde_json"
-version = "1.0.133"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377"
-dependencies = [
- "itoa",
- "memchr",
- "ryu",
- "serde",
-]
-
-[[package]]
-name = "serde_spanned"
-version = "0.6.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1"
-dependencies = [
- "serde",
-]
-
-[[package]]
-name = "sharded-slab"
-version = "0.1.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"
-dependencies = [
- "lazy_static",
-]
-
-[[package]]
-name = "shared_memory"
-version = "0.12.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba8593196da75d9dc4f69349682bd4c2099f8cde114257d1ef7ef1b33d1aba54"
-dependencies = [
- "cfg-if",
- "libc",
- "nix 0.23.2",
- "rand",
- "win-sys",
-]
-
-[[package]]
-name = "shlex"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
-
-[[package]]
-name = "smallvec"
-version = "1.13.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
-
-[[package]]
-name = "static_assertions"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
-
-[[package]]
-name = "strsim"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
-
-[[package]]
-name = "strsim"
-version = "0.11.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
-
-[[package]]
-name = "structopt"
-version = "0.3.26"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10"
-dependencies = [
- "clap 2.34.0",
- "lazy_static",
- "structopt-derive",
-]
-
-[[package]]
-name = "structopt-derive"
-version = "0.4.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0"
-dependencies = [
- "heck 0.3.3",
- "proc-macro-error",
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
-[[package]]
-name = "syn"
-version = "1.0.109"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
-[[package]]
-name = "syn"
-version = "2.0.100"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
-[[package]]
-name = "tempfile"
-version = "3.14.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c"
-dependencies = [
- "cfg-if",
- "fastrand",
- "once_cell",
- "rustix",
- "windows-sys 0.59.0",
-]
-
-[[package]]
-name = "termcolor"
-version = "1.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755"
-dependencies = [
- "winapi-util",
-]
-
-[[package]]
-name = "textwrap"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
-dependencies = [
- "unicode-width",
-]
-
-[[package]]
-name = "thiserror"
-version = "1.0.69"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
-dependencies = [
- "thiserror-impl",
-]
-
-[[package]]
-name = "thiserror-impl"
-version = "1.0.69"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.100",
-]
-
-[[package]]
-name = "thread_local"
-version = "1.1.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c"
-dependencies = [
- "cfg-if",
- "once_cell",
-]
-
-[[package]]
-name = "time"
-version = "0.3.37"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21"
-dependencies = [
- "deranged",
- "itoa",
- "num-conv",
- "powerfmt",
- "serde",
- "time-core",
- "time-macros",
-]
-
-[[package]]
-name = "time-core"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
-
-[[package]]
-name = "time-macros"
-version = "0.2.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de"
-dependencies = [
- "num-conv",
- "time-core",
-]
-
-[[package]]
-name = "tinytemplate"
-version = "1.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
-dependencies = [
- "serde",
- "serde_json",
-]
-
-[[package]]
-name = "tinyvec"
-version = "1.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938"
-dependencies = [
- "tinyvec_macros",
-]
-
-[[package]]
-name = "tinyvec_macros"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
-
-[[package]]
-name = "toml"
-version = "0.8.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e"
-dependencies = [
- "serde",
- "serde_spanned",
- "toml_datetime",
- "toml_edit",
-]
-
-[[package]]
-name = "toml_datetime"
-version = "0.6.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
-dependencies = [
- "serde",
-]
-
-[[package]]
-name = "toml_edit"
-version = "0.22.22"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5"
-dependencies = [
- "indexmap",
- "serde",
- "serde_spanned",
- "toml_datetime",
- "winnow",
-]
-
-[[package]]
-name = "tracing"
-version = "0.1.41"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0"
-dependencies = [
- "pin-project-lite",
- "tracing-attributes",
- "tracing-core",
-]
-
-[[package]]
-name = "tracing-appender"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3566e8ce28cc0a3fe42519fc80e6b4c943cc4c8cef275620eb8dac2d3d4e06cf"
-dependencies = [
- "crossbeam-channel",
- "thiserror",
- "time",
- "tracing-subscriber",
-]
-
-[[package]]
-name = "tracing-attributes"
-version = "0.1.28"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.100",
-]
-
-[[package]]
-name = "tracing-core"
-version = "0.1.33"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c"
-dependencies = [
- "once_cell",
- "valuable",
-]
-
-[[package]]
-name = "tracing-log"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
-dependencies = [
- "log",
- "once_cell",
- "tracing-core",
-]
-
-[[package]]
-name = "tracing-serde"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1"
-dependencies = [
- "serde",
- "tracing-core",
-]
-
-[[package]]
-name = "tracing-subscriber"
-version = "0.3.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008"
-dependencies = [
- "chrono",
- "nu-ansi-term",
- "serde",
- "serde_json",
- "sharded-slab",
- "smallvec",
- "thread_local",
- "tracing-core",
- "tracing-log",
- "tracing-serde",
-]
-
-[[package]]
-name = "trash_cleaner"
-version = "0.1.0"
-dependencies = [
- "chrono",
- "libc",
- "nix 0.29.0",
- "scopeguard 0.3.3",
- "serde",
- "serde_json",
- "structopt",
- "tempfile",
- "tracing",
- "tracing-appender",
- "tracing-subscriber",
-]
-
-[[package]]
-name = "unicode-ident"
-version = "1.0.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83"
-
-[[package]]
-name = "unicode-segmentation"
-version = "1.12.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
-
-[[package]]
-name = "unicode-width"
-version = "0.1.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
-
-[[package]]
-name = "utf8parse"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
-
-[[package]]
-name = "uuid"
-version = "1.15.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0f540e3240398cce6128b64ba83fdbdd86129c16a3aa1a3a252efd66eb3d587"
-dependencies = [
- "getrandom 0.3.1",
-]
-
-[[package]]
-name = "valuable"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
-
-[[package]]
-name = "vcpkg"
-version = "0.2.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
-
-[[package]]
-name = "vec_map"
-version = "0.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
-
-[[package]]
-name = "version_check"
-version = "0.9.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
-
-[[package]]
-name = "walkdir"
-version = "2.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
-dependencies = [
- "same-file",
- "winapi-util",
-]
-
-[[package]]
-name = "wasi"
-version = "0.11.0+wasi-snapshot-preview1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
-
-[[package]]
-name = "wasi"
-version = "0.13.3+wasi-0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2"
-dependencies = [
- "wit-bindgen-rt",
-]
-
-[[package]]
-name = "wasm-bindgen"
-version = "0.2.97"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d15e63b4482863c109d70a7b8706c1e364eb6ea449b201a76c5b89cedcec2d5c"
-dependencies = [
- "cfg-if",
- "once_cell",
- "wasm-bindgen-macro",
-]
-
-[[package]]
-name = "wasm-bindgen-backend"
-version = "0.2.97"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d36ef12e3aaca16ddd3f67922bc63e48e953f126de60bd33ccc0101ef9998cd"
-dependencies = [
- "bumpalo",
- "log",
- "once_cell",
- "proc-macro2",
- "quote",
- "syn 2.0.100",
- "wasm-bindgen-shared",
-]
-
-[[package]]
-name = "wasm-bindgen-macro"
-version = "0.2.97"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "705440e08b42d3e4b36de7d66c944be628d579796b8090bfa3471478a2260051"
-dependencies = [
- "quote",
- "wasm-bindgen-macro-support",
-]
-
-[[package]]
-name = "wasm-bindgen-macro-support"
-version = "0.2.97"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "98c9ae5a76e46f4deecd0f0255cc223cfa18dc9b261213b8aa0c7b36f61b3f1d"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.100",
- "wasm-bindgen-backend",
- "wasm-bindgen-shared",
-]
-
-[[package]]
-name = "wasm-bindgen-shared"
-version = "0.2.97"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ee99da9c5ba11bd675621338ef6fa52296b76b83305e9b6e5c77d4c286d6d49"
-
-[[package]]
-name = "web-sys"
-version = "0.3.74"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a98bc3c33f0fe7e59ad7cd041b89034fa82a7c2d4365ca538dda6cdaf513863c"
-dependencies = [
- "js-sys",
- "wasm-bindgen",
-]
-
-[[package]]
-name = "win-sys"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b7b128a98c1cfa201b09eb49ba285887deb3cbe7466a98850eb1adabb452be5"
-dependencies = [
- "windows",
-]
-
-[[package]]
-name = "winapi"
-version = "0.3.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
-dependencies = [
- "winapi-i686-pc-windows-gnu",
- "winapi-x86_64-pc-windows-gnu",
-]
-
-[[package]]
-name = "winapi-i686-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
-
-[[package]]
-name = "winapi-util"
-version = "0.1.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
-dependencies = [
- "windows-sys 0.59.0",
-]
-
-[[package]]
-name = "winapi-x86_64-pc-windows-gnu"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
-
-[[package]]
-name = "windows"
-version = "0.34.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "45296b64204227616fdbf2614cefa4c236b98ee64dfaaaa435207ed99fe7829f"
-dependencies = [
- "windows_aarch64_msvc 0.34.0",
- "windows_i686_gnu 0.34.0",
- "windows_i686_msvc 0.34.0",
- "windows_x86_64_gnu 0.34.0",
- "windows_x86_64_msvc 0.34.0",
-]
-
-[[package]]
-name = "windows-core"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
-dependencies = [
- "windows-targets",
-]
-
-[[package]]
-name = "windows-sys"
-version = "0.42.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7"
-dependencies = [
- "windows_aarch64_gnullvm 0.42.2",
- "windows_aarch64_msvc 0.42.2",
- "windows_i686_gnu 0.42.2",
- "windows_i686_msvc 0.42.2",
- "windows_x86_64_gnu 0.42.2",
- "windows_x86_64_gnullvm 0.42.2",
- "windows_x86_64_msvc 0.42.2",
-]
-
-[[package]]
-name = "windows-sys"
-version = "0.52.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
-dependencies = [
- "windows-targets",
-]
-
-[[package]]
-name = "windows-sys"
-version = "0.59.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
-dependencies = [
- "windows-targets",
-]
-
-[[package]]
-name = "windows-targets"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
-dependencies = [
- "windows_aarch64_gnullvm 0.52.6",
- "windows_aarch64_msvc 0.52.6",
- "windows_i686_gnu 0.52.6",
- "windows_i686_gnullvm",
- "windows_i686_msvc 0.52.6",
- "windows_x86_64_gnu 0.52.6",
- "windows_x86_64_gnullvm 0.52.6",
- "windows_x86_64_msvc 0.52.6",
-]
-
-[[package]]
-name = "windows_aarch64_gnullvm"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
-
-[[package]]
-name = "windows_aarch64_gnullvm"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
-
-[[package]]
-name = "windows_aarch64_msvc"
-version = "0.34.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d"
-
-[[package]]
-name = "windows_aarch64_msvc"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
-
-[[package]]
-name = "windows_aarch64_msvc"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
-
-[[package]]
-name = "windows_i686_gnu"
-version = "0.34.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed"
-
-[[package]]
-name = "windows_i686_gnu"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
-
-[[package]]
-name = "windows_i686_gnu"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
-
-[[package]]
-name = "windows_i686_gnullvm"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
-
-[[package]]
-name = "windows_i686_msvc"
-version = "0.34.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956"
-
-[[package]]
-name = "windows_i686_msvc"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
-
-[[package]]
-name = "windows_i686_msvc"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
-
-[[package]]
-name = "windows_x86_64_gnu"
-version = "0.34.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4"
-
-[[package]]
-name = "windows_x86_64_gnu"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
-
-[[package]]
-name = "windows_x86_64_gnu"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
-
-[[package]]
-name = "windows_x86_64_gnullvm"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
-
-[[package]]
-name = "windows_x86_64_gnullvm"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
-
-[[package]]
-name = "windows_x86_64_msvc"
-version = "0.34.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9"
-
-[[package]]
-name = "windows_x86_64_msvc"
-version = "0.42.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
-
-[[package]]
-name = "windows_x86_64_msvc"
-version = "0.52.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
-
-[[package]]
-name = "winnow"
-version = "0.6.20"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b"
-dependencies = [
- "memchr",
-]
-
-[[package]]
-name = "wit-bindgen-rt"
-version = "0.33.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c"
-dependencies = [
- "bitflags 2.6.0",
-]
-
-[[package]]
-name = "zerocopy"
-version = "0.7.35"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
-dependencies = [
- "byteorder",
- "zerocopy-derive",
-]
-
-[[package]]
-name = "zerocopy-derive"
-version = "0.7.35"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.100",
-]
-
-[[package]]
-name = "zstd-sys"
-version = "2.0.13+zstd.1.5.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa"
-dependencies = [
- "cc",
- "pkg-config",
-]

asian_ai/chinese/3FS/Cargo.toml

@@ -1,22 +0,0 @@
-[workspace]
-members = [
-  "src/client/trash_cleaner",
-  "src/storage/chunk_engine",
-  "src/lib/rs/hf3fs-usrbio-sys"
-]
-default-members = [
-  "src/client/trash_cleaner",
-  "src/storage/chunk_engine",
-]
-resolver = "2"
-
-[workspace.package]
-authors = ["dev <noreply@deepseek.com>"]
-edition = "2021"
-license = "MIT"
-rust-version = "1.85.0" # MSRV
-
-[profile.release-cmake]
-debug = true
-inherits = "release"
-lto = true

asian_ai/chinese/3FS/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2025 DeepSeek
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

asian_ai/chinese/3FS/README.md

@@ -1,129 +0,0 @@
-#  Fire-Flyer File System
-
-[![Build](https://github.com/deepseek-ai/3fs/actions/workflows/build.yml/badge.svg)](https://github.com/deepseek-ai/3fs/actions/workflows/build.yml)
-[![License](https://img.shields.io/badge/LICENSE-MIT-blue.svg)](LICENSE)
-
-The Fire-Flyer File System (3FS) is a high-performance distributed file system designed to address the challenges of AI training and inference workloads. It leverages modern SSDs and RDMA networks to provide a shared storage layer that simplifies development of distributed applications. Key features and benefits of 3FS include:
-
-- Performance and Usability
-  - **Disaggregated Architecture** Combines the throughput of thousands of SSDs and the network bandwidth of hundreds of storage nodes, enabling applications to access storage resource in a locality-oblivious manner.
-  - **Strong Consistency** Implements Chain Replication with Apportioned Queries (CRAQ) for strong consistency, making application code simple and easy to reason about.
-  - **File Interfaces** Develops stateless metadata services backed by a transactional key-value store (e.g., FoundationDB). The file interface is well known and used everywhere. There is no need to learn a new storage API.
-
-- Diverse Workloads
-  - **Data Preparation** Organizes outputs of data analytics pipelines into hierarchical directory structures and manages a large volume of intermediate outputs efficiently.
-  - **Dataloaders** Eliminates the need for prefetching or shuffling datasets by enabling random access to training samples across compute nodes.
-  - **Checkpointing** Supports high-throughput parallel checkpointing for large-scale training.
-  - **KVCache for Inference** Provides a cost-effective alternative to DRAM-based caching, offering high throughput and significantly larger capacity.
-
-## Documentation
-
-* [Design Notes](docs/design_notes.md)
-* [Setup Guide](deploy/README.md)
-* [USRBIO API Reference](src/lib/api/UsrbIo.md)
-* [P Specifications](./specs/README.md)
-
-## Performance
-
-### 1. Peak throughput
-
-The following figure demonstrates the throughput of read stress test on a large 3FS cluster. This cluster consists of 180 storage nodes, each equipped with 2×200Gbps InfiniBand NICs and sixteen 14TiB NVMe SSDs. Approximately 500+ client nodes were used for the read stress test, with each client node configured with 1x200Gbps InfiniBand NIC. The final aggregate read throughput reached approximately 6.6 TiB/s with background traffic from training jobs.
-
-![Large block read throughput under stress test on a 180-node cluster](docs/images/peak_throughput.jpg)
-
-To benchmark 3FS, please use our [fio engine for USRBIO](benchmarks/fio_usrbio/README.md).
-
-### 2. GraySort
-
-We evaluated [smallpond](https://github.com/deepseek-ai/smallpond) using the GraySort benchmark, which measures sort performance on large-scale datasets. Our implementation adopts a two-phase approach: (1) partitioning data via shuffle using the prefix bits of keys, and (2) in-partition sorting. Both phases read/write data from/to 3FS.
-
-The test cluster comprised 25 storage nodes (2 NUMA domains/node, 1 storage service/NUMA, 2×400Gbps NICs/node) and 50 compute nodes (2 NUMA domains, 192 physical cores, 2.2 TiB RAM, and 1×200 Gbps NIC/node). Sorting 110.5 TiB of data across 8,192 partitions completed in 30 minutes and 14 seconds, achieving an average throughput of *3.66 TiB/min*.
-
-![](docs/images/gray_sort_server.png)
-![](docs/images/gray_sort_client.png)
-
-### 3. KVCache
-
-KVCache is a technique used to optimize the LLM inference process. It avoids redundant computations by caching the key and value vectors of previous tokens in the decoder layers.
-The top figure demonstrates the read throughput of all KVCache clients (1×400Gbps NIC/node), highlighting both peak and average values, with peak throughput reaching up to 40 GiB/s. The bottom figure presents the IOPS of removing ops from garbage collection (GC) during the same time period.
-
-![KVCache Read Throughput](./docs/images/kvcache_read_throughput.png)
-![KVCache GC IOPS](./docs/images/kvcache_gc_iops.png)
-
-## Check out source code
-
-Clone 3FS repository from GitHub:
-
-	git clone https://github.com/deepseek-ai/3fs
-
-When `deepseek-ai/3fs` has been cloned to a local file system, run the
-following commands to check out the submodules:
-
-```bash
-cd 3fs
-git submodule update --init --recursive
-./patches/apply.sh
-```
-
-## Install dependencies
-
-Install dependencies:
-
-```bash
-# for Ubuntu 20.04.
-apt install cmake libuv1-dev liblz4-dev liblzma-dev libdouble-conversion-dev libdwarf-dev libunwind-dev \
-  libaio-dev libgflags-dev libgoogle-glog-dev libgtest-dev libgmock-dev clang-format-14 clang-14 clang-tidy-14 lld-14 \
-  libgoogle-perftools-dev google-perftools libssl-dev libclang-rt-14-dev gcc-10 g++-10 libboost1.71-all-dev build-essential
-
-# for Ubuntu 22.04.
-apt install cmake libuv1-dev liblz4-dev liblzma-dev libdouble-conversion-dev libdwarf-dev libunwind-dev \
-  libaio-dev libgflags-dev libgoogle-glog-dev libgtest-dev libgmock-dev clang-format-14 clang-14 clang-tidy-14 lld-14 \
-  libgoogle-perftools-dev google-perftools libssl-dev gcc-12 g++-12 libboost-all-dev build-essential
-
-# for openEuler 2403sp1
-yum install cmake libuv-devel lz4-devel xz-devel double-conversion-devel libdwarf-devel libunwind-devel \
-    libaio-devel gflags-devel glog-devel gtest-devel gmock-devel clang-tools-extra clang lld \
-    gperftools-devel gperftools openssl-devel gcc gcc-c++ boost-devel
-
-# for OpenCloudOS 9 and TencentOS 4
-dnf install epol-release wget git meson cmake perl lld gcc gcc-c++ autoconf lz4 lz4-devel xz xz-devel \
-    double-conversion-devel libdwarf-devel libunwind-devel libaio-devel gflags-devel glog-devel \
-    libuv-devel gmock-devel gperftools gperftools-devel openssl-devel boost-static boost-devel mono-devel \
-    libevent-devel libibverbs-devel numactl-devel python3-devel
-```
-
-Install other build prerequisites:
-
-- [`libfuse`](https://github.com/libfuse/libfuse/releases/tag/fuse-3.16.1) 3.16.1 or newer version
-- [FoundationDB](https://apple.github.io/foundationdb/getting-started-linux.html) 7.1 or newer version
-- [Rust](https://www.rust-lang.org/tools/install) toolchain: minimal 1.75.0, recommended 1.85.0 or newer version (latest stable version) 
-
-## Build 3FS
-
-Build 3FS in `build` folder:
-
-```bash
-# Replace <method> with 'g++10' or 'g++11' based on your environment
-cmake -S . -B build \
-      -DCMAKE_CXX_COMPILER=clang++-14 -DCMAKE_C_COMPILER=clang-14 \
-      -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
-      -DSHUFFLE_METHOD=<method>
-cmake --build build -j 32
-```
-
-Due to the historical use of `std::shuffle`, binaries compiled with different compiler versions (e.g., `g++10` vs. `g++11 +`) may be incompatible ([issue](https://github.com/deepseek-ai/3FS/issues/368)). To resolve this, you must explicitly specify `-DSHUFFLE_METHOD` during compilation to lock in a consistent shuffle algorithm:
-
-- Existing Clusters: Use the method corresponding to the compiler version previously used to deploy the cluster (`g++10` or `g++11`).
-- New Clusters: You can choose either `g++10` or `g++11`. However, once the cluster is deployed, you must stay with the same configuration for all future builds to maintain compatibility.
-
-### Build 3FS use Docker
-- For TencentOS-4:  `docker pull docker.io/tencentos/tencentos4-deepseek3fs-build:latest`
-- For OpenCloudOS-9:  `docker pull docker.io/opencloudos/opencloudos9-deepseek3fs-build:latest`
-  
-## Run a test cluster
-
-Follow instructions in [setup guide](deploy/README.md) to run a test cluster.
-
-## Report Issues
-
-Please visit https://github.com/deepseek-ai/3fs/issues to report issues.

asian_ai/chinese/3FS/benchmarks/CMakeLists.txt

@@ -1 +0,0 @@
-add_subdirectory(storage_bench)

asian_ai/chinese/3FS/benchmarks/fio_usrbio/Makefile

@@ -1,23 +0,0 @@
-HF3FS_INCLUDE_DIR ?= /usr/include
-HF3FS_LIB_DIR ?= /usr/lib
-FIO_SRC_DIR ?= /usr/include
-
-PLUGIN_NAME = hf3fs_usrbio
-SO_NAME = ${PLUGIN_NAME}.so
-SRC = ${PLUGIN_NAME}.cpp
-OBJ = ${PLUGIN_NAME}.o
-
-CXX = g++
-CXXFLAGS = -fPIC -fpermissive -O3 -D_GNU_SOURCE -shared -rdynamic -I${HF3FS_INCLUDE_DIR} -I${FIO_SRC_DIR} -include config-host.h
-LDFLAGS = -L${HF3FS_LIB_DIR} -lhf3fs_api_shared -Wl,-rpath=${HF3FS_LIB_DIR}
-
-.PHONY: all clean
-
-all: ${SO_NAME}
-
-${SO_NAME}: ${SRC}
-	${CXX} ${CXXFLAGS} $^ -o $@ ${LDFLAGS}
-
-clean:
-	rm -rf ${OBJ} ${SO_NAME}
-

asian_ai/chinese/3FS/benchmarks/fio_usrbio/README.md

@@ -1,40 +0,0 @@
-# FIO engine for 3FS USRBIO
-
-This repository contains the [fio] external plugin used for benchmarking [3FS] USRBIO.
-
-## Build
-
-First, build 3FS and fio.
-
-Configure the following variables:
-- `HF3FS_LIB_DIR`: directory contains `libhf3fs_api_shared.so`, the default path in 3FS repo is `3FS/build/src/lib/api`.
-- `HF3FS_INCLUDE_DIR`: directory contains `hf3fs_usrbio.h`, the default path in 3FS repo is `3FS/src/lib/api`.
-- `FIO_SRC_DIR`: directory contains `config-host.h`. After building fio, this header will be in the root of the fio repo.
-
-Then run:
-```
-make HF3FS_LIB_DIR=${HF3FS_LIB_DIR} HF3FS_INCLUDE_DIR=${HF3FS_INCLUDE_DIR} FIO_SRC_DIR=${FIO_SRC_DIR}
-```
-
-You will get the external plugin as `hf3fs_usrbio.so`.
-
-## Usage
-
-To use this plugin, set the `ioengine` args in fio as `external:hf3fs_usrbio.so`. Please refer to [fio documentation] for further explanation.
-
-Since multiple 3FS instances might be mounted, you need to specify the 3FS instance during benchmarking. This can be done by adding the `mountpoint` arguments (in the `.fio`):
-```
-mountpoint=/hf3fs/mount/point
-```
-
-To benchmark batched small I/Os, please set these four parameters to `batch_size` simultaneously:
-```
-iodepth=1024
-iodepth_batch_submit=1024
-iodepth_batch_complete_min=1024
-iodepth_batch_complete_max=1024
-```
-
-[fio]: https://github.com/axboe/fio
-[3FS]: https://github.com/deepseek-ai/3FS
-[fio documentation]: https://fio.readthedocs.io/en/latest/fio_doc.html

asian_ai/chinese/3FS/benchmarks/fio_usrbio/hf3fs_usrbio.cpp

@@ -1,286 +0,0 @@
-#include <cstdio>
-#include <cstdlib>
-#include <iostream>
-#include <fstream>
-#include <unistd.h>
-#include <chrono>
-#include <iomanip>
-#include <ctime>
-#include "hf3fs_usrbio.h"
-#include <atomic>
-#include <errno.h>
-#include <vector>
-#include <sys/stat.h>
-
-extern "C" {
-#include "fio.h"
-#include "optgroup.h"
-}
-
-struct hf3fs_usrbio_options {
-    int dummy;
-    char *mountpoint;
-    int ior_depth;
-    int ior_timeout;
-};
-
-static struct fio_option options[] = {
-    {
-        .name = "mountpoint",
-        .lname = "hf3fs mount point",
-        .type = FIO_OPT_STR_STORE,
-        .off1 = offsetof(struct hf3fs_usrbio_options, mountpoint),
-        .help = "Mount point (e.g. /hf3fs/mount/point)",
-        .def = "",
-        .category = FIO_OPT_C_ENGINE,
-        .group = FIO_OPT_G_NETIO,
-    },
-    {
-        .name = "ior_depth",
-        .lname = "hf3fs ior depth",
-        .type = FIO_OPT_INT,
-        .off1 = offsetof(struct hf3fs_usrbio_options, ior_depth),
-        .help = "Ior depth",
-        .def = "0",
-        .category = FIO_OPT_C_ENGINE,
-        .group = FIO_OPT_G_NETIO,
-    },
-    {
-        .name = "ior_timeout",
-        .lname = "hf3fs ior timeout (in ms)",
-        .type = FIO_OPT_INT,
-        .off1 = offsetof(struct hf3fs_usrbio_options, ior_timeout),
-        .help = "Ior timeout",
-        .def = "1",
-        .category = FIO_OPT_C_ENGINE,
-        .group = FIO_OPT_G_NETIO,
-    },
-    {
-        .name = NULL,
-    },
-};
-
-#define LAST_POS(f) ((f)->engine_pos)
-
-struct hf3fs_usrbio_data {
-    struct hf3fs_iov iov;
-    struct hf3fs_ior ior_r;
-    struct hf3fs_ior ior_w;
-    std::vector<struct io_u *> io_us;
-    int queued;
-    int events;
-    enum fio_ddir last_ddir;
-};
-
-static int hf3fs_usrbio_init(struct thread_data *td) {
-    td->io_ops_data = static_cast<void *>(new hf3fs_usrbio_data);
-    struct hf3fs_usrbio_options *options = td->eo;
-
-    auto &ior_r = static_cast<hf3fs_usrbio_data *>(td->io_ops_data)->ior_r;
-    auto res = hf3fs_iorcreate3(&ior_r, options->mountpoint, td->o.iodepth, true, options->ior_depth, 0, options->ior_timeout, -1);
-    if (res < 0) {
-        return res;
-    }
-
-    auto &ior_w = static_cast<hf3fs_usrbio_data *>(td->io_ops_data)->ior_w;
-    res = hf3fs_iorcreate(&ior_w, options->mountpoint, td->o.iodepth, false, options->ior_depth, -1);
-    if (res < 0) {
-        return res;
-    }
-
-    auto *data = static_cast<hf3fs_usrbio_data *>(td->io_ops_data);
-    data->io_us.resize(td->o.iodepth);
-    data->queued = 0;
-    data->events = 0;
-
-    return 0;
-}
-
-static int fio_io_end(struct thread_data *td, struct io_u *io_u, int ret) {
-    if (io_u->file && ret >= 0 && ddir_rw(io_u->ddir)) {
-        LAST_POS(io_u->file) = io_u->offset + ret;
-    }
-
-    if (ret != (int) io_u->xfer_buflen) {
-        if (ret >= 0) {
-            io_u->resid = io_u->xfer_buflen - ret;
-            io_u->error = 0;
-            return FIO_Q_COMPLETED;
-        } else {
-            io_u->error = errno;
-        }
-    }
-
-    if (io_u->error) {
-        io_u_log_error(td, io_u);
-        td_verror(td, io_u->error, "xfer");
-    }
-
-    return FIO_Q_COMPLETED;
-}
-
-static enum fio_q_status hf3fs_usrbio_queue(struct thread_data *td, struct io_u *io_u) {
-    auto &vec = static_cast<hf3fs_usrbio_data *>(td->io_ops_data)->io_us;
-    auto *sd = static_cast<hf3fs_usrbio_data *>(td->io_ops_data);
-
-    if (io_u->ddir != sd->last_ddir) {
-        if (sd->queued != 0) {
-            return FIO_Q_BUSY;
-        } else {
-            vec[sd->queued++] = io_u;
-            sd->last_ddir = io_u->ddir;
-            return FIO_Q_QUEUED;
-        }
-    } else {
-        if (sd->queued == td->o.iodepth) {
-            return FIO_Q_BUSY;
-        }
-        vec[sd->queued++] = io_u;
-        return FIO_Q_QUEUED;
-    }
-}
-
-static int hf3fs_usrbio_commit(struct thread_data *td) {
-    auto &vec = static_cast<hf3fs_usrbio_data *>(td->io_ops_data)->io_us;
-    auto *sd = static_cast<hf3fs_usrbio_data *>(td->io_ops_data);
-    auto &ior_r = static_cast<hf3fs_usrbio_data *>(td->io_ops_data)->ior_r;
-    auto &ior_w = static_cast<hf3fs_usrbio_data *>(td->io_ops_data)->ior_w;
-    auto &iov = static_cast<hf3fs_usrbio_data *>(td->io_ops_data)->iov;
-
-    if (sd->queued == 0) {
-        return 0;
-    }
-
-    io_u_mark_submit(td, sd->queued);
-
-    int res = 0;
-    bool read = (sd->last_ddir == DDIR_READ);
-    auto &ior = read ? ior_r : ior_w;
-    for (int i = 0; i < sd->queued; i++) {
-        res = hf3fs_prep_io(&ior, &iov, read, vec[i]->xfer_buf, vec[i]->file->fd, vec[i]->offset, vec[i]->xfer_buflen, nullptr);
-        if (res < 0) {
-            std::cout << "prep " << res << " " << vec[i]->file->fd << std::endl;
-            return res;
-        }
-    }
-    res = hf3fs_submit_ios(&ior);
-    if (res < 0) {
-        std::cout << "submit " << res << std::endl;
-        return res;
-    }
-
-    std::vector<struct hf3fs_cqe> cqe(sd->queued);
-    res = hf3fs_wait_for_ios(&ior, cqe.data(), sd->queued, sd->queued, nullptr);
-    if (res < 0) {
-        std::cout << "wait " << res << std::endl;
-        return res;
-    }
-
-    for (int i = 0; i < sd->queued; i++) {
-        if (cqe[i].result < 0) {
-            std::cout << "cqe error " << res << std::endl;
-            return res;
-        }
-    }
-
-    sd->events = sd->queued;
-    sd->queued = 0;
-
-    return 0;
-}
-
-static int hf3fs_usrbio_getevents(struct thread_data *td, unsigned int min, unsigned int max, const struct timespec fio_unused *t) {
-    auto &vec = static_cast<hf3fs_usrbio_data *>(td->io_ops_data)->io_us;
-    auto *sd = static_cast<hf3fs_usrbio_data *>(td->io_ops_data);
-    int ret = 0;
-    if (min) {
-        ret = sd->events;
-        sd->events = 0;
-    }
-    return ret;
-}
-
-static struct io_u *hf3fs_usrbio_event(struct thread_data *td, int event) {
-    auto &vec = static_cast<hf3fs_usrbio_data *>(td->io_ops_data)->io_us;
-    return vec[event];
-}
-
-static void hf3fs_usrbio_cleanup(struct thread_data *td) {
-    delete static_cast<hf3fs_usrbio_data *>(td->io_ops_data);
-}
-
-static int hf3fs_usrbio_open(struct thread_data *td, struct fio_file *f) {
-    int flags = 0;
-    if (td_write(td)) {
-        if (!read_only) {
-            flags = O_RDWR;
-        }
-    } else if (td_read(td)) {
-        if (!read_only) {
-            flags = O_RDWR;
-        } else {
-            flags = O_RDONLY;
-        }
-    }
-
-    f->fd = open(f->file_name, flags);
-    hf3fs_reg_fd(f->fd, 0);
-    td->o.open_files++;
-    return 0;
-}
-
-static int hf3fs_usrbio_close(struct thread_data *td, struct fio_file *f) {
-    hf3fs_dereg_fd(f->fd);
-    close(f->fd);
-    f->fd = -1;
-    return 0;
-}
-
-static int hf3fs_usrbio_alloc(struct thread_data *td, size_t total_mem) {
-    struct hf3fs_usrbio_options *options = td->eo;
-
-    auto &iov = static_cast<hf3fs_usrbio_data *>(td->io_ops_data)->iov;
-    auto res = hf3fs_iovcreate(&iov, options->mountpoint, total_mem, 0, -1);
-    if (res < 0) {
-        return res;
-    }
-
-    td->orig_buffer = iov.base;
-    return 0;
-}
-
-static void hf3fs_usrbio_free(struct thread_data *td) {
-    auto &iov = static_cast<hf3fs_usrbio_data *>(td->io_ops_data)->iov;
-    hf3fs_iovdestroy(&iov);
-}
-
-static int hf3fs_invalidate(struct thread_data *td, struct fio_file *f) {
-    return 0;
-}
-
-extern "C" {
-
-static struct ioengine_ops ioengine;
-void get_ioengine(struct ioengine_ops **ioengine_ptr) {
-    *ioengine_ptr = &ioengine;
-
-    ioengine.name = "hf3fs_usrbio",
-    ioengine.version = FIO_IOOPS_VERSION;
-    ioengine.flags = FIO_NODISKUTIL;
-    ioengine.init = hf3fs_usrbio_init;
-    ioengine.queue = hf3fs_usrbio_queue;
-    ioengine.commit = hf3fs_usrbio_commit;
-    ioengine.getevents = hf3fs_usrbio_getevents;
-    ioengine.event = hf3fs_usrbio_event;
-    ioengine.cleanup = hf3fs_usrbio_cleanup;
-    ioengine.open_file = hf3fs_usrbio_open;
-    ioengine.close_file = hf3fs_usrbio_close;
-    ioengine.invalidate = hf3fs_invalidate;
-    ioengine.get_file_size = generic_get_file_size;
-    ioengine.iomem_alloc = hf3fs_usrbio_alloc;
-    ioengine.iomem_free = hf3fs_usrbio_free;
-    ioengine.option_struct_size = sizeof(struct hf3fs_usrbio_options);
-    ioengine.options = options;
-}
-
-}

asian_ai/chinese/3FS/benchmarks/storage_bench/CMakeLists.txt

@@ -1 +0,0 @@
-target_add_bin(storage_bench "StorageBench.cc" test-fabric-lib storage-client storage memory-common follybenchmark gmock fdb mgmtd)

asian_ai/chinese/3FS/benchmarks/storage_bench/StorageBench.cc

@@ -1,291 +0,0 @@
-#include "StorageBench.h"
-
-#include <folly/init/Init.h>
-
-#include "common/monitor/Monitor.h"
-#include "memory/common/OverrideCppNewDelete.h"
-
-DEFINE_bool(benchmarkNetwork, false, "Run in network benchmark mode");
-DEFINE_bool(benchmarkStorage, false, "Run in storage benchmark mode");
-DEFINE_bool(ignoreIOError, false, "Ignore all IO errors");
-DEFINE_bool(injectRandomServerError, false, "Inject random server errors");
-DEFINE_bool(injectRandomClientError, false, "Inject random client errors");
-DEFINE_bool(retryPermanentError, false, "Retry requests with permanent errors");
-DEFINE_bool(verifyReadData, false, "Check if the read data are correct");
-DEFINE_bool(verifyReadChecksum, false, "Verify the checksum of read IOs");
-DEFINE_bool(verifyWriteChecksum, true, "Verify the checksum of write IOs");
-DEFINE_bool(randomShuffleChunkIds, false, "Random shuffle generated chunk IDs");
-DEFINE_bool(generateTestData, true, "Generate test data for read test");
-DEFINE_bool(sparseChunkIds, false, "Generate sparse chunk IDs");
-DEFINE_bool(truncateChunks, false, "Truncate chunks");
-DEFINE_bool(cleanupChunks, false, "Clean up (remove) chunks after benchmark");
-DEFINE_bool(cleanupChunksBeforeBench, false, "Clean up (remove) chunks before benchmark");
-DEFINE_bool(serverMode, false, "Run in server mode");
-DEFINE_bool(clientMode, false, "Run in client mode");
-DEFINE_bool(clusterMode, false, "Run in cluster mode (get routing info from mgmtd)");
-DEFINE_bool(printMetrics, false, "Enable printing metrics in logs");
-DEFINE_bool(reportMetrics, false, "Enable reporting metrics to ClickHouse");
-DEFINE_uint32(metaStoreType, 0, "Metadata store type (0=LevelDB, 1=RocksDB, 2=MemDB)");
-DEFINE_uint32(chunkSizeKB, 512, "Chunk size (KB)");
-DEFINE_uint32(chainTableId, 0, "Chain table id");
-DEFINE_uint32(chainTableVersion, 0, "Chain table version");
-DEFINE_string(chainIds, "", "List of chain ids");
-DEFINE_string(storageNodeIds, "", "List of storage node ids");
-DEFINE_uint32(numChains, 1, "Number of chains");
-DEFINE_uint32(numReplicas, 1, "Number of replicas");
-DEFINE_uint32(numStorageNodes, 1, "Number of storage nodes");
-DEFINE_uint32(numChunks, 1, "Number of chunks");
-DEFINE_uint32(readSize, 4096, "Read IO size");
-DEFINE_uint32(writeSize, 131072, "Write IO size");
-DEFINE_uint32(memoryAlignment, 1, "Alignment size of each IO buffer");
-DEFINE_uint32(readOffAlignment, 0, "Alignment size of each read IO offset");
-DEFINE_uint32(batchSize, 1, "Read/write batch size");
-DEFINE_uint32(readBatchSize, 0, "Read batch size");
-DEFINE_uint32(writeBatchSize, 0, "Write batch size");
-DEFINE_uint32(removeBatchSize, 0, "Remove batch size");
-DEFINE_uint32(numReadSecs, 0, "Read test duration");
-DEFINE_uint32(numWriteSecs, 0, "Write test duration");
-DEFINE_uint32(numCoroutines, 1, "Number of coroutines");
-DEFINE_uint32(numTestThreads, 1, "Number of test threads");
-DEFINE_uint32(randSeed, 0, "Random seed for chunk id generation");
-DEFINE_uint32(chunkIdPrefix, 0xFFFF, "The most significant 2 bytes of chunk ids");
-DEFINE_uint32(serviceLevel, 0, "Service level");
-DEFINE_uint32(listenPort, 0, "Listen port");
-DEFINE_uint32(clientTimeoutMS, 30000, "Client timeout (milliseconds)");
-DEFINE_string(dataPaths, folly::fs::temp_directory_path().string(), "Comma or space separated list of paths");
-DEFINE_string(clientConfig, "", "Path of client config");
-DEFINE_string(serverConfig, "", "Path of server config");
-DEFINE_string(statsFilePath, "./perfstats.csv", "Path of performance stats file");
-DEFINE_string(ibvDevices, "mlx5_0,mlx5_1", "Comma or space separated list of ibv devices");
-DEFINE_string(ibnetZones, "", "Comma or space separated list of IB network zones");
-DEFINE_string(clusterId, "stage", "Cluster id used to connect to mgmtd");
-DEFINE_string(mgmtdEndpoints,
-              "",
-              "Comma or space separated list of mgmtd endpoints, "
-              "e.g. 'RDMA://10.1.1.1:1234,RDMA://10.1.1.2:1234'");
-DEFINE_string(storageEndpoints,
-              "",
-              "Comma or space separated list of storage ids and endpoints, "
-              "e.g. '1@RDMA://10.1.1.1:1234,2@RDMA://10.1.1.2:1234'");
-DEFINE_string(monitorEndpoint, "", "Monitor endpoint");
-DEFINE_uint32(defaultPKeyIndex, 1, "IB default pkey index");
-
-namespace hf3fs::storage::benchmark {
-
-using namespace hf3fs::storage::client;
-
-std::vector<uint32_t> stringToIntVec(const std::string &str) {
-  std::vector<uint32_t> vec;
-  std::vector<std::string> substrs;
-  boost::split(substrs, str, boost::is_any_of(", "));
-
-  for (auto s : substrs) {
-    boost::trim(s);
-    if (s.empty()) continue;
-
-    uint32_t n = std::stoul(s);
-    vec.push_back(n);
-  }
-
-  return vec;
-}
-
-bool runBenchmarks() {
-  std::vector<std::string> dataPathStrs;
-  boost::split(dataPathStrs, FLAGS_dataPaths, boost::is_any_of(", "));
-
-  std::vector<hf3fs::Path> dataPaths;
-  dataPaths.reserve(dataPathStrs.size());
-
-  for (auto str : dataPathStrs) {
-    boost::trim(str);
-    if (str.empty()) continue;
-
-    dataPaths.emplace_back(str);
-  }
-
-  std::vector<std::string> endpointRawStrs;
-  boost::split(endpointRawStrs, FLAGS_storageEndpoints, boost::is_any_of(", "));
-
-  std::map<NodeId, net::Address> storageEndpoints;
-
-  for (auto str : endpointRawStrs) {
-    boost::trim(str);
-    if (str.empty()) continue;
-
-    std::vector<std::string> nodeEndpointStrs;
-    boost::split(nodeEndpointStrs, str, boost::is_any_of("@"));
-
-    if (nodeEndpointStrs.size() != 2) {
-      XLOGF(ERR, "Wrong node endpoint string: {}", str);
-      return false;
-    }
-
-    auto nodeIdStr = nodeEndpointStrs[0];
-    auto endpointStr = nodeEndpointStrs[1];
-
-    auto nodeId = (NodeId)std::stoul(nodeIdStr);
-    auto endpoint = net::Address::fromString(endpointStr);
-    storageEndpoints[nodeId] = endpoint;
-    XLOGF(WARN, "Add storage endpoint: {} @ {}", nodeId, endpoint);
-  }
-
-  if (FLAGS_clientMode && storageEndpoints.empty()) {
-    XLOGF(ERR, "No storage endpoint specified for client mode");
-    return false;
-  }
-
-  if (FLAGS_readSize > FLAGS_chunkSizeKB * 1024) {
-    XLOGF(ERR, "Read size {} is greater than chunk size {}", FLAGS_readSize, FLAGS_chunkSizeKB * 1024);
-    return false;
-  }
-
-  auto metaStoreType = static_cast<kv::KVStore::Type>(FLAGS_metaStoreType);
-
-  test::SystemSetupConfig setupConfig = {
-      FLAGS_chunkSizeKB * 1024 /*chunkSize*/,
-      FLAGS_numChains /*numChains*/,
-      FLAGS_numReplicas /*numReplicas*/,
-      FLAGS_numStorageNodes /*numStorageNodes*/,
-      dataPaths /*dataPaths*/,
-      FLAGS_clientConfig,
-      FLAGS_serverConfig,
-      storageEndpoints,
-      FLAGS_serviceLevel,
-      FLAGS_listenPort,
-      StorageClient::ImplementationType::RPC /*clientImplType*/,
-      metaStoreType,
-      true /*useFakeMgmtdClient*/,
-      !FLAGS_clientMode /*startStorageServer*/,
-      false,
-  };
-
-  std::vector<std::string> ibvDevices;
-  boost::split(ibvDevices, FLAGS_ibvDevices, boost::is_any_of(", "));
-
-  std::vector<std::string> ibnetZones;
-  boost::split(ibnetZones, FLAGS_ibnetZones, boost::is_any_of(", "));
-
-  endpointRawStrs.clear();
-  boost::split(endpointRawStrs, FLAGS_mgmtdEndpoints, boost::is_any_of(", "));
-
-  std::vector<net::Address> mgmtdEndpoints;
-
-  for (auto str : endpointRawStrs) {
-    boost::trim(str);
-    if (str.empty()) continue;
-
-    auto endpoint = net::Address::fromString(str);
-    mgmtdEndpoints.push_back(endpoint);
-    XLOGF(WARN, "Add mgmtd endpoint: {}", endpoint);
-  }
-
-  StorageBench::Options benchOptions{FLAGS_numChunks,
-                                     FLAGS_readSize,
-                                     FLAGS_writeSize,
-                                     FLAGS_batchSize,
-                                     FLAGS_numReadSecs,
-                                     FLAGS_numWriteSecs,
-                                     FLAGS_clientTimeoutMS,
-                                     FLAGS_numCoroutines,
-                                     FLAGS_numTestThreads,
-                                     FLAGS_randSeed,
-                                     (uint16_t)FLAGS_chunkIdPrefix,
-                                     FLAGS_benchmarkNetwork,
-                                     FLAGS_benchmarkStorage,
-                                     FLAGS_ignoreIOError,
-                                     FLAGS_injectRandomServerError,
-                                     FLAGS_injectRandomClientError,
-                                     FLAGS_retryPermanentError,
-                                     FLAGS_verifyReadData,
-                                     FLAGS_verifyReadChecksum,
-                                     FLAGS_verifyWriteChecksum,
-                                     FLAGS_randomShuffleChunkIds,
-                                     FLAGS_generateTestData,
-                                     FLAGS_sparseChunkIds,
-                                     FLAGS_statsFilePath,
-                                     ibvDevices,
-                                     ibnetZones,
-                                     mgmtdEndpoints,
-                                     FLAGS_clusterId,
-                                     FLAGS_chainTableId,
-                                     FLAGS_chainTableVersion,
-                                     stringToIntVec(FLAGS_chainIds),
-                                     stringToIntVec(FLAGS_storageNodeIds),
-                                     FLAGS_memoryAlignment,
-                                     FLAGS_readOffAlignment,
-                                     FLAGS_defaultPKeyIndex,
-                                     FLAGS_readBatchSize,
-                                     FLAGS_writeBatchSize,
-                                     FLAGS_removeBatchSize};
-
-  StorageBench bench(setupConfig, benchOptions);
-
-  if (FLAGS_clusterMode) {
-    if (!bench.connect()) {
-      XLOGF(WARN, "Failed to connect to cluster");
-      return false;
-    }
-  } else {
-    if (!bench.setup()) {
-      XLOGF(WARN, "Failed to set up benchmark");
-      return false;
-    }
-  }
-
-  bench.generateChunkIds();
-
-  if (FLAGS_cleanupChunksBeforeBench) {
-    bench.cleanup();
-  }
-
-  bool runOK = true;
-
-  if (FLAGS_serverMode) {
-    XLOGF(WARN, "Waiting...");
-    while (true) {
-      ::sleep(1);
-    }
-  } else {
-    runOK = bench.run();
-  }
-
-  if (FLAGS_truncateChunks) {
-    bench.truncate();
-  }
-
-  if (FLAGS_cleanupChunks) {
-    bench.cleanup();
-  }
-
-  bench.teardown();
-
-  return runOK;
-}
-
-}  // namespace hf3fs::storage::benchmark
-
-int main(int argc, char **argv) {
-  folly::init(&argc, &argv, true);
-  hf3fs::monitor::Monitor::Config monitorConfig;
-
-  if (FLAGS_printMetrics || FLAGS_reportMetrics) {
-    if (FLAGS_printMetrics) {
-      monitorConfig.reporters(0).set_type("log");
-    } else if (FLAGS_reportMetrics) {
-      monitorConfig.reporters(0).set_type("monitor_collector");
-      monitorConfig.reporters(0).monitor_collector().set_remote_ip(FLAGS_monitorEndpoint);
-      monitorConfig.set_reporters_length(1);
-    }
-
-    auto monitorResult = hf3fs::monitor::Monitor::start(monitorConfig);
-    XLOGF_IF(FATAL, !monitorResult, "Failed to start monitor: {}", monitorResult.error());
-  }
-
-  bool ok = hf3fs::storage::benchmark::runBenchmarks();
-
-  hf3fs::monitor::Monitor::stop();
-  hf3fs::memory::shutdown();
-
-  return ok ? EXIT_SUCCESS : EXIT_FAILURE;
-}

asian_ai/chinese/3FS/benchmarks/storage_bench/StorageBench.h

@@ -1,906 +0,0 @@
-#pragma once
-
-#include <boost/algorithm/string.hpp>
-#include <boost/core/ignore_unused.hpp>
-#include <boost/filesystem/string_file.hpp>
-#include <common/utils/UtcTime.h>
-#include <folly/experimental/coro/Collect.h>
-#include <folly/futures/Barrier.h>
-#include <folly/stats/TDigest.h>
-#include <numeric>
-#include <optional>
-#include <random>
-#include <vector>
-#include <fstream>
-
-#include "common/logging/LogInit.h"
-#include "common/net/ib/IBDevice.h"
-#include "common/utils/Duration.h"
-#include "common/utils/SysResource.h"
-#include "tests/lib/UnitTestFabric.h"
-
-namespace hf3fs::storage::benchmark {
-
-using namespace hf3fs::storage::client;
-
-class StorageBench : public test::UnitTestFabric {
- public:
-  struct Options {
-    const size_t numChunks;
-    const size_t readSize;
-    const size_t writeSize;
-    const size_t batchSize;
-    const uint64_t numReadSecs;
-    const uint64_t numWriteSecs;
-    const uint64_t clientTimeoutMS;
-    const size_t numCoroutines;
-    const size_t numTestThreads;
-    const uint32_t randSeed = 0;
-    const uint16_t chunkIdPrefix = 0xFFFF;
-    const bool benchmarkNetwork = false;
-    const bool benchmarkStorage = false;
-    const bool ignoreIOError = false;
-    const bool injectRandomServerError = false;
-    const bool injectRandomClientError = false;
-    const bool retryPermanentError = false;
-    const bool verifyReadData = false;
-    const bool verifyReadChecksum = false;
-    const bool verifyWriteChecksum = true;
-    const bool randomShuffleChunkIds = true;
-    const bool generateTestData = true;
-    const bool sparseChunkIds = true;
-    const std::string statsFilePath = "./perfstats.csv";
-    const std::vector<std::string> ibvDevices = {};
-    const std::vector<std::string> ibnetZones = {};
-    const std::vector<net::Address> mgmtdEndpoints = {};
-    const std::string clusterId = kClusterId;
-    const uint32_t chainTableId = 0;
-    const uint32_t chainTableVersion = 0;
-    const std::vector<uint32_t> chainIds = {};
-    const std::vector<uint32_t> storageNodeIds = {};
-    const size_t memoryAlignment = 1;
-    const size_t readOffAlignment = 0;
-    const size_t defaultPKeyIndex = 1;
-    size_t readBatchSize = 0;
-    size_t writeBatchSize = 0;
-    size_t removeBatchSize = 0;
-  };
-
- private:
-  static constexpr uint32_t kTDigestMaxSize = 1000;
-
-  struct ChunkInfo {
-    ChainId chainId;
-    ChunkId chunkId;
-    size_t size;
-  };
-
-  Options benchOptions_;
-  std::vector<folly::TDigest> writeLatencyDigests_;
-  std::vector<folly::TDigest> readLatencyDigests_;
-  folly::CPUThreadPoolExecutor testExecutor_;
-  std::atomic_uint64_t numWriteBytes_;
-  std::atomic_uint64_t numReadBytes_;
-  folly::Random::DefaultGenerator randGen_;
-  std::vector<std::vector<ChunkInfo>> chunkInfos_;
-  std::vector<size_t> numCreatedChunks_;
-  size_t totalNumChunks_;
-  double totalChunkGiB_;
-
- public:
-  StorageBench(const test::SystemSetupConfig &setupConfig, const Options &options)
-      : UnitTestFabric(setupConfig),
-        benchOptions_(options),
-        writeLatencyDigests_(benchOptions_.numCoroutines, folly::TDigest(kTDigestMaxSize)),
-        readLatencyDigests_(benchOptions_.numCoroutines, folly::TDigest(kTDigestMaxSize)),
-        testExecutor_(benchOptions_.numTestThreads),
-        numWriteBytes_(0),
-        numReadBytes_(0),
-        randGen_(folly::Random::create()),
-        chunkInfos_(benchOptions_.numCoroutines),
-        numCreatedChunks_(benchOptions_.numCoroutines) {
-    if (benchOptions_.readBatchSize == 0) benchOptions_.readBatchSize = benchOptions_.batchSize;
-    if (benchOptions_.writeBatchSize == 0) benchOptions_.writeBatchSize = benchOptions_.batchSize;
-    if (benchOptions_.removeBatchSize == 0) benchOptions_.removeBatchSize = benchOptions_.batchSize;
-  }
-
-  void generateChunkIds() {
-    static_assert(sizeof(benchOptions_.chunkIdPrefix) == 2);
-    uint64_t chunkIdPrefix64 = ((uint64_t)benchOptions_.chunkIdPrefix) << (UINT64_WIDTH - UINT16_WIDTH);
-    std::sort(chainIds_.begin(), chainIds_.end());
-    static thread_local std::mt19937 generator;
-    randGen_.seed(benchOptions_.randSeed);
-
-    XLOGF(WARN,
-          "Generating {} chunk ids with prefix {:08X} and random seed {}...",
-          totalNumChunks_,
-          chunkIdPrefix64,
-          benchOptions_.randSeed);
-
-    for (auto &chunkInfos : chunkInfos_) {
-      uint64_t instancePrefix = chunkIdPrefix64 | folly::Random::rand64(randGen_);
-      XLOGF(DBG3, "Random chunk id prefix {:08X}", instancePrefix);
-
-      chunkInfos.reserve(chainIds_.size() * benchOptions_.numChunks);
-
-      for (auto chainId : chainIds_) {
-        for (size_t chunkIndex = 0; chunkIndex < benchOptions_.numChunks; chunkIndex++) {
-          if (benchOptions_.sparseChunkIds) {
-            uint64_t chunkIdHigh = chunkIdPrefix64 | (folly::Random::rand64(randGen_) & 0x000000FFFFFFFFFF);
-            uint64_t chunkIdLow = (folly::Random::rand64(randGen_) << UINT32_WIDTH) + chunkIndex;
-            chunkInfos.push_back({chainId, ChunkId(chunkIdHigh, chunkIdLow), 0});
-          } else {
-            chunkInfos.push_back({chainId, ChunkId(instancePrefix, chunkIndex), 0});
-          }
-        }
-      }
-
-      if (benchOptions_.randomShuffleChunkIds) std::shuffle(chunkInfos.begin(), chunkInfos.end(), generator);
-    }
-  }
-
-  bool connect() {
-    XLOGF(INFO, "Start to connect...");
-
-    if (!setupIBSock()) {
-      return false;
-    }
-
-    mgmtdClientConfig_.set_mgmtd_server_addresses(benchOptions_.mgmtdEndpoints);
-    mgmtdClientConfig_.set_enable_auto_refresh(true);
-    mgmtdClientConfig_.set_enable_auto_heartbeat(false);
-    mgmtdClientConfig_.set_enable_auto_extend_client_session(true);
-    mgmtdClientConfig_.set_auto_refresh_interval(3_s);
-    mgmtdClientConfig_.set_auto_heartbeat_interval(3_s);
-    mgmtdClientConfig_.set_auto_extend_client_session_interval(3_s);
-    mgmtdClientConfig_.set_accept_incomplete_routing_info_during_mgmtd_bootstrapping(false);
-
-    if (!client_.start()) {
-      XLOGF(ERR, "Failed to start net client for mgmtd client");
-      return false;
-    }
-
-    XLOGF(INFO, "Creating mgmtd client...");
-
-    auto stubFactory = std::make_unique<hf3fs::stubs::RealStubFactory<hf3fs::mgmtd::MgmtdServiceStub>>(
-        stubs::ClientContextCreator{[this](net::Address addr) { return client_.serdeCtx(addr); }});
-    auto mgmtdClient = std::make_unique<hf3fs::client::MgmtdClientForClient>(benchOptions_.clusterId,
-                                                                             std::move(stubFactory),
-                                                                             mgmtdClientConfig_);
-
-    auto physicalHostnameRes = SysResource::hostname(/*physicalMachineName=*/true);
-    if (!physicalHostnameRes) {
-      XLOGF(ERR, "getHostname(true) failed: {}", physicalHostnameRes.error());
-      return false;
-    }
-
-    auto containerHostnameRes = SysResource::hostname(/*physicalMachineName=*/false);
-    if (!containerHostnameRes) {
-      XLOGF(ERR, "getHostname(false) failed: {}", containerHostnameRes.error());
-      return false;
-    }
-
-    mgmtdClient->setClientSessionPayload({clientId_.uuid.toHexString(),
-                                          flat::NodeType::CLIENT,
-                                          flat::ClientSessionData::create(
-                                              /*universalId=*/*physicalHostnameRes,
-                                              /*description=*/fmt::format("StorageBench: {}", *containerHostnameRes),
-                                              /*serviceGroups=*/std::vector<flat::ServiceGroupInfo>{},
-                                              flat::ReleaseVersion::fromVersionInfo()),
-                                          flat::UserInfo{}});
-    folly::coro::blockingWait(mgmtdClient->start(&client_.tpg().bgThreadPool().randomPick()));
-    mgmtdForClient_ = std::move(mgmtdClient);
-
-    // get routing info
-
-    for (size_t retry = 0; retry < 15; retry++) {
-      auto routingInfo = mgmtdForClient_->getRoutingInfo();
-
-      if (routingInfo == nullptr || routingInfo->raw()->chains.empty()) {
-        XLOGF(WARN, "Empty routing info, #{} retry...", retry + 1);
-        std::this_thread::sleep_for(std::chrono::milliseconds(1000));
-      } else {
-        for (const auto &[tableId, tableVersions] : routingInfo->raw()->chainTables) {
-          if (tableId == benchOptions_.chainTableId) {
-            if (tableVersions.empty()) {
-              XLOGF(WARN, "No version found for chain table with id {}", tableId);
-              return false;
-            }
-
-            XLOGF(INFO, "Found {} version(s) of chain table {}", tableVersions.size(), benchOptions_.chainTableId);
-
-            flat::ChainTable chainTable;
-
-            if (benchOptions_.chainTableVersion > 0) {
-              flat::ChainTableVersion tableVersion(benchOptions_.chainTableVersion);
-              auto tableIter = tableVersions.find(tableVersion);
-
-              if (tableIter == tableVersions.end()) {
-                XLOGF(WARN, "Version {} not found in chain table with id {}", tableVersion, tableId);
-                return false;
-              }
-
-              chainTable = tableIter->second;
-              XLOGF(INFO,
-                    "Found version {} of chain table {}: {}",
-                    benchOptions_.chainTableVersion,
-                    benchOptions_.chainTableId,
-                    chainTable.chainTableVersion);
-            } else {
-              const auto iter = --tableVersions.cend();
-              const auto &latestTable = iter->second;
-              chainTable = latestTable;
-              XLOGF(INFO,
-                    "Found latest version of chain table {}: {}",
-                    benchOptions_.chainTableId,
-                    chainTable.chainTableVersion);
-            }
-
-            XLOGF(WARN,
-                  "Selected chain table: {}@{} [{}] {} chains",
-                  chainTable.chainTableId,
-                  chainTable.chainTableVersion,
-                  chainTable.desc,
-                  chainTable.chains.size());
-
-            if (!benchOptions_.storageNodeIds.empty()) {
-              for (const auto &chainId : chainTable.chains) {
-                const auto chainInfo = routingInfo->raw()->getChain(chainId);
-                for (const auto &target : chainInfo->targets) {
-                  const auto targetInfo = routingInfo->raw()->getTarget(target.targetId);
-                  auto nodeIter = std::find(benchOptions_.storageNodeIds.begin(),
-                                            benchOptions_.storageNodeIds.end(),
-                                            *targetInfo->nodeId);
-                  if (nodeIter != benchOptions_.storageNodeIds.end()) {
-                    chainIds_.push_back(chainId);
-                    break;
-                  }
-                }
-              }
-            } else if (!benchOptions_.chainIds.empty()) {
-              for (const auto &chainId : chainTable.chains) {
-                auto chainIter = std::find(benchOptions_.chainIds.begin(), benchOptions_.chainIds.end(), chainId);
-                if (chainIter != benchOptions_.chainIds.end()) {
-                  chainIds_.push_back(chainId);
-                }
-              }
-            } else {
-              chainIds_ = chainTable.chains;
-            }
-
-            break;
-          }
-        }
-
-        if (!chainIds_.empty()) break;
-      }
-    }
-
-    if (chainIds_.empty()) {
-      XLOGF(ERR, "Failed to get chain table with id {}", benchOptions_.chainTableId);
-      return false;
-    } else {
-      XLOGF(WARN, "Selected {} replication chains for benchmark", chainIds_.size());
-    }
-
-    // create storage client
-
-    if (setupConfig_.client_config().empty()) {
-      XLOGF(ERR, "Storage client config not specified");
-      return false;
-    }
-
-    auto configRes = clientConfig_.atomicallyUpdate(setupConfig_.client_config(), false /*isHotUpdate*/);
-    if (!configRes) {
-      XLOGF(ERR, "Cannot load client config from {}, error: {}", setupConfig_.client_config(), configRes.error());
-      return false;
-    }
-
-    totalNumChunks_ = chainIds_.size() * benchOptions_.numCoroutines * benchOptions_.numChunks;
-    totalChunkGiB_ = (double)totalNumChunks_ * setupConfig_.chunk_size() / 1_GB;
-    clientConfig_.retry().set_max_retry_time(Duration(std::chrono::milliseconds(benchOptions_.clientTimeoutMS)));
-    clientConfig_.net_client().io_worker().ibsocket().set_sl(setupConfig_.service_level());
-
-    XLOGF(INFO, "Creating storage client...");
-    storageClient_ = client::StorageClient::create(clientId_, clientConfig_, *mgmtdForClient_);
-
-    return true;
-  }
-
-  bool setupIBSock() {
-    XLOGF(WARN, "Setting up IB socket...");
-
-    std::vector<net::IBConfig::Subnet> subnets;
-
-    for (const auto &ibnetZoneStr : benchOptions_.ibnetZones) {
-      std::vector<std::string> ibnetZoneSubnet;
-      boost::split(ibnetZoneSubnet, ibnetZoneStr, boost::is_any_of(":"));
-
-      if (ibnetZoneSubnet.size() != 2) {
-        XLOGF(CRITICAL, "Invalid IB zone subnet: {}", ibnetZoneStr);
-        return false;
-      }
-
-      auto zone = boost::trim_copy(ibnetZoneSubnet[0]);
-      auto subnet = boost::trim_copy(ibnetZoneSubnet[1]);
-
-      if (zone.empty() || subnet.empty()) {
-        XLOGF(CRITICAL, "Invalid IB zone subnet: {}", ibnetZoneStr);
-        return false;
-      }
-
-      subnets.emplace_back();
-      subnets.back().set_network_zones({zone});
-      subnets.back().set_subnet(*net::IBConfig::Network::from(subnet));
-      XLOGF(WARN, "Add IB network zone: {} -- {}", zone, subnet);
-    }
-
-    net::IBConfig ibConfig;
-    ibConfig.set_subnets(subnets);
-    ibConfig.set_allow_unknown_zone(false);
-    ibConfig.set_default_network_zone("$HF3FS_NETWORK_ZONE");
-    ibConfig.set_device_filter(benchOptions_.ibvDevices);
-    ibConfig.set_default_pkey_index(benchOptions_.defaultPKeyIndex);
-
-    auto ibResult = net::IBManager::start(ibConfig);
-    if (ibResult.hasError()) {
-      XLOGF(CRITICAL, "Cannot initialize IB device: {}", ibResult.error());
-      return false;
-    }
-
-    return true;
-  }
-
-  bool setup() {
-    XLOGF(WARN, "Setting up benchmark...");
-
-    if (!setupIBSock()) {
-      return false;
-    }
-
-    bool ok = setUpStorageSystem();
-
-    totalNumChunks_ = chainIds_.size() * benchOptions_.numCoroutines * benchOptions_.numChunks;
-    totalChunkGiB_ = (double)totalNumChunks_ * setupConfig_.chunk_size() / 1_GB;
-    clientConfig_.retry().set_max_retry_time(Duration(std::chrono::milliseconds(benchOptions_.clientTimeoutMS)));
-
-    return ok;
-  }
-
-  void teardown() {
-    tearDownStorageSystem();
-    net::IBManager::stop();
-  }
-
-  void printThroughput(hf3fs::SteadyClock::duration elapsedMicro, double totalGiB) {
-    auto elapsedMilli = std::chrono::duration_cast<std::chrono::milliseconds>(elapsedMicro);
-    double throughput = totalGiB / (elapsedMilli.count() / 1000.0);
-    XLOGF(WARN, "Average throughput: {:.3f}GiB/s, total {:.3f} GiB", throughput, totalGiB);
-  }
-
-  void printLatencyDigest(const folly::TDigest &digest) {
-    XLOGF(WARN, "latency summary ({} samples)", digest.count());
-    XLOGF(WARN, "min: {:10.1f}us", digest.min());
-    XLOGF(WARN, "max: {:10.1f}us", digest.max());
-    XLOGF(WARN, "avg: {:10.1f}us", digest.mean());
-    for (double p : {0.1, 0.2, 0.5, 0.9, 0.95, 0.99}) {
-      XLOGF(WARN, "{}%: {:10.1f}us", p * 100.0, digest.estimateQuantile(p));
-    }
-  }
-
-  void dumpPerfStats(const std::string &testName,
-                     const folly::TDigest &digest,
-                     hf3fs::SteadyClock::duration elapsedTime,
-                     double totalGiB,
-                     bool readIO) {
-    if (benchOptions_.statsFilePath.empty()) return;
-
-    boost::filesystem::path outFilePath(benchOptions_.statsFilePath);
-
-    if (!boost::filesystem::exists(outFilePath) || boost::filesystem::is_empty(outFilePath)) {
-      XLOGF(INFO, "Create a file for perfermance stats at {}", outFilePath);
-      std::ofstream outFile(outFilePath);
-      if (!outFile) {
-          throw std::runtime_error("Failed to open file: " + outFilePath.string());
-      }
-
-      outFile << "test name,#storages,#chains,#replicas,concurrency,batch size,"
-                 "io size (bytes),effective batch size (batch size / #replicas),elapsed time (us),"
-                 "QPS,IOPS,bandwidth (MB/s),latency samples,min latency (us),max latency (us),avg latency (us),"
-                 "latency P50 (us),latency P75 (us),latency P90 (us),latency P95 (us),latency P99 (us)\n";
-
-      if (!outFile) {
-          throw std::runtime_error("Failed to write to file: " + outFilePath.string());
-      }
-
-      outFile.close();
-    }
-
-    auto elapsedMicro = std::chrono::duration_cast<std::chrono::microseconds>(elapsedTime);
-    double bandwidthMBps = totalGiB * 1024.0 / (elapsedMicro.count() / 1'000'000.0);
-    size_t ioSize = readIO ? benchOptions_.readSize : benchOptions_.writeSize;
-    size_t batchSize = readIO ? benchOptions_.readBatchSize : benchOptions_.writeBatchSize;
-    double iops = bandwidthMBps * 1024.0 * 1024.0 / ioSize;
-    double qps = bandwidthMBps * 1024.0 * 1024.0 / (batchSize * ioSize);
-
-    boost::filesystem::ofstream fout(outFilePath, std::ios_base::app);
-
-    fout << fmt::format("{},{},{},{},{},{},{},{:.1f},{},{:.1f},{:.1f},{:.3f},{},{:.1f},{:.1f},{:.1f}",
-                        testName,
-                        setupConfig_.num_storage_nodes(),
-                        setupConfig_.num_chains(),
-                        setupConfig_.num_replicas(),
-                        benchOptions_.numCoroutines,
-                        batchSize,
-                        ioSize,
-                        double(batchSize) / setupConfig_.num_storage_nodes(),
-                        elapsedMicro.count(),
-                        qps,
-                        iops,
-                        bandwidthMBps,
-                        digest.count(),
-                        digest.min(),
-                        digest.max(),
-                        digest.mean());
-
-    for (double p : {0.5, 0.75, 0.9, 0.95, 0.99}) {
-      fout << fmt::format(",{:.1f}", digest.estimateQuantile(p));
-    }
-
-    fout << "\n";
-    fout.close();
-  }
-
-  CoTask<uint32_t> batchWrite(uint32_t instanceId, size_t writeBatchSize, size_t writeSize, uint32_t numWriteSecs) {
-    // create an aligned memory block
-    size_t memoryBlockSize = ALIGN_UPPER(setupConfig_.chunk_size(), benchOptions_.memoryAlignment);
-    auto memoryBlock = (uint8_t *)folly::aligned_malloc(memoryBlockSize, sysconf(_SC_PAGESIZE));
-    auto deleter = [](uint8_t *ptr) { folly::aligned_free(ptr); };
-    std::unique_ptr<uint8_t, decltype(deleter)> memoryBlockPtr(memoryBlock, deleter);
-    std::memset(memoryBlock, 0xFF, memoryBlockSize);
-
-    if (benchOptions_.verifyReadData) {
-      for (size_t byteIndex = 0; byteIndex < memoryBlockSize; byteIndex++) {
-        memoryBlock[byteIndex] = byteIndex;
-      }
-    }
-
-    // register a block of memory
-    auto regRes = storageClient_->registerIOBuffer(memoryBlock, memoryBlockSize);
-
-    if (regRes.hasError()) {
-      co_return regRes.error().code();
-    }
-
-    // create write IOs
-
-    auto ioBuffer = std::move(*regRes);
-
-    WriteOptions options;
-    options.set_enableChecksum(benchOptions_.verifyWriteChecksum);
-    options.debug().set_bypass_disk_io(benchOptions_.benchmarkNetwork);
-    options.debug().set_bypass_rdma_xmit(benchOptions_.benchmarkStorage);
-    options.debug().set_inject_random_server_error(benchOptions_.injectRandomServerError);
-    options.debug().set_inject_random_client_error(benchOptions_.injectRandomClientError);
-    options.retry().set_retry_permanent_error(benchOptions_.retryPermanentError);
-
-    std::vector<double> elapsedMicroSecs;
-    uint64_t numWriteBytes = 0;
-
-    std::vector<WriteIO> writeIOs;
-    writeIOs.reserve(writeBatchSize);
-
-    auto benchStart = hf3fs::SteadyClock::now();
-    std::vector<ChunkInfo> &chunkInfos = chunkInfos_[instanceId];
-    size_t &numCreatedChunks = numCreatedChunks_[instanceId];
-    size_t seqChunkIndex = 0;
-
-    while (true) {
-      if (numWriteSecs) {
-        auto accumElapsedSecs =
-            std::chrono::duration_cast<std::chrono::seconds>(hf3fs::SteadyClock::now() - benchStart);
-        if (accumElapsedSecs >= std::chrono::seconds(numWriteSecs)) break;
-      } else {
-        if (numCreatedChunks >= chunkInfos.size()) break;
-      }
-
-      writeIOs.clear();
-
-      for (size_t writeIndex = 0; writeIndex < writeBatchSize; writeIndex++) {
-        auto &[chainId, chunkId, chunkSize] = chunkInfos[seqChunkIndex++ % chunkInfos.size()];
-        size_t writeOffset = 0;
-        size_t writeLength = 0;
-
-        if (chunkSize < setupConfig_.chunk_size()) {
-          writeOffset = chunkSize;
-          writeLength = std::min(writeSize, setupConfig_.chunk_size() - writeOffset);
-          chunkSize += writeLength;
-          numCreatedChunks += chunkSize == setupConfig_.chunk_size();
-        } else {
-          writeOffset = folly::Random::rand32(0, setupConfig_.chunk_size() - writeSize);
-          writeLength = writeSize;
-        }
-
-        auto writeIO = storageClient_->createWriteIO(chainId,
-                                                     chunkId,
-                                                     writeOffset,
-                                                     writeLength,
-                                                     setupConfig_.chunk_size(),
-                                                     &memoryBlock[writeOffset],
-                                                     &ioBuffer);
-        writeIOs.push_back(std::move(writeIO));
-        numWriteBytes += writeLength;
-      }
-
-      auto rpcStart = hf3fs::SteadyClock::now();
-
-      co_await storageClient_->batchWrite(writeIOs, flat::UserInfo(), options);
-
-      auto elapsedMicro = std::chrono::duration_cast<std::chrono::microseconds>(hf3fs::SteadyClock::now() - rpcStart);
-      elapsedMicroSecs.push_back(elapsedMicro.count());
-
-      if (!benchOptions_.ignoreIOError) {
-        for (const auto &writeIO : writeIOs) {
-          if (writeIO.result.lengthInfo.hasError()) {
-            XLOGF(ERR, "Error in write result: {}", writeIO.result);
-            co_return writeIO.result.lengthInfo.error().code();
-          }
-          if (writeIO.length != *writeIO.result.lengthInfo) {
-            XLOGF(ERR, "Unexpected write length: {} != {}", *writeIO.result.lengthInfo, writeIO.length);
-            co_return StorageClientCode::kRemoteIOError;
-          }
-        }
-      }
-    }
-
-    folly::TDigest digest;
-    writeLatencyDigests_[instanceId] = digest.merge(elapsedMicroSecs);
-    numWriteBytes_ += numWriteBytes;
-
-    co_return StatusCode::kOK;
-  }
-
-  CoTask<uint32_t> batchRead(uint32_t instanceId) {
-    // create an aligned memory block
-    size_t alignedBufSize = ALIGN_UPPER(std::max(size_t(1), benchOptions_.readSize), benchOptions_.memoryAlignment);
-    size_t memoryBlockSize = alignedBufSize * benchOptions_.readBatchSize;
-    auto memoryBlock = (uint8_t *)folly::aligned_malloc(memoryBlockSize, sysconf(_SC_PAGESIZE));
-    auto deleter = [](uint8_t *ptr) { folly::aligned_free(ptr); };
-    std::unique_ptr<uint8_t, decltype(deleter)> memoryBlockPtr(memoryBlock, deleter);
-    std::memset(memoryBlock, 0, memoryBlockSize);
-
-    // register a block of memory
-    auto regRes = storageClient_->registerIOBuffer(memoryBlock, memoryBlockSize);
-
-    if (regRes.hasError()) {
-      co_return regRes.error().code();
-    }
-
-    std::vector<uint8_t> expectedChunkData(setupConfig_.chunk_size());
-
-    if (benchOptions_.verifyReadData) {
-      for (size_t byteIndex = 0; byteIndex < expectedChunkData.size(); byteIndex++) {
-        expectedChunkData[byteIndex] = byteIndex;
-      }
-    }
-
-    // create read IOs
-
-    auto ioBuffer = std::move(*regRes);
-
-    ReadOptions options;
-    options.set_enableChecksum(benchOptions_.verifyReadChecksum);
-    options.debug().set_bypass_disk_io(benchOptions_.benchmarkNetwork);
-    options.debug().set_bypass_rdma_xmit(benchOptions_.benchmarkStorage);
-    options.debug().set_inject_random_server_error(benchOptions_.injectRandomServerError);
-    options.debug().set_inject_random_client_error(benchOptions_.injectRandomClientError);
-    options.retry().set_retry_permanent_error(benchOptions_.retryPermanentError);
-
-    std::vector<double> elapsedMicroSecs;
-    uint64_t numReadBytes = 0;
-    size_t offsetAlignment =
-        benchOptions_.readOffAlignment ? benchOptions_.readOffAlignment : std::max(size_t(1), benchOptions_.readSize);
-
-    std::vector<client::ReadIO> readIOs;
-    readIOs.reserve(benchOptions_.readBatchSize);
-
-    auto benchStart = hf3fs::SteadyClock::now();
-    std::vector<ChunkInfo> &chunkInfos = chunkInfos_[instanceId];
-
-    while (true) {
-      auto accumElapsedSecs = std::chrono::duration_cast<std::chrono::seconds>(hf3fs::SteadyClock::now() - benchStart);
-      if (accumElapsedSecs >= std::chrono::seconds(benchOptions_.numReadSecs)) break;
-
-      readIOs.clear();
-
-      for (size_t readIndex = 0; readIndex < benchOptions_.readBatchSize; readIndex++) {
-        uint64_t randChunkIndex = folly::Random::rand64(0, chunkInfos.size());
-        const auto &[chainId, chunkId, chunkSize] = chunkInfos[randChunkIndex];
-        uint32_t offset = folly::Random::rand32(0, setupConfig_.chunk_size() - benchOptions_.readSize);
-        uint32_t alignedOffset = ALIGN_LOWER(offset, offsetAlignment);
-        auto readIO = storageClient_->createReadIO(chainId,
-                                                   chunkId,
-                                                   alignedOffset /*offset*/,
-                                                   benchOptions_.readSize /*length*/,
-                                                   &memoryBlock[readIndex * alignedBufSize],
-                                                   &ioBuffer);
-        readIOs.push_back(std::move(readIO));
-        numReadBytes += benchOptions_.readSize;
-      }
-
-      auto rpcStart = hf3fs::SteadyClock::now();
-
-      co_await storageClient_->batchRead(readIOs, flat::UserInfo(), options);
-
-      auto elapsedMicro = std::chrono::duration_cast<std::chrono::microseconds>(hf3fs::SteadyClock::now() - rpcStart);
-      elapsedMicroSecs.push_back(elapsedMicro.count());
-
-      if (!benchOptions_.ignoreIOError) {
-        for (const auto &readIO : readIOs) {
-          if (readIO.result.lengthInfo.hasError()) {
-            XLOGF(ERR, "Error in read result: {}", readIO.result);
-            co_return readIO.result.lengthInfo.error().code();
-          }
-          if (readIO.length != *readIO.result.lengthInfo) {
-            XLOGF(ERR, "Unexpected read length: {} != {}", *readIO.result.lengthInfo, readIO.length);
-            co_return StorageClientCode::kRemoteIOError;
-          }
-        }
-      }
-
-      if (benchOptions_.verifyReadData) {
-        for (const auto &readIO : readIOs) {
-          auto diffPos = std::mismatch(&readIO.data[0], &readIO.data[readIO.length], &expectedChunkData[readIO.offset]);
-          uint32_t byteIndex = diffPos.first - &readIO.data[0];
-          if (byteIndex < readIO.length) {
-            XLOGF(ERR,
-                  "Wrong data at bytes index {} and chunk offset {}: data {:#x} != expected {:#x}",
-                  byteIndex,
-                  readIO.offset + byteIndex,
-                  *diffPos.first,
-                  *diffPos.second);
-            co_return StorageClientCode::kFoundBug;
-          }
-        }
-      }
-    }
-
-    folly::TDigest digest;
-    readLatencyDigests_[instanceId] = digest.merge(elapsedMicroSecs);
-    numReadBytes_ += numReadBytes;
-
-    co_return StatusCode::kOK;
-  }
-
-  uint32_t generateChunks() {
-    XLOGF(WARN, "Generating {} test chunks ({:.3f} GiB)...", totalNumChunks_, totalChunkGiB_);
-
-    auto testStart = hf3fs::SteadyClock::now();
-    std::vector<folly::SemiFuture<uint32_t>> writeTasks;
-    numWriteBytes_ = 0;
-
-    size_t writeBatchSize =
-        std::max(benchOptions_.writeBatchSize,
-                 clientConfig_.traffic_control().write().max_concurrent_requests() / benchOptions_.numCoroutines);
-
-    for (size_t instanceId = 0; instanceId < benchOptions_.numCoroutines; instanceId++) {
-      writeTasks.push_back(batchWrite(instanceId, writeBatchSize, setupConfig_.chunk_size(), 0 /*numWriteSecs*/)
-                               .scheduleOn(folly::Executor::getKeepAliveToken(testExecutor_))
-                               .start());
-    }
-
-    auto results = folly::coro::blockingWait(folly::coro::collectAllRange(std::move(writeTasks)));
-
-    for (auto res : results) {
-      if (res != StatusCode::kOK) {
-        XLOGF(WARN, "Test task failed with status code: {}", res);
-        return res;
-      }
-    }
-
-    auto elapsedTime = hf3fs::SteadyClock::now() - testStart;
-    double totalGiB = (double)numWriteBytes_ / 1_GB;
-    printThroughput(elapsedTime, totalGiB);
-
-    auto mergedDigest = folly::TDigest::merge(writeLatencyDigests_);
-    printLatencyDigest(mergedDigest);
-
-    return StatusCode::kOK;
-  }
-
-  uint32_t runWriteBench() {
-    XLOGF(WARN,
-          "Running write benchmark ({} secs, {} chunks, {:.3f} GiB)...",
-          benchOptions_.numWriteSecs,
-          totalNumChunks_,
-          totalChunkGiB_);
-
-    auto testStart = hf3fs::SteadyClock::now();
-    std::vector<folly::SemiFuture<uint32_t>> writeTasks;
-    numWriteBytes_ = 0;
-
-    for (size_t instanceId = 0; instanceId < benchOptions_.numCoroutines; instanceId++) {
-      writeTasks.push_back(
-          batchWrite(instanceId, benchOptions_.writeBatchSize, benchOptions_.writeSize, benchOptions_.numWriteSecs)
-              .scheduleOn(folly::Executor::getKeepAliveToken(testExecutor_))
-              .start());
-    }
-
-    auto results = folly::coro::blockingWait(folly::coro::collectAllRange(std::move(writeTasks)));
-
-    for (auto res : results) {
-      if (res != StatusCode::kOK) {
-        XLOGF(WARN, "Test task failed with status code: {}", res);
-        return res;
-      }
-    }
-
-    auto elapsedTime = hf3fs::SteadyClock::now() - testStart;
-    double totalGiB = (double)numWriteBytes_ / 1_GB;
-    printThroughput(elapsedTime, totalGiB);
-
-    auto mergedDigest = folly::TDigest::merge(writeLatencyDigests_);
-    printLatencyDigest(mergedDigest);
-
-    dumpPerfStats("batch write", mergedDigest, elapsedTime, totalGiB, false /*readIO*/);
-
-    return StatusCode::kOK;
-  }
-
-  uint32_t runReadBench() {
-    XLOGF(WARN, "Running read benchmark ({} secs)...", benchOptions_.numReadSecs);
-
-    auto testStart = hf3fs::SteadyClock::now();
-    std::vector<folly::SemiFuture<uint32_t>> readTasks;
-    numReadBytes_ = 0;
-
-    for (size_t instanceId = 0; instanceId < benchOptions_.numCoroutines; instanceId++) {
-      readTasks.push_back(batchRead(instanceId).scheduleOn(folly::Executor::getKeepAliveToken(testExecutor_)).start());
-    }
-
-    auto results = folly::coro::blockingWait(folly::coro::collectAllRange(std::move(readTasks)));
-
-    for (auto res : results) {
-      if (res != StatusCode::kOK) {
-        XLOGF(WARN, "Test task failed with status code: {}", res);
-        return res;
-      }
-    }
-
-    auto elapsedTime = hf3fs::SteadyClock::now() - testStart;
-    double totalGiB = (double)numReadBytes_ / 1_GB;
-    printThroughput(elapsedTime, totalGiB);
-
-    auto mergedDigest = folly::TDigest::merge(readLatencyDigests_);
-    printLatencyDigest(mergedDigest);
-
-    dumpPerfStats("batch read", mergedDigest, elapsedTime, totalGiB, false /*readIO*/);
-
-    return StatusCode::kOK;
-  }
-
-  uint32_t cleanup() {
-    XLOGF(WARN, "Clean up chunks...");
-
-    std::vector<folly::SemiFuture<uint32_t>> removeTasks;
-
-    for (size_t instanceId = 0; instanceId < benchOptions_.numCoroutines; instanceId++) {
-      auto batchRemove = [this](size_t instanceId) -> folly::coro::Task<uint32_t> {
-        std::vector<client::RemoveChunksOp> removeOps;
-        size_t totalNumChunksRemoved = 0;
-
-        for (const auto &[chainId, chunkId, chunkSize] : chunkInfos_[instanceId]) {
-          removeOps.push_back(storageClient_->createRemoveOp(chainId, chunkId, ChunkId(chunkId, 1)));
-
-          if (removeOps.size() >= benchOptions_.removeBatchSize) {
-            WriteOptions options;
-            options.debug().set_inject_random_server_error(benchOptions_.injectRandomServerError);
-            options.debug().set_inject_random_client_error(benchOptions_.injectRandomClientError);
-            options.retry().set_retry_permanent_error(benchOptions_.retryPermanentError);
-
-            co_await storageClient_->removeChunks(removeOps, flat::UserInfo(), options);
-
-            for (const auto &removeOp : removeOps) {
-              if (removeOp.result.statusCode.hasError()) {
-                XLOGF(WARN, "Remove operation failed with error: {}", removeOp.result.statusCode.error());
-                co_return removeOp.result.statusCode.error().code();
-              }
-
-              XLOGF_IF(DBG5,
-                       removeOp.result.numChunksRemoved != 1,
-                       "{} chunks removed in range {}",
-                       removeOp.result.numChunksRemoved,
-                       removeOp.chunkRange());
-              totalNumChunksRemoved += removeOp.result.numChunksRemoved;
-            }
-
-            removeOps.clear();
-          }
-        }
-
-        XLOGF(WARN, "{} chunks removed by instance #{}", totalNumChunksRemoved, instanceId);
-        co_return StatusCode::kOK;
-      };
-
-      removeTasks.push_back(
-          batchRemove(instanceId).scheduleOn(folly::Executor::getKeepAliveToken(testExecutor_)).start());
-    }
-
-    auto results = folly::coro::blockingWait(folly::coro::collectAllRange(std::move(removeTasks)));
-
-    for (auto res : results) {
-      if (res != StatusCode::kOK) {
-        XLOGF(WARN, "Test task failed with status code: {}", res);
-        return res;
-      }
-    }
-
-    return StatusCode::kOK;
-  };
-
-  uint32_t truncate() {
-    XLOGF(WARN, "Truncate chunks...");
-
-    std::vector<folly::SemiFuture<uint32_t>> truncateTasks;
-
-    for (size_t instanceId = 0; instanceId < benchOptions_.numCoroutines; instanceId++) {
-      auto batchTruncate = [this](size_t instanceId) -> folly::coro::Task<uint32_t> {
-        std::vector<client::TruncateChunkOp> truncateOps;
-
-        for (const auto &[chainId, chunkId, chunkSize] : chunkInfos_[instanceId]) {
-          truncateOps.push_back(storageClient_->createTruncateOp(chainId, chunkId, 0, setupConfig_.chunk_size()));
-
-          if (truncateOps.size() >= benchOptions_.writeBatchSize) {
-            WriteOptions options;
-            options.debug().set_inject_random_server_error(benchOptions_.injectRandomServerError);
-            options.debug().set_inject_random_client_error(benchOptions_.injectRandomClientError);
-            options.retry().set_retry_permanent_error(benchOptions_.retryPermanentError);
-
-            co_await storageClient_->truncateChunks(truncateOps, flat::UserInfo(), options);
-
-            for (const auto &truncateOp : truncateOps) {
-              if (truncateOp.result.lengthInfo.hasError()) {
-                XLOGF(WARN, "Truncate operation failed with error: {}", truncateOp.result.lengthInfo.error());
-                co_return truncateOp.result.lengthInfo.error().code();
-              }
-            }
-
-            truncateOps.clear();
-          }
-        }
-
-        co_return StatusCode::kOK;
-      };
-
-      truncateTasks.push_back(
-          batchTruncate(instanceId).scheduleOn(folly::Executor::getKeepAliveToken(testExecutor_)).start());
-    }
-
-    auto results = folly::coro::blockingWait(folly::coro::collectAllRange(std::move(truncateTasks)));
-
-    for (auto res : results) {
-      if (res != StatusCode::kOK) {
-        XLOGF(WARN, "Test task failed with status code: {}", res);
-        return res;
-      }
-    }
-
-    return StatusCode::kOK;
-  };
-
-  bool run() {
-    if (benchOptions_.numWriteSecs > 0)
-      if (runWriteBench() != StatusCode::kOK) return false;
-    if (benchOptions_.generateTestData)
-      if (generateChunks() != StatusCode::kOK) return false;
-    if (benchOptions_.numReadSecs > 0)
-      if (runReadBench() != StatusCode::kOK) return false;
-    return true;
-  }
-
-  uint64_t getWriteBytes() { return numWriteBytes_; }
-
-  uint64_t getReadBytes() { return numReadBytes_; }
-};
-
-}  // namespace hf3fs::storage::benchmark

asian_ai/chinese/3FS/cmake/AddCrate.cmake

@@ -1,33 +0,0 @@
-if (CMAKE_BUILD_TYPE STREQUAL "Debug")
-    set(CARGO_CMD cargo build)
-    set(TARGET_DIR "debug")
-else ()
-    set(CARGO_CMD cargo build --release)
-    set(TARGET_DIR "release")
-endif ()
-
-add_custom_target(
-    cargo_build_all ALL
-    COMMAND ${CARGO_CMD}
-    WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}"
-)
-
-macro(add_crate NAME)
-    set(LIBRARY "${PROJECT_SOURCE_DIR}/target/${TARGET_DIR}/lib${NAME}.a")
-    set(SOURCES
-        "${PROJECT_SOURCE_DIR}/target/cxxbridge/${NAME}/src/cxx.rs.h"
-        "${PROJECT_SOURCE_DIR}/target/cxxbridge/${NAME}/src/cxx.rs.cc"
-    )
-
-    add_custom_command(
-        OUTPUT ${SOURCES} ${LIBRARY}
-        COMMAND ${CARGO_CMD}
-        WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/${NAME}"
-    )
-
-    add_library(${NAME} STATIC ${SOURCES} ${LIBRARY})
-    target_link_libraries(${NAME} pthread dl ${LIBRARY})
-    target_include_directories(${NAME} PUBLIC "${PROJECT_SOURCE_DIR}/target/cxxbridge")
-    target_compile_options(${NAME} PUBLIC -Wno-dollar-in-identifier-extension)
-    add_dependencies(${NAME} cargo_build_all)
-endmacro()

asian_ai/chinese/3FS/cmake/ApacheArrow.cmake

@@ -1,61 +0,0 @@
-add_library(apache_arrow_static INTERFACE)
-add_library(arrow_static STATIC IMPORTED)
-add_library(parquet_static STATIC IMPORTED)
-add_library(arrow_dependencies STATIC IMPORTED)
-
-set(PREFIX "${CMAKE_CURRENT_BINARY_DIR}")
-set(ARROW_RELEASE_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/src/apache-arrow-cpp/cpp/build/release")
-
-# https://cmake.org/cmake/help/latest/policy/CMP0097.html
-# Starting with CMake 3.16, explicitly setting GIT_SUBMODULES to an empty string
-# means no submodules will be initialized or updated.
-cmake_policy(SET CMP0097 NEW)
-
-if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.28)
-    set(ARROW_BUILD_PARALLEL "")
-else()
-    set(ARROW_BUILD_PARALLEL "-j8")
-endif()
-
-include(ExternalProject)
-ExternalProject_Add(
-    apache-arrow-cpp
-    PREFIX ${PREFIX}
-    GIT_REPOSITORY https://github.com/apache/arrow.git
-    GIT_TAG b7d2f7ffca66c868bd2fce5b3749c6caa002a7f0
-    GIT_SHALLOW ON
-    GIT_PROGRESS ON
-    GIT_SUBMODULES ""
-    SOURCE_SUBDIR "cpp"
-    BUILD_IN_SOURCE ON
-    INSTALL_DIR ${PREFIX}
-    CONFIGURE_COMMAND bash -x -c "\
-    ( cd thirdparty && [[ -f export.sh ]] || ./download_dependencies.sh | tee export.sh ) && \
-    source thirdparty/export.sh && cmake -S . -B . \
-        -DCMAKE_BUILD_TYPE=Release \
-        -DARROW_USE_CCACHE=OFF \
-        -DARROW_USE_SCCACHE=OFF \
-        -DARROW_DEPENDENCY_SOURCE=BUNDLED \
-        -DARROW_BUILD_STATIC=ON \
-        -DARROW_JEMALLOC=ON \
-        -DARROW_SIMD_LEVEL=DEFAULT \
-        -DARROW_BUILD_EXAMPLES=OFF \
-        -DARROW_PARQUET=ON -DARROW_CSV=ON \
-        -DARROW_WITH_ZSTD=ON -DARROW_WITH_LZ4=ON -DARROW_WITH_ZLIB=ON"
-    BUILD_COMMAND bash -x -c "source thirdparty/export.sh && cmake --build . ${ARROW_BUILD_PARALLEL}"
-    BUILD_JOB_SERVER_AWARE 1
-    INSTALL_COMMAND cmake --install . --prefix "${PREFIX}"
-    BUILD_BYPRODUCTS
-        "${ARROW_RELEASE_BUILD_DIR}/libarrow.a"
-        "${ARROW_RELEASE_BUILD_DIR}/libparquet.a"
-        "${ARROW_RELEASE_BUILD_DIR}/libarrow_bundled_dependencies.a"
-  )
-
-add_dependencies(arrow_static apache-arrow-cpp)
-add_dependencies(parquet_static apache-arrow-cpp)
-add_dependencies(arrow_dependencies apache-arrow-cpp)
-set_target_properties(arrow_static PROPERTIES IMPORTED_LOCATION "${ARROW_RELEASE_BUILD_DIR}/libarrow.a")
-set_target_properties(parquet_static PROPERTIES IMPORTED_LOCATION "${ARROW_RELEASE_BUILD_DIR}/libparquet.a")
-set_target_properties(arrow_dependencies PROPERTIES IMPORTED_LOCATION "${ARROW_RELEASE_BUILD_DIR}/libarrow_bundled_dependencies.a")
-target_include_directories(apache_arrow_static SYSTEM INTERFACE "${PREFIX}/include")
-target_link_libraries(apache_arrow_static INTERFACE parquet_static arrow_static arrow_dependencies)

asian_ai/chinese/3FS/cmake/CLangFormat.cmake

@@ -1,24 +0,0 @@
-set(CLANG_FORMAT "/usr/bin/clang-format-14")
-if(EXISTS ${CLANG_FORMAT})
-    message(STATUS "Found clang-format at ${CLANG_FORMAT}")
-
-    set(SOURCE_DIRS
-        ${CMAKE_SOURCE_DIR}/src
-        ${CMAKE_SOURCE_DIR}/tests
-        ${CMAKE_SOURCE_DIR}/benchmarks
-    )
-
-    # For now, it just hard codes the source files list to globs. That works
-    # fine until we have another directory in `src/`. We should ideally gather
-    # this from SOURCE_FILES list. But, should filter the thirs_party sources.
-    # Taking a quick route for now. We should deal with it sometime down the line.
-    add_custom_target(format
-            COMMENT "Running clang-format"
-            COMMAND find ${SOURCE_DIRS} -name '*.cc' -o -name '*.cpp' -o -name '*.h' | grep -v "_generated.h" | xargs ${CLANG_FORMAT} -i)
-
-    add_custom_target(check-format
-            COMMENT "Running clang-format"
-            COMMAND find ${SOURCE_DIRS} -name '*.cc' -o -name '*.cpp' -o -name '*.h' | grep -v "_generated.h" | xargs ${CLANG_FORMAT} --Werror --dry-run)
-else()
-    message(FATAL_ERROR "clang-format-14 not found")
-endif()

asian_ai/chinese/3FS/cmake/CLangTidy.cmake

@@ -1,52 +0,0 @@
-# clang-tidy generates too many warnings, so just disable it by default.
-option(ENABLE_CLANG_TIDY "Run clang-tidy during build" OFF)
-
-find_program(CLANG_TIDY NAMES clang-tidy-14)
-if(CLANG_TIDY)
-    if(CMake_SOURCE_DIR STREQUAL CMake_BINARY_DIR)
-        message(FATAL_ERROR "CMake_RUN_CLANG_TIDY requires an out-of-source build!")
-    endif()
-
-    if(NOT CMAKE_EXPORT_COMPILE_COMMANDS)
-        message(WARNING "CMAKE_EXPORT_COMPILE_COMMANDS=OFF, clang-tidy may not works!!!")
-    endif()
-
-    set(HEADER_FILTER "${CMAKE_SOURCE_DIR}/\\(src\\|tests\\|benchmarks\\|demos\\)")
-
-    if(ENABLE_CLANG_TIDY)
-        set(CMAKE_CXX_CLANG_TIDY ${CLANG_TIDY} --header-filter ${HEADER_FILTER})
-
-        # Create a preprocessor definition that depends on .clang-tidy content so
-        # the compile command will change when .clang-tidy changes.  This ensures
-        # that a subsequent build re-runs clang-tidy on all sources even if they
-        # do not otherwise need to be recompiled.  Nothing actually uses this
-        # definition.  We add it to targets on which we run clang-tidy just to
-        # get the build dependency on the .clang-tidy file.
-        file(SHA1 ${CMAKE_CURRENT_SOURCE_DIR}/.clang-tidy clang_tidy_sha1)
-        set(CLANG_TIDY_DEFINITIONS "CLANG_TIDY_SHA1=${clang_tidy_sha1}")
-        unset(clang_tidy_sha1)
-
-        configure_file(.clang-tidy .clang-tidy COPYONLY)
-    endif()
-
-    set(SOURCE_DIRS
-        ${CMAKE_SOURCE_DIR}/src
-        ${CMAKE_SOURCE_DIR}/tests
-        ${CMAKE_SOURCE_DIR}/demos
-        ${CMAKE_SOURCE_DIR}/benchmarks
-    )
-
-    # For now, it just hard codes the source files list to globs. That works
-    # fine until we have another directory in `src/`. We should ideally gather
-    # this from SOURCE_FILES list. But, should filter the thirs_party sources.
-    # Taking a quick route for now. We should deal with it sometime down the line.
-    add_custom_target(clang-tidy
-        COMMENT "Running clang-tidy"
-        COMMAND run-clang-tidy-14 -header-filter ${HEADER_FILTER} `find ${SOURCE_DIRS} -name "*.cc" -o -name "*.cpp" -not -name "*.actor.cpp" ` -quiet)
-
-    add_custom_target(clang-tidy-fix
-        COMMENT "Running clang-tidy -fix"
-        COMMAND run-clang-tidy-14 -header-filter ${HEADER_FILTER} `find ${SOURCE_DIRS} -name "*.cc" -o -name "*.cpp" -not -name "*.actor.cpp" ` -fix -quiet)
-else()
-    message(WARNING "clang-tidy-14 not found!!!")
-endif()

asian_ai/chinese/3FS/cmake/CTestCustom.cmake

@@ -1,12 +0,0 @@
-file (STRINGS "@CMAKE_BINARY_DIR@/CTestTestfile.cmake" LINES)
-
-# overwrite the file....
-file(WRITE "@CMAKE_BINARY_DIR@/CTestTestfile.cmake" "")
-
-# loop through the lines,
-foreach(LINE IN LISTS LINES)
-  # remove unwanted parts
-  string(REGEX REPLACE ".*third_party.*" "" STRIPPED "${LINE}")
-  # and write the (changed) line ...
-  file(APPEND "@CMAKE_BINARY_DIR@/CTestTestfile.cmake" "${STRIPPED}\n")
-endforeach()

asian_ai/chinese/3FS/cmake/CodeCoverage.cmake

@@ -1,27 +0,0 @@
-option(ENABLE_CODE_COVERAGE "Enable code coverage" OFF)
-if(ENABLE_CODE_COVERAGE)
-    if(CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG" )
-        message(STATUS "Enable code coverage with debug mode.")
-    else()
-        message(WARNING "Code coverage with no debug mode!!!")
-    endif()
-
-    if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-        set(COVERAGE_COMPILER_FLAGS "-g -fprofile-arcs -ftest-coverage")
-        link_libraries(gcov)
-    elseif(CMAKE_CXX_COMPILER_ID MATCHES "[Cc]lang")
-        set(COVERAGE_COMPILER_FLAGS "-g -fprofile-instr-generate -fcoverage-mapping")
-    else()
-        message(FATAL_ERROR "Unknown compiler iid ${CMAKE_CXX_COMPILER_ID}")
-    endif()
-
-    include(CheckCXXCompilerFlag)
-    check_cxx_compiler_flag(-fprofile-abs-path HAVE_fprofile_abs_path)
-    if(HAVE_fprofile_abs_path)
-        set(COVERAGE_COMPILER_FLAGS "${COVERAGE_COMPILER_FLAGS} -fprofile-abs-path")
-    endif()
-
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COVERAGE_COMPILER_FLAGS}")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COVERAGE_COMPILER_FLAGS}")
-    message(STATUS "Appending code coverage compiler flags: ${COVERAGE_COMPILER_FLAGS}")
-endif()

asian_ai/chinese/3FS/cmake/CompileFlags.cmake

@@ -1,21 +0,0 @@
-macro(store_compile_flags)
-set(ORIGINAL_C_FLAGS "${CMAKE_C_FLAGS}")
-set(ORIGINAL_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}")
-set(ORIGINAL_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}")
-set(ORIGINAL_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO}")
-set(ORIGINAL_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
-set(ORIGINAL_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}")
-set(ORIGINAL_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}")
-set(ORIGINAL_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}")
-endmacro()
-
-macro(restore_compile_flags)
-set(CMAKE_C_FLAGS "${ORIGINAL_C_FLAGS}")
-set(CMAKE_C_FLAGS_DEBUG "${ORIGINAL_C_FLAGS_DEBUG}")
-set(CMAKE_C_FLAGS_RELEASE "${ORIGINAL_C_FLAGS_RELEASE}")
-set(CMAKE_C_FLAGS_RELWITHDEBINFO "${ORIGINAL_C_FLAGS_RELWITHDEBINFO}")
-set(CMAKE_CXX_FLAGS "${ORIGINAL_CXX_FLAGS}")
-set(CMAKE_CXX_FLAGS_DEBUG "${ORIGINAL_CXX_FLAGS_DEBUG}")
-set(CMAKE_CXX_FLAGS_RELEASE "${ORIGINAL_CXX_FLAGS_RELEASE}")
-set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${ORIGINAL_CXX_FLAGS_RELWITHDEBINFO}")
-endmacro()

asian_ai/chinese/3FS/cmake/DumpConfig.cmake

@@ -1,15 +0,0 @@
-if (ENABLE_FUSE_APPLICATION)
-add_custom_target(dump-config
-            COMMENT "Running dump config"
-            COMMAND find ${CMAKE_CURRENT_BINARY_DIR} -name 'meta_main' -o -name 'mgmtd_main' -o -name 'storage_main' -o -name 'admin_cli' -o -name 'hf3fs_fuse_main' | xargs -I {} bash -c '{} --dump_default_cfg > ${CMAKE_SOURCE_DIR}/configs/`basename {}`.toml'
-            COMMAND find ${CMAKE_CURRENT_BINARY_DIR} -name 'meta_main' -o -name 'mgmtd_main' -o -name 'storage_main' -o -name 'hf3fs_fuse_main' | xargs -I {} bash -c '{} --dump_default_app_cfg > ${CMAKE_SOURCE_DIR}/configs/`basename {}`_app.toml'
-            COMMAND find ${CMAKE_CURRENT_BINARY_DIR} -name 'meta_main' -o -name 'mgmtd_main' -o -name 'storage_main' -o -name 'hf3fs_fuse_main' | xargs -I {} bash -c '{} --dump_default_launcher_cfg > ${CMAKE_SOURCE_DIR}/configs/`basename {}`_launcher.toml')
-add_dependencies(dump-config meta_main mgmtd_main storage_main admin_cli hf3fs_fuse_main)
-else()
-add_custom_target(dump-config
-            COMMENT "Running dump config"
-            COMMAND find ${CMAKE_CURRENT_BINARY_DIR} -name 'meta_main' -o -name 'mgmtd_main' -o -name 'storage_main' -o -name 'admin_cli' | xargs -I {} bash -c '{} --dump_default_cfg > ${CMAKE_SOURCE_DIR}/configs/`basename {}`.toml'
-            COMMAND find ${CMAKE_CURRENT_BINARY_DIR} -name 'meta_main' -o -name 'mgmtd_main' -o -name 'storage_main' | xargs -I {} bash -c '{} --dump_default_app_cfg > ${CMAKE_SOURCE_DIR}/configs/`basename {}`_app.toml'
-            COMMAND find ${CMAKE_CURRENT_BINARY_DIR} -name 'meta_main' -o -name 'mgmtd_main' -o -name 'storage_main' | xargs -I {} bash -c '{} --dump_default_launcher_cfg > ${CMAKE_SOURCE_DIR}/configs/`basename {}`_launcher.toml')
-add_dependencies(dump-config meta_main mgmtd_main storage_main admin_cli)
-endif()

asian_ai/chinese/3FS/cmake/GitVersion.cmake

@@ -1,127 +0,0 @@
-find_package(Git REQUIRED)
-
-if (NOT DEFINED PRE_CONFIGURE_DIR)
-    set(PRE_CONFIGURE_DIR ${PROJECT_SOURCE_DIR}/src/common/utils)
-endif ()
-
-if (NOT DEFINED POST_BUILD_DIR)
-    set(POST_BUILD_DIR ${PROJECT_BINARY_DIR})
-endif ()
-
-set(PRE_CONFIGURE_FILE ${PRE_CONFIGURE_DIR}/VersionInfo.cc.in)
-set(POST_CONFIGURE_FILE ${POST_BUILD_DIR}/src/common/utils/VersionInfo.cc)
-
-function(CheckGitWrite git_hash)
-    file(WRITE ${POST_BUILD_DIR}/git-state.txt ${git_hash})
-endfunction()
-
-function(CheckGitRead git_hash)
-    if (EXISTS ${POST_BUILD_DIR}/git-state.txt)
-        file(STRINGS ${POST_BUILD_DIR}/git-state.txt CONTENT)
-        LIST(GET CONTENT 0 var)
-        set(${git_hash} ${var} PARENT_SCOPE)
-    endif ()
-endfunction()
-
-function(CheckGitVersion)
-    # Get the latest abbreviated commit hash of the working branch
-    execute_process(
-            COMMAND ${GIT_EXECUTABLE} rev-parse --short=8 HEAD
-            WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
-            OUTPUT_VARIABLE BUILD_COMMIT_HASH_SHORT
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-    )
-    execute_process(
-            COMMAND ${GIT_EXECUTABLE} rev-parse HEAD
-            WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
-            OUTPUT_VARIABLE BUILD_COMMIT_HASH_FULL
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-    )
-    execute_process(
-            COMMAND ${GIT_EXECUTABLE} log -1 --format=%at --date=local
-            WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
-            OUTPUT_VARIABLE BUILD_TIMESTAMP
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-    )
-    execute_process(
-            COMMAND date -d @${BUILD_TIMESTAMP} +%Y%m%d
-            WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
-            OUTPUT_VARIABLE BUILD_DATE
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-    )
-    execute_process(
-            COMMAND ${GIT_EXECUTABLE} describe --tags --abbrev=0
-            WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
-            OUTPUT_VARIABLE BUILD_TAG
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-    )
-    execute_process(
-            COMMAND bash -c "${GIT_EXECUTABLE} describe --tags --long | sed -E 's/(.*)-([0-9]+)-(\\w+)/\\2/g'"
-            WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
-            OUTPUT_VARIABLE BUILD_TAG_SEQ_NUM
-            OUTPUT_STRIP_TRAILING_WHITESPACE
-    )
-    if(NOT DEFINED BUILD_TAG OR BUILD_TAG STREQUAL "")
-        set(BUILD_TAG "250228")
-    endif()
-    if(NOT DEFINED BUILD_TAG_SEQ_NUM OR BUILD_TAG_SEQ_NUM STREQUAL "")
-        set(BUILD_TAG_SEQ_NUM "1")
-    endif()
-    message(STATUS "Git Commit hash: ${BUILD_COMMIT_HASH_SHORT} ${BUILD_COMMIT_HASH_FULL}")
-    message(STATUS "Git Commit Date & Timestamp: ${BUILD_DATE} ${BUILD_TIMESTAMP}")
-    message(STATUS "Git Commit Tag & Seq Num: ${BUILD_TAG} ${BUILD_TAG_SEQ_NUM}")
-
-    set(BUILD_VERSION_MAJOR "${PROJECT_VERSION_MAJOR}")
-    set(BUILD_VERSION_MINOR "${PROJECT_VERSION_MINOR}")
-    set(BUILD_VERSION_PATCH "${PROJECT_VERSION_PATCH}")
-    set(BUILD_VERSION "${PROJECT_VERSION}")
-
-    CheckGitRead(GIT_HASH_CACHE)
-
-    if (NOT DEFINED GIT_HASH_CACHE)
-        set(GIT_HASH_CACHE "INVALID")
-    endif ()
-
-    if (NOT DEFINED BUILD_ON_RELEASE_BRANCH)
-      set(BUILD_ON_RELEASE_BRANCH "false")
-    endif ()
-
-    if (NOT DEFINED BUILD_PIPELINE_ID)
-        set(BUILD_PIPELINE_ID "999999")
-    endif()
-
-    # Only update the git_version.cpp if the hash has changed. This will
-    # prevent us from rebuilding the project more than we need to.
-    if (NOT ${BUILD_COMMIT_HASH_FULL} STREQUAL ${GIT_HASH_CACHE} OR NOT EXISTS ${POST_CONFIGURE_FILE})
-        # Set che GIT_HASH_CACHE variable the next build won't have
-        # to regenerate the source file.
-        CheckGitWrite("${BUILD_COMMIT_HASH_FULL}")
-        configure_file(${PRE_CONFIGURE_FILE} ${POST_CONFIGURE_FILE} @ONLY)
-    endif ()
-endfunction()
-
-function(CheckGitSetup project_src_dir)
-    add_custom_target(AlwaysCheckGit COMMAND ${CMAKE_COMMAND}
-        -DRUN_CHECK_GIT_VERSION=1
-        -DPRE_CONFIGURE_DIR=${PRE_CONFIGURE_DIR}
-        -DPOST_BUILD_DIR=${POST_BUILD_DIR}
-        -DGIT_HASH_CACHE=${GIT_HASH_CACHE}
-        -DPROJECT_VERSION_MAJOR=${PROJECT_VERSION_MAJOR}
-        -DPROJECT_VERSION_MINOR=${PROJECT_VERSION_MINOR}
-        -DPROJECT_VERSION_PATCH=${PROJECT_VERSION_PATCH}
-        -DPROJECT_VERSION=${PROJECT_VERSION}
-        -DPROJECT_SOURCE_DIR=${project_src_dir}
-        -P ${PROJECT_SOURCE_DIR}/cmake/GitVersion.cmake
-        DEPENDS ${PRE_CONFIGURE_FILE}
-        BYPRODUCTS ${POST_CONFIGURE_FILE}
-    )
-
-    add_library(version-info STATIC ${POST_CONFIGURE_FILE})
-    target_include_directories(version-info PUBLIC ${PROJECT_SOURCE_DIR}/src)
-    add_dependencies(version-info AlwaysCheckGit)
-    CheckGitVersion()
-endfunction()
-
-if (RUN_CHECK_GIT_VERSION)
-    CheckGitVersion()
-endif ()

asian_ai/chinese/3FS/cmake/Jemalloc.cmake

@@ -1,20 +0,0 @@
-add_library(jemalloc INTERFACE)
-add_library(hf3fs_jemalloc_shared SHARED IMPORTED)
-
-include(ExternalProject)
-set(JEMALLOC_DIR "${CMAKE_BINARY_DIR}/third_party/jemalloc")
-
-ExternalProject_add(Hf3fsJemalloc_project
-  SOURCE_DIR "${PROJECT_SOURCE_DIR}/third_party/jemalloc"
-  BUILD_BYPRODUCTS "${JEMALLOC_DIR}/include/jemalloc/jemalloc.h"
-  "${JEMALLOC_DIR}/lib/libjemalloc.so.2"
-  CONFIGURE_COMMAND ./autogen.sh && ./configure --prefix=${JEMALLOC_DIR} --disable-cxx --enable-prof --disable-initial-exec-tls
-  BUILD_IN_SOURCE ON
-  BUILD_COMMAND make -j 6
-  INSTALL_DIR "${JEMALLOC_DIR}"
-  INSTALL_COMMAND make install)
-
-add_dependencies(hf3fs_jemalloc_shared Hf3fsJemalloc_project)
-set_target_properties(hf3fs_jemalloc_shared PROPERTIES IMPORTED_LOCATION "${JEMALLOC_DIR}/lib/libjemalloc.so.2")
-target_include_directories(hf3fs_jemalloc_shared INTERFACE "${JEMALLOC_DIR}/include")
-target_link_libraries(jemalloc INTERFACE hf3fs_jemalloc_shared)

asian_ai/chinese/3FS/cmake/Sanitizers.cmake

@@ -1,242 +0,0 @@
-#
-# Copyright (C) 2018-2022 by George Cave - gcave@stablecoder.ca
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may not
-# use this file except in compliance with the License. You may obtain a copy of
-# the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations under
-# the License.
-
-include(CheckCXXSourceCompiles)
-
-set(SANITIZER "" CACHE STRING "Sanitizer: ASAN, MSAN, UBSAN, TSAN")
-if(SANITIZER STREQUAL "ASAN")
-    message(STATUS "Build with Address Sanitizer.")
-    set(USE_SANITIZER "Address")
-    set(USE_ASAN ON)
-elseif(SANITIZER STREQUAL "MSAN")
-    message(STATUS "Build with Memory Sanitizer.")
-    set(USE_SANITIZER "Memory")
-    set(USE_MSAN ON)
-elseif(SANITIZER STREQUAL "UBSAN")
-    message(STATUS "Build with Undefined behavior Sanitizer.")
-    message(WARNING "With clang++ v14.0.6 undefined behavior sanitizer doesn't work with coroutine!!!")
-    set(USE_SANITIZER "Undefined")
-    set(USE_UBSAN ON)
-elseif(SANITIZER STREQUAL "TSAN")
-    message(STATUS "Build with Thread Sanitizer.")
-    set(USE_SANITIZER "Thread")
-    set(USE_TSAN ON)
-else()
-    message(STATUS "Sanitizer not enabled. ${SANITIZER}")
-    set(USE_SANITIZER "")
-endif()
-
-# set(USE_SANITIZER "" CACHE STRING
-#     "Compile with a sanitizer. Options are: Address, Memory, MemoryWithOrigins, Undefined, Thread, Leak, 'Address;Undefined', CFI"
-# )
-
-function(append value)
-    foreach(variable ${ARGN})
-        set(${variable}
-                "${${variable}} ${value}"
-                PARENT_SCOPE)
-    endforeach(variable)
-endfunction()
-
-function(append_quoteless value)
-    foreach(variable ${ARGN})
-        set(${variable}
-                ${${variable}} ${value}
-                PARENT_SCOPE)
-    endforeach(variable)
-endfunction()
-
-function(test_san_flags return_var flags)
-    set(QUIET_BACKUP ${CMAKE_REQUIRED_QUIET})
-    set(CMAKE_REQUIRED_QUIET TRUE)
-    unset(${return_var} CACHE)
-    set(FLAGS_BACKUP ${CMAKE_REQUIRED_FLAGS})
-    set(CMAKE_REQUIRED_FLAGS "${flags}")
-    check_cxx_source_compiles("int main() { return 0; }" ${return_var})
-    set(CMAKE_REQUIRED_FLAGS "${FLAGS_BACKUP}")
-    set(CMAKE_REQUIRED_QUIET "${QUIET_BACKUP}")
-endfunction()
-
-if(USE_SANITIZER)
-    append("-fno-omit-frame-pointer" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
-
-    unset(SANITIZER_SELECTED_FLAGS)
-
-    if(UNIX)
-        if(uppercase_CMAKE_BUILD_TYPE STREQUAL "DEBUG")
-            append("-O1" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
-        endif()
-
-        if(USE_SANITIZER MATCHES "([Aa]ddress)")
-            # Optional: -fno-optimize-sibling-calls -fsanitize-address-use-after-scope
-            message(STATUS "Testing with Address sanitizer")
-            set(SANITIZER_ADDR_FLAG "-fsanitize=address")
-            test_san_flags(SANITIZER_ADDR_AVAILABLE ${SANITIZER_ADDR_FLAG})
-            if(SANITIZER_ADDR_AVAILABLE)
-                message(STATUS "  Building with Address sanitizer")
-                append("${SANITIZER_ADDR_FLAG}" SANITIZER_SELECTED_FLAGS)
-
-                if(AFL)
-                    append_quoteless(AFL_USE_ASAN=1 CMAKE_C_COMPILER_LAUNCHER
-                                                     CMAKE_CXX_COMPILER_LAUNCHER)
-                endif()
-            else()
-                message(
-                    FATAL_ERROR
-                        "Address sanitizer not available for ${CMAKE_CXX_COMPILER}")
-            endif()
-        endif()
-
-        if(USE_SANITIZER MATCHES "([Mm]emory([Ww]ith[Oo]rigins)?)")
-            # Optional: -fno-optimize-sibling-calls -fsanitize-memory-track-origins=2
-            set(SANITIZER_MEM_FLAG "-fsanitize=memory")
-            if(USE_SANITIZER MATCHES "([Mm]emory[Ww]ith[Oo]rigins)")
-                message(STATUS "Testing with MemoryWithOrigins sanitizer")
-                append("-fsanitize-memory-track-origins" SANITIZER_MEM_FLAG)
-            else()
-                message(STATUS "Testing with Memory sanitizer")
-            endif()
-            test_san_flags(SANITIZER_MEM_AVAILABLE ${SANITIZER_MEM_FLAG})
-            if(SANITIZER_MEM_AVAILABLE)
-                if(USE_SANITIZER MATCHES "([Mm]emory[Ww]ith[Oo]rigins)")
-                    message(STATUS "  Building with MemoryWithOrigins sanitizer")
-                else()
-                    message(STATUS "  Building with Memory sanitizer")
-                endif()
-                append("${SANITIZER_MEM_FLAG}" SANITIZER_SELECTED_FLAGS)
-
-                if(AFL)
-                    append_quoteless(AFL_USE_MSAN=1 CMAKE_C_COMPILER_LAUNCHER
-                                                     CMAKE_CXX_COMPILER_LAUNCHER)
-                endif()
-            else()
-                message(
-                    FATAL_ERROR
-                        "Memory [With Origins] sanitizer not available for ${CMAKE_CXX_COMPILER}"
-                )
-            endif()
-        endif()
-
-        if(USE_SANITIZER MATCHES "([Uu]ndefined)")
-            message(STATUS "Testing with Undefined Behaviour sanitizer")
-            set(SANITIZER_UB_FLAG "-fsanitize=undefined")
-            if(EXISTS "${BLACKLIST_FILE}")
-                append("-fsanitize-blacklist=${BLACKLIST_FILE}" SANITIZER_UB_FLAG)
-            endif()
-            test_san_flags(SANITIZER_UB_AVAILABLE ${SANITIZER_UB_FLAG})
-            if(SANITIZER_UB_AVAILABLE)
-                message(STATUS "  Building with Undefined Behaviour sanitizer")
-                append("${SANITIZER_UB_FLAG}" SANITIZER_SELECTED_FLAGS)
-
-                if(AFL)
-                    append_quoteless(AFL_USE_UBSAN=1 CMAKE_C_COMPILER_LAUNCHER
-                                                     CMAKE_CXX_COMPILER_LAUNCHER)
-                endif()
-            else()
-                message(
-                    FATAL_ERROR
-                        "Undefined Behaviour sanitizer not available for ${CMAKE_CXX_COMPILER}"
-                )
-            endif()
-        endif()
-
-        if(USE_SANITIZER MATCHES "([Tt]hread)")
-            message(STATUS "Testing with Thread sanitizer")
-            set(SANITIZER_THREAD_FLAG "-fsanitize=thread -fsanitize-ignorelist=${CMAKE_SOURCE_DIR}/tsan_ignore.txt")
-            test_san_flags(SANITIZER_THREAD_AVAILABLE ${SANITIZER_THREAD_FLAG})
-            if(SANITIZER_THREAD_AVAILABLE)
-                message(STATUS "  Building with Thread sanitizer")
-                append("${SANITIZER_THREAD_FLAG}" SANITIZER_SELECTED_FLAGS)
-
-                if(AFL)
-                    append_quoteless(AFL_USE_TSAN=1 CMAKE_C_COMPILER_LAUNCHER
-                                                     CMAKE_CXX_COMPILER_LAUNCHER)
-                endif()
-            else()
-                message(
-                    FATAL_ERROR "Thread sanitizer not available for ${CMAKE_CXX_COMPILER}"
-                )
-            endif()
-        endif()
-
-        if(USE_SANITIZER MATCHES "([Ll]eak)")
-            message(STATUS "Testing with Leak sanitizer")
-            set(SANITIZER_LEAK_FLAG "-fsanitize=leak")
-            test_san_flags(SANITIZER_LEAK_AVAILABLE ${SANITIZER_LEAK_FLAG})
-            if(SANITIZER_LEAK_AVAILABLE)
-                message(STATUS "  Building with Leak sanitizer")
-                append("${SANITIZER_LEAK_FLAG}" SANITIZER_SELECTED_FLAGS)
-
-                if(AFL)
-                    append_quoteless(AFL_USE_LSAN=1 CMAKE_C_COMPILER_LAUNCHER
-                                                     CMAKE_CXX_COMPILER_LAUNCHER)
-                endif()
-            else()
-                message(
-                    FATAL_ERROR "Thread sanitizer not available for ${CMAKE_CXX_COMPILER}"
-                )
-            endif()
-        endif()
-
-        if(USE_SANITIZER MATCHES "([Cc][Ff][Ii])")
-            message(STATUS "Testing with Control Flow Integrity(CFI) sanitizer")
-            set(SANITIZER_CFI_FLAG "-fsanitize=cfi")
-            test_san_flags(SANITIZER_CFI_AVAILABLE ${SANITIZER_CFI_FLAG})
-            if(SANITIZER_CFI_AVAILABLE)
-                message(STATUS "  Building with Control Flow Integrity(CFI) sanitizer")
-                append("${SANITIZER_LEAK_FLAG}" SANITIZER_SELECTED_FLAGS)
-
-                if(AFL)
-                    append_quoteless(AFL_USE_CFISAN=1 CMAKE_C_COMPILER_LAUNCHER
-                                                     CMAKE_CXX_COMPILER_LAUNCHER)
-                endif()
-            else()
-                message(
-                    FATAL_ERROR
-                        "Control Flow Integrity(CFI) sanitizer not available for ${CMAKE_CXX_COMPILER}"
-                )
-            endif()
-        endif()
-
-        message(STATUS "Sanitizer flags: ${SANITIZER_SELECTED_FLAGS}")
-        test_san_flags(SANITIZER_SELECTED_COMPATIBLE ${SANITIZER_SELECTED_FLAGS})
-        if(SANITIZER_SELECTED_COMPATIBLE)
-            message(STATUS " Building with ${SANITIZER_SELECTED_FLAGS}")
-            append("${SANITIZER_SELECTED_FLAGS}" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
-        else()
-            message(
-                FATAL_ERROR
-                    " Sanitizer flags ${SANITIZER_SELECTED_FLAGS} are not compatible.")
-        endif()
-    elseif(MSVC)
-        if(USE_SANITIZER MATCHES "([Aa]ddress)")
-            message(STATUS "Building with Address sanitizer")
-            append("-fsanitize=address" CMAKE_C_FLAGS CMAKE_CXX_FLAGS)
-
-            if(AFL)
-                append_quoteless(AFL_USE_ASAN=1 CMAKE_C_COMPILER_LAUNCHER
-                                                 CMAKE_CXX_COMPILER_LAUNCHER)
-            endif()
-        else()
-            message(
-                FATAL_ERROR
-                    "This sanitizer not yet supported in the MSVC environment: ${USE_SANITIZER}"
-            )
-        endif()
-    else()
-        message(FATAL_ERROR "USE_SANITIZER is not supported on this platform.")
-    endif()
-
-endif()

asian_ai/chinese/3FS/cmake/Target.cmake

@@ -1,109 +0,0 @@
-# Check if IPO is supported
-include(CheckIPOSupported)
-check_ipo_supported(RESULT HAVE_IPO)
-
-# Enable IPO in non-debug build
-macro(target_enable_ipo NAME)
-    if(NOT CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG" AND HAVE_IPO)
-        set_property(TARGET ${NAME} PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE)
-        message (STATUS "Enabled IPO for target: ${NAME}")
-    endif()
-endmacro()
-
-macro(target_add_lib NAME)
-    file(GLOB_RECURSE FILES CONFIGURE_DEPENDS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc" "*.h")
-    add_library(${NAME} STATIC ${FILES} ${FBS_FILES})
-    target_include_directories(${NAME}
-        PUBLIC
-            $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/src>
-            ${PROJECT_SOURCE_DIR}
-            ${PROJECT_BINARY_DIR}/src
-            ${PROJECT_BINARY_DIR}
-    )
-    target_link_libraries(${NAME} ${ARGN} "")
-endmacro()
-
-macro(target_add_shared_lib NAME)
-    file(GLOB_RECURSE FILES CONFIGURE_DEPENDS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc" "*.h")
-    add_library(${NAME} SHARED ${FILES} ${FBS_FILES})
-    target_include_directories(${NAME}
-        PUBLIC
-            $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/src>
-            ${PROJECT_SOURCE_DIR}
-            ${PROJECT_BINARY_DIR}/src
-            ${PROJECT_BINARY_DIR}
-    )
-    target_link_libraries(${NAME} ${ARGN} "")
-    target_enable_ipo(${NAME})
-endmacro()
-
-macro(target_add_bin NAME MAIN_FILE)
-  add_executable(${NAME} ${MAIN_FILE})
-    target_link_libraries(${NAME} ${ARGN} "")
-    target_include_directories(${NAME}
-        PUBLIC
-            $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/src>
-            ${PROJECT_SOURCE_DIR}
-            ${PROJECT_SOURCE_DIR}/src/lib/api
-            ${PROJECT_BINARY_DIR}/src
-            ${PROJECT_BINARY_DIR}
-    )
-    set_target_properties(${NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin")
-    target_enable_ipo(${NAME})
-endmacro()
-
-macro(target_add_test NAME)
-    file(GLOB_RECURSE FILES CONFIGURE_DEPENDS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
-    add_executable(${NAME} ${FILES})
-    target_link_libraries(${NAME} gmock test_main ${ARGN} "")
-    target_include_directories(${NAME}
-        PUBLIC
-            $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/src>
-            ${PROJECT_SOURCE_DIR}
-            ${PROJECT_BINARY_DIR}/src
-            ${PROJECT_BINARY_DIR}
-    )
-    add_test(NAME ${NAME} COMMAND ${NAME})
-    set_target_properties(${NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tests")
-endmacro()
-
-macro(target_add_fbs NAME PATH)
-    cmake_parse_arguments(FBS "SERVICE" "" "DEPS" ${ARGN})
-
-    message("Fbs " ${NAME} " FBS_SERVICE " ${FBS_SERVICE} " ARGN " ${ARGN})
-  
-    set(FLATBUFFERS_FLATC_SCHEMA_EXTRA_ARGS
-        --scoped-enums
-        --gen-object-api
-        --gen-mutable
-        --gen-compare
-        --cpp-std=c++17
-        --python
-        --hf3fs
-        --keep-prefix
-    )
-
-    get_filename_component(NAME_WE ${PATH} NAME_WE)
-    get_filename_component(DIR ${PATH} DIRECTORY)
-
-    build_flatbuffers(${PATH}
-        "${CMAKE_CURRENT_SOURCE_DIR}/${DIR};${CMAKE_SOURCE_DIR}/src"
-        "${NAME_WE}-generated"
-        ""
-        "${CMAKE_CURRENT_BINARY_DIR}/${DIR}"
-        ""
-        ""
-        )
-
-      add_library(${NAME} INTERFACE)
-      target_link_libraries(${NAME} INTERFACE common)
-      target_include_directories(${NAME}
-          INTERFACE
-              $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/src>
-              ${PROJECT_SOURCE_DIR}
-              ${PROJECT_BINARY_DIR}/src
-              ${PROJECT_BINARY_DIR}
-      )
-
-    add_dependencies(${NAME} "${NAME_WE}-generated" ${FBS_DEPS})
-endmacro()

asian_ai/chinese/3FS/configs/admin_cli.toml

@@ -1,338 +0,0 @@
-break_multi_line_command_on_failure = false
-cluster_id = ''
-log = 'DBG:normal; normal=file:path=/var/log/3fs/cli.log,async=true,sync_level=ERR'
-num_timeout_ms = 1000
-profile = false
-verbose = false
-
-[client]
-default_compression_level = 0
-default_compression_threshold = '128KB'
-default_log_long_running_threshold = '0ns'
-default_report_metrics = false
-default_send_retry_times = 1
-default_timeout = '1s'
-enable_rdma_control = false
-force_use_tcp = false
-
-[client.io_worker]
-num_event_loop = 1
-rdma_connect_timeout = '5s'
-read_write_rdma_in_event_thread = false
-read_write_tcp_in_event_thread = false
-tcp_connect_timeout = '1s'
-wait_to_retry_send = '100ms'
-
-[client.io_worker.connect_concurrency_limiter]
-max_concurrency = 4
-
-[client.io_worker.ibsocket]
-buf_ack_batch = 8
-buf_signal_batch = 8
-buf_size = 16384
-drain_timeout = '5s'
-drop_connections = 0
-event_ack_batch = 128
-max_rd_atomic = 16
-max_rdma_wr = 128
-max_rdma_wr_per_post = 32
-max_sge = 16
-min_rnr_timer = 1
-record_bytes_per_peer = false
-record_latency_per_peer = false
-retry_cnt = 7
-rnr_retry = 0
-send_buf_cnt = 32
-sl = 0
-start_psn = 0
-timeout = 14
-
-[client.io_worker.transport_pool]
-max_connections = 1
-
-[client.processor]
-enable_coroutines_pool = true
-max_coroutines_num = 256
-max_processing_requests_num = 4096
-response_compression_level = 1
-response_compression_threshold = '128KB'
-
-[client.rdma_control]
-max_concurrent_transmission = 64
-
-[client.thread_pool]
-bg_thread_pool_stratetry = 'SHARED_QUEUE'
-collect_stats = false
-enable_work_stealing = false
-io_thread_pool_stratetry = 'SHARED_QUEUE'
-num_bg_threads = 2
-num_connect_threads = 2
-num_io_threads = 2
-num_proc_threads = 2
-proc_thread_pool_stratetry = 'SHARED_QUEUE'
-
-[fdb]
-casual_read_risky = false
-clusterFile = ''
-default_backoff = 0
-enableMultipleClient = false
-externalClientDir = ''
-externalClientPath = ''
-multipleClientThreadNum = 4
-readonly = false
-trace_file = ''
-trace_format = 'json'
-
-[ib_devices]
-allow_no_usable_devices = false
-allow_unknown_zone = true
-default_network_zone = 'UNKNOWN'
-default_pkey_index = 0
-default_roce_pkey_index = 0
-default_traffic_class = 0
-device_filter = []
-fork_safe = true
-prefer_ibdevice = true
-skip_inactive_ports = true
-skip_unusable_device = true
-subnets = []
-
-[meta_client]
-check_server_interval = '5s'
-dynamic_stripe = false
-max_concurrent_requests = 128
-network_type = 'RDMA'
-remove_chunks_batch_size = 32
-remove_chunks_max_iters = 1024
-selection_mode = 'RandomFollow'
-
-[meta_client.background_closer]
-prune_session_batch_count = 128
-prune_session_batch_interval = '10s'
-retry_first_wait = '100ms'
-retry_max_wait = '10s'
-task_scan = '50ms'
-
-[meta_client.background_closer.coroutine_pool]
-coroutines_num = 8
-enable_work_stealing = false
-queue_size = 128
-
-[meta_client.retry_default]
-max_failures_before_failover = 1
-retry_fast = '1s'
-retry_init_wait = '500ms'
-retry_max_wait = '5s'
-retry_send = 1
-retry_total_time = '1min'
-rpc_timeout = '5s'
-
-[mgmtd_client]
-accept_incomplete_routing_info_during_mgmtd_bootstrapping = true
-auto_extend_client_session_interval = '10s'
-auto_heartbeat_interval = '10s'
-auto_refresh_interval = '1s'
-enable_auto_extend_client_session = false
-enable_auto_heartbeat = false
-enable_auto_refresh = true
-mgmtd_server_addresses = []
-work_queue_size = 100
-
-[monitor]
-collect_period = '1s'
-num_collectors = 1
-reporters = []
-
-[storage_client]
-check_overlapping_read_buffers = true
-check_overlapping_write_buffers = false
-chunk_checksum_type = 'CRC32C'
-create_net_client_for_updates = false
-implementation_type = 'RPC'
-max_inline_read_bytes = '0'
-max_inline_write_bytes = '0'
-max_read_io_bytes = '0'
-
-[storage_client.net_client]
-default_compression_level = 0
-default_compression_threshold = '128KB'
-default_log_long_running_threshold = '0ns'
-default_report_metrics = false
-default_send_retry_times = 1
-default_timeout = '1s'
-enable_rdma_control = false
-force_use_tcp = false
-
-[storage_client.net_client.io_worker]
-num_event_loop = 1
-rdma_connect_timeout = '5s'
-read_write_rdma_in_event_thread = false
-read_write_tcp_in_event_thread = false
-tcp_connect_timeout = '1s'
-wait_to_retry_send = '100ms'
-
-[storage_client.net_client.io_worker.connect_concurrency_limiter]
-max_concurrency = 4
-
-[storage_client.net_client.io_worker.ibsocket]
-buf_ack_batch = 8
-buf_signal_batch = 8
-buf_size = 16384
-drain_timeout = '5s'
-drop_connections = 0
-event_ack_batch = 128
-max_rd_atomic = 16
-max_rdma_wr = 128
-max_rdma_wr_per_post = 32
-max_sge = 16
-min_rnr_timer = 1
-record_bytes_per_peer = false
-record_latency_per_peer = false
-retry_cnt = 7
-rnr_retry = 0
-send_buf_cnt = 32
-sl = 0
-start_psn = 0
-timeout = 14
-
-[storage_client.net_client.io_worker.transport_pool]
-max_connections = 1
-
-[storage_client.net_client.processor]
-enable_coroutines_pool = true
-max_coroutines_num = 256
-max_processing_requests_num = 4096
-response_compression_level = 1
-response_compression_threshold = '128KB'
-
-[storage_client.net_client.rdma_control]
-max_concurrent_transmission = 64
-
-[storage_client.net_client.thread_pool]
-bg_thread_pool_stratetry = 'SHARED_QUEUE'
-collect_stats = false
-enable_work_stealing = false
-io_thread_pool_stratetry = 'SHARED_QUEUE'
-num_bg_threads = 2
-num_connect_threads = 2
-num_io_threads = 2
-num_proc_threads = 2
-proc_thread_pool_stratetry = 'SHARED_QUEUE'
-
-[storage_client.net_client_for_updates]
-default_compression_level = 0
-default_compression_threshold = '128KB'
-default_log_long_running_threshold = '0ns'
-default_report_metrics = false
-default_send_retry_times = 1
-default_timeout = '1s'
-enable_rdma_control = false
-force_use_tcp = false
-
-[storage_client.net_client_for_updates.io_worker]
-num_event_loop = 1
-rdma_connect_timeout = '5s'
-read_write_rdma_in_event_thread = false
-read_write_tcp_in_event_thread = false
-tcp_connect_timeout = '1s'
-wait_to_retry_send = '100ms'
-
-[storage_client.net_client_for_updates.io_worker.connect_concurrency_limiter]
-max_concurrency = 4
-
-[storage_client.net_client_for_updates.io_worker.ibsocket]
-buf_ack_batch = 8
-buf_signal_batch = 8
-buf_size = 16384
-drain_timeout = '5s'
-drop_connections = 0
-event_ack_batch = 128
-max_rd_atomic = 16
-max_rdma_wr = 128
-max_rdma_wr_per_post = 32
-max_sge = 16
-min_rnr_timer = 1
-record_bytes_per_peer = false
-record_latency_per_peer = false
-retry_cnt = 7
-rnr_retry = 0
-send_buf_cnt = 32
-sl = 0
-start_psn = 0
-timeout = 14
-
-[storage_client.net_client_for_updates.io_worker.transport_pool]
-max_connections = 1
-
-[storage_client.net_client_for_updates.processor]
-enable_coroutines_pool = true
-max_coroutines_num = 256
-max_processing_requests_num = 4096
-response_compression_level = 1
-response_compression_threshold = '128KB'
-
-[storage_client.net_client_for_updates.rdma_control]
-max_concurrent_transmission = 64
-
-[storage_client.net_client_for_updates.thread_pool]
-bg_thread_pool_stratetry = 'SHARED_QUEUE'
-collect_stats = false
-enable_work_stealing = false
-io_thread_pool_stratetry = 'SHARED_QUEUE'
-num_bg_threads = 2
-num_connect_threads = 2
-num_io_threads = 2
-num_proc_threads = 2
-proc_thread_pool_stratetry = 'SHARED_QUEUE'
-
-[storage_client.retry]
-init_wait_time = '10s'
-max_failures_before_failover = 1
-max_retry_time = '1min'
-max_wait_time = '30s'
-
-[storage_client.traffic_control.query]
-max_batch_bytes = '4MB'
-max_batch_size = 128
-max_concurrent_requests = 32
-max_concurrent_requests_per_server = 8
-process_batches_in_parallel = true
-random_shuffle_requests = true
-
-[storage_client.traffic_control.read]
-max_batch_bytes = '4MB'
-max_batch_size = 128
-max_concurrent_requests = 32
-max_concurrent_requests_per_server = 8
-process_batches_in_parallel = true
-random_shuffle_requests = true
-
-[storage_client.traffic_control.remove]
-max_batch_bytes = '4MB'
-max_batch_size = 128
-max_concurrent_requests = 32
-max_concurrent_requests_per_server = 8
-process_batches_in_parallel = true
-random_shuffle_requests = true
-
-[storage_client.traffic_control.truncate]
-max_batch_bytes = '4MB'
-max_batch_size = 128
-max_concurrent_requests = 32
-max_concurrent_requests_per_server = 8
-process_batches_in_parallel = true
-random_shuffle_requests = true
-
-[storage_client.traffic_control.write]
-max_batch_bytes = '4MB'
-max_batch_size = 128
-max_concurrent_requests = 32
-max_concurrent_requests_per_server = 8
-process_batches_in_parallel = true
-random_shuffle_requests = true
-
-[user_info]
-gid = -1
-gids = []
-token = ''
-uid = -1

asian_ai/chinese/3FS/configs/hf3fs_client_agent.toml

@@ -1,557 +0,0 @@
-[[common.log.categories]]
-categories = [ '.' ]
-handlers = [ 'normal', 'err', 'fatal' ]
-inherit = true
-level = 'INFO'
-propagate = 'NONE'
-
-[[common.log.handlers]]
-async = true
-file_path = ''
-max_file_size = '10MB'
-max_files = 100
-name = 'normal'
-rotate = true
-rotate_on_open = false
-start_level = 'NONE'
-stream_type = 'STDERR'
-writer_type = 'FILE'
-
-[[common.log.handlers]]
-async = false
-file_path = ''
-max_file_size = '10MB'
-max_files = 100
-name = 'err'
-rotate = true
-rotate_on_open = false
-start_level = 'ERR'
-stream_type = 'STDERR'
-writer_type = 'FILE'
-
-[[common.log.handlers]]
-async = false
-file_path = ''
-max_file_size = '10MB'
-max_files = 100
-name = 'fatal'
-rotate = true
-rotate_on_open = false
-start_level = 'FATAL'
-stream_type = 'STDERR'
-writer_type = 'STREAM'
-
-[common.memory]
-prof_active = false
-prof_prefix = ''
-
-[common.monitor]
-reporters = []
-
-[server.agent]
-allow_read_holes = true
-auth_timeout = '5min'
-background_ibreg = true
-list_entry_limit = 100
-max_concurrent_iovallocs = 0
-mock_storage_dir = ''
-mount_name = ''
-read_only_mode = false
-truncate_if_write_after_eof = false
-
-[server.agent.inode_cache]
-capacity = 4194304
-entry_lifetime = '5min'
-
-[server.agent.limit_per_process]
-fd = 1048576
-shm = 1048576
-
-[server.agent.periodic_sync]
-interval = '10min'
-on = false
-
-[server.agent.proc_watch]
-interval = '1min'
-on = true
-
-[server.agent.storage_io.read]
-enableChecksum = false
-
-[server.agent.storage_io.read.debug]
-bypass_disk_io = false
-bypass_rdma_xmit = false
-inject_random_client_error = false
-inject_random_server_error = false
-max_num_of_injection_points = 100
-
-[server.agent.storage_io.read.retry]
-init_wait_time = '0ns'
-max_retry_time = '0ns'
-max_wait_time = '0ns'
-retry_permanent_error = false
-
-[server.agent.storage_io.read.targetSelection]
-mode = 'Default'
-targetIndex = 0
-trafficZone = ''
-
-[server.agent.storage_io.write]
-enableChecksum = true
-
-[server.agent.storage_io.write.debug]
-bypass_disk_io = false
-bypass_rdma_xmit = false
-inject_random_client_error = false
-inject_random_server_error = false
-max_num_of_injection_points = 100
-
-[server.agent.storage_io.write.retry]
-init_wait_time = '0ns'
-max_retry_time = '0ns'
-max_wait_time = '0ns'
-retry_permanent_error = false
-
-[server.agent.storage_io.write.targetSelection]
-mode = 'Default'
-targetIndex = 0
-trafficZone = ''
-
-[server.agent.storage_ops]
-enable_read = true
-enable_write = true
-
-[server.background_client]
-default_compression_level = 0
-default_compression_threshold = '128KB'
-default_log_long_running_threshold = '0ns'
-default_send_retry_times = 1
-default_timeout = '1s'
-enable_rdma_control = false
-force_use_tcp = false
-
-[server.background_client.io_worker]
-num_event_loop = 1
-rdma_connect_timeout = '5s'
-read_write_rdma_in_event_thread = false
-read_write_tcp_in_event_thread = false
-tcp_connect_timeout = '1s'
-wait_to_retry_send = '100ms'
-
-[server.background_client.io_worker.connect_concurrency_limiter]
-max_concurrency = 4
-
-[server.background_client.io_worker.ibsocket]
-buf_ack_batch = 8
-buf_signal_batch = 8
-buf_size = 16384
-drain_timeout = '5s'
-drop_connections = 0
-event_ack_batch = 128
-max_rd_atomic = 16
-max_rdma_wr = 128
-max_rdma_wr_per_post = 32
-max_sge = 16
-min_rnr_timer = 1
-record_bytes_per_peer = false
-record_latency_per_peer = false
-retry_cnt = 7
-rnr_retry = 0
-send_buf_cnt = 32
-sl = 0
-start_psn = 0
-timeout = 14
-
-[server.background_client.io_worker.transport_pool]
-max_connections = 1
-
-[server.background_client.processor]
-enable_coroutines_pool = true
-max_coroutines_num = 256
-max_processing_requests_num = 4096
-response_compression_level = 1
-response_compression_threshold = '128KB'
-
-[server.background_client.rdma_control]
-max_concurrent_transmission = 64
-
-[server.background_client.thread_pool]
-bg_thread_pool_stratetry = 'SHARED_QUEUE'
-collect_stats = false
-enable_work_stealing = false
-io_thread_pool_stratetry = 'SHARED_QUEUE'
-num_bg_threads = 2
-num_connect_threads = 2
-num_io_threads = 2
-num_proc_threads = 2
-proc_thread_pool_stratetry = 'SHARED_QUEUE'
-
-[server.base.independent_thread_pool]
-bg_thread_pool_stratetry = 'SHARED_QUEUE'
-collect_stats = false
-enable_work_stealing = false
-io_thread_pool_stratetry = 'SHARED_QUEUE'
-num_bg_threads = 2
-num_connect_threads = 2
-num_io_threads = 2
-num_proc_threads = 2
-proc_thread_pool_stratetry = 'SHARED_QUEUE'
-
-[server.base.thread_pool]
-bg_thread_pool_stratetry = 'SHARED_QUEUE'
-collect_stats = false
-enable_work_stealing = false
-io_thread_pool_stratetry = 'SHARED_QUEUE'
-num_bg_threads = 2
-num_connect_threads = 2
-num_io_threads = 64
-num_proc_threads = 64
-proc_thread_pool_stratetry = 'SHARED_QUEUE'
-
-[[server.base.groups]]
-check_connections_interval = '1min'
-connection_expiration_time = '1day'
-network_type = 'LOCAL'
-services = [ 'ClientAgentSerde' ]
-use_independent_thread_pool = false
-
-[server.base.groups.io_worker]
-num_event_loop = 1
-rdma_connect_timeout = '5s'
-read_write_rdma_in_event_thread = false
-read_write_tcp_in_event_thread = false
-tcp_connect_timeout = '1s'
-wait_to_retry_send = '100ms'
-
-[server.base.groups.io_worker.connect_concurrency_limiter]
-max_concurrency = 4
-
-[server.base.groups.io_worker.ibsocket]
-buf_ack_batch = 8
-buf_signal_batch = 8
-buf_size = 16384
-drain_timeout = '5s'
-drop_connections = 0
-event_ack_batch = 128
-max_rd_atomic = 16
-max_rdma_wr = 128
-max_rdma_wr_per_post = 32
-max_sge = 16
-min_rnr_timer = 1
-record_bytes_per_peer = false
-record_latency_per_peer = false
-retry_cnt = 7
-rnr_retry = 0
-send_buf_cnt = 32
-sl = 0
-start_psn = 0
-timeout = 14
-
-[server.base.groups.io_worker.transport_pool]
-max_connections = 1
-
-[server.base.groups.listener]
-filter_list = []
-listen_port = 0
-listen_queue_depth = 4096
-rdma_accept_timeout = '15s'
-rdma_listen_ethernet = true
-reuse_port = false
-
-[server.base.groups.processor]
-enable_coroutines_pool = true
-max_coroutines_num = 256
-max_processing_requests_num = 4096
-response_compression_level = 1
-response_compression_threshold = '128KB'
-
-[[server.base.groups]]
-check_connections_interval = '1min'
-connection_expiration_time = '1day'
-network_type = 'TCP'
-services = [ 'Core' ]
-use_independent_thread_pool = true
-
-[server.base.groups.io_worker]
-num_event_loop = 1
-rdma_connect_timeout = '5s'
-read_write_rdma_in_event_thread = false
-read_write_tcp_in_event_thread = false
-tcp_connect_timeout = '1s'
-wait_to_retry_send = '100ms'
-
-[server.base.groups.io_worker.connect_concurrency_limiter]
-max_concurrency = 4
-
-[server.base.groups.io_worker.ibsocket]
-buf_ack_batch = 8
-buf_signal_batch = 8
-buf_size = 16384
-drain_timeout = '5s'
-drop_connections = 0
-event_ack_batch = 128
-max_rd_atomic = 16
-max_rdma_wr = 128
-max_rdma_wr_per_post = 32
-max_sge = 16
-min_rnr_timer = 1
-record_bytes_per_peer = false
-record_latency_per_peer = false
-retry_cnt = 7
-rnr_retry = 0
-send_buf_cnt = 32
-sl = 0
-start_psn = 0
-timeout = 14
-
-[server.base.groups.io_worker.transport_pool]
-max_connections = 1
-
-[server.base.groups.listener]
-filter_list = []
-listen_port = 9000
-listen_queue_depth = 4096
-rdma_accept_timeout = '15s'
-rdma_listen_ethernet = true
-reuse_port = false
-
-[server.base.groups.processor]
-enable_coroutines_pool = true
-max_coroutines_num = 256
-max_processing_requests_num = 4096
-response_compression_level = 1
-response_compression_threshold = '128KB'
-
-[server.meta]
-check_server_interval = '5s'
-dynamic_stripe = false
-max_concurrent_requests = 128
-network_type = 'RDMA'
-remove_chunks_batch_size = 32
-remove_chunks_max_iters = 1024
-selection_mode = 'RandomFollow'
-
-[server.meta.background_closer]
-prune_session_batch_count = 128
-prune_session_batch_interval = '10s'
-retry_first_wait = '100ms'
-retry_max_wait = '10s'
-task_scan = '50ms'
-
-[server.meta.background_closer.coroutine_pool]
-coroutines_num = 8
-enable_work_stealing = false
-queue_size = 128
-
-[server.meta.retry_default]
-max_failures_before_failover = 1
-retry_init_wait = '500ms'
-retry_max_wait = '5s'
-retry_send = 1
-retry_total_time = '1min'
-rpc_timeout = '2s'
-
-[server.meta.retry_truncate]
-max_failures_before_failover = 1
-retry_init_wait = '2s'
-retry_max_wait = '5s'
-retry_send = 1
-retry_total_time = '1min'
-rpc_timeout = '15s'
-
-[server.mgmtd]
-accept_incomplete_routing_info_during_mgmtd_bootstrapping = true
-auto_extend_client_session_interval = '10s'
-auto_heartbeat_interval = '10s'
-auto_refresh_interval = '10s'
-enable_auto_extend_client_session = true
-enable_auto_heartbeat = false
-enable_auto_refresh = true
-mgmtd_server_addresses = []
-work_queue_size = 100
-
-[server.storage]
-check_overlapping_read_buffers = true
-check_overlapping_write_buffers = false
-chunk_checksum_type = 'CRC32C'
-create_net_client_for_updates = false
-implementation_type = 'RPC'
-max_inline_read_bytes = '0'
-max_inline_write_bytes = '0'
-max_read_io_bytes = '0'
-
-[server.storage.net_client]
-default_compression_level = 0
-default_compression_threshold = '128KB'
-default_log_long_running_threshold = '0ns'
-default_send_retry_times = 1
-default_timeout = '1s'
-enable_rdma_control = false
-force_use_tcp = false
-
-[server.storage.net_client.io_worker]
-num_event_loop = 1
-rdma_connect_timeout = '5s'
-read_write_rdma_in_event_thread = false
-read_write_tcp_in_event_thread = false
-tcp_connect_timeout = '1s'
-wait_to_retry_send = '100ms'
-
-[server.storage.net_client.io_worker.connect_concurrency_limiter]
-max_concurrency = 4
-
-[server.storage.net_client.io_worker.ibsocket]
-buf_ack_batch = 8
-buf_signal_batch = 8
-buf_size = 16384
-drain_timeout = '5s'
-drop_connections = 0
-event_ack_batch = 128
-max_rd_atomic = 16
-max_rdma_wr = 128
-max_rdma_wr_per_post = 32
-max_sge = 16
-min_rnr_timer = 1
-record_bytes_per_peer = false
-record_latency_per_peer = false
-retry_cnt = 7
-rnr_retry = 0
-send_buf_cnt = 32
-sl = 0
-start_psn = 0
-timeout = 14
-
-[server.storage.net_client.io_worker.transport_pool]
-max_connections = 1
-
-[server.storage.net_client.processor]
-enable_coroutines_pool = true
-max_coroutines_num = 256
-max_processing_requests_num = 4096
-response_compression_level = 1
-response_compression_threshold = '128KB'
-
-[server.storage.net_client.rdma_control]
-max_concurrent_transmission = 64
-
-[server.storage.net_client.thread_pool]
-bg_thread_pool_stratetry = 'SHARED_QUEUE'
-collect_stats = false
-enable_work_stealing = false
-io_thread_pool_stratetry = 'SHARED_QUEUE'
-num_bg_threads = 2
-num_connect_threads = 2
-num_io_threads = 2
-num_proc_threads = 2
-proc_thread_pool_stratetry = 'SHARED_QUEUE'
-
-[server.storage.net_client_for_updates]
-default_compression_level = 0
-default_compression_threshold = '128KB'
-default_log_long_running_threshold = '0ns'
-default_send_retry_times = 1
-default_timeout = '1s'
-enable_rdma_control = false
-force_use_tcp = false
-
-[server.storage.net_client_for_updates.io_worker]
-num_event_loop = 1
-rdma_connect_timeout = '5s'
-read_write_rdma_in_event_thread = false
-read_write_tcp_in_event_thread = false
-tcp_connect_timeout = '1s'
-wait_to_retry_send = '100ms'
-
-[server.storage.net_client_for_updates.io_worker.connect_concurrency_limiter]
-max_concurrency = 4
-
-[server.storage.net_client_for_updates.io_worker.ibsocket]
-buf_ack_batch = 8
-buf_signal_batch = 8
-buf_size = 16384
-drain_timeout = '5s'
-drop_connections = 0
-event_ack_batch = 128
-max_rd_atomic = 16
-max_rdma_wr = 128
-max_rdma_wr_per_post = 32
-max_sge = 16
-min_rnr_timer = 1
-record_bytes_per_peer = false
-record_latency_per_peer = false
-retry_cnt = 7
-rnr_retry = 0
-send_buf_cnt = 32
-sl = 0
-start_psn = 0
-timeout = 14
-
-[server.storage.net_client_for_updates.io_worker.transport_pool]
-max_connections = 1
-
-[server.storage.net_client_for_updates.processor]
-enable_coroutines_pool = true
-max_coroutines_num = 256
-max_processing_requests_num = 4096
-response_compression_level = 1
-response_compression_threshold = '128KB'
-
-[server.storage.net_client_for_updates.rdma_control]
-max_concurrent_transmission = 64
-
-[server.storage.net_client_for_updates.thread_pool]
-bg_thread_pool_stratetry = 'SHARED_QUEUE'
-collect_stats = false
-enable_work_stealing = false
-io_thread_pool_stratetry = 'SHARED_QUEUE'
-num_bg_threads = 2
-num_connect_threads = 2
-num_io_threads = 2
-num_proc_threads = 2
-proc_thread_pool_stratetry = 'SHARED_QUEUE'
-
-[server.storage.retry]
-init_wait_time = '10s'
-max_failures_before_failover = 1
-max_retry_time = '1min'
-max_wait_time = '30s'
-
-[server.storage.traffic_control.query]
-max_batch_bytes = '4MB'
-max_batch_size = 128
-max_concurrent_requests = 32
-max_concurrent_requests_per_server = 8
-process_batches_in_parallel = true
-random_shuffle_requests = true
-
-[server.storage.traffic_control.read]
-max_batch_bytes = '4MB'
-max_batch_size = 128
-max_concurrent_requests = 32
-max_concurrent_requests_per_server = 8
-process_batches_in_parallel = true
-random_shuffle_requests = true
-
-[server.storage.traffic_control.remove]
-max_batch_bytes = '4MB'
-max_batch_size = 128
-max_concurrent_requests = 32
-max_concurrent_requests_per_server = 8
-process_batches_in_parallel = true
-random_shuffle_requests = true
-
-[server.storage.traffic_control.truncate]
-max_batch_bytes = '4MB'
-max_batch_size = 128
-max_concurrent_requests = 32
-max_concurrent_requests_per_server = 8
-process_batches_in_parallel = true
-random_shuffle_requests = true
-
-[server.storage.traffic_control.write]
-max_batch_bytes = '4MB'
-max_batch_size = 128
-max_concurrent_requests = 32
-max_concurrent_requests_per_server = 8
-process_batches_in_parallel = true
-random_shuffle_requests = true

asian_ai/chinese/3FS/configs/hf3fs_client_agent_app.toml

@@ -1 +0,0 @@
-

asian_ai/chinese/3FS/configs/hf3fs_client_agent_launcher.toml

@@ -1,88 +0,0 @@
-allow_dev_version = true
-cluster_id = ''
-
-[client]
-default_compression_level = 0
-default_compression_threshold = '128KB'
-default_log_long_running_threshold = '0ns'
-default_send_retry_times = 1
-default_timeout = '1s'
-enable_rdma_control = false
-force_use_tcp = false
-
-[client.io_worker]
-num_event_loop = 1
-rdma_connect_timeout = '5s'
-read_write_rdma_in_event_thread = false
-read_write_tcp_in_event_thread = false
-tcp_connect_timeout = '1s'
-wait_to_retry_send = '100ms'
-
-[client.io_worker.connect_concurrency_limiter]
-max_concurrency = 4
-
-[client.io_worker.ibsocket]
-buf_ack_batch = 8
-buf_signal_batch = 8
-buf_size = 16384
-drain_timeout = '5s'
-drop_connections = 0
-event_ack_batch = 128
-max_rd_atomic = 16
-max_rdma_wr = 128
-max_rdma_wr_per_post = 32
-max_sge = 16
-min_rnr_timer = 1
-record_bytes_per_peer = false
-record_latency_per_peer = false
-retry_cnt = 7
-rnr_retry = 0
-send_buf_cnt = 32
-sl = 0
-start_psn = 0
-timeout = 14
-
-[client.io_worker.transport_pool]
-max_connections = 1
-
-[client.processor]
-enable_coroutines_pool = true
-max_coroutines_num = 256
-max_processing_requests_num = 4096
-response_compression_level = 1
-response_compression_threshold = '128KB'
-
-[client.rdma_control]
-max_concurrent_transmission = 64
-
-[client.thread_pool]
-bg_thread_pool_stratetry = 'SHARED_QUEUE'
-collect_stats = false
-enable_work_stealing = false
-io_thread_pool_stratetry = 'SHARED_QUEUE'
-num_bg_threads = 2
-num_connect_threads = 2
-num_io_threads = 2
-num_proc_threads = 2
-proc_thread_pool_stratetry = 'SHARED_QUEUE'
-
-[ib_devices]
-allow_unknown_zone = true
-default_network_zone = 'UNKNOWN'
-default_pkey_index = 0
-default_traffic_class = 0
-device_filter = []
-fork_safe = true
-skip_inactive_ports = true
-subnets = []
-
-[mgmtd_client]
-accept_incomplete_routing_info_during_mgmtd_bootstrapping = true
-auto_extend_client_session_interval = '10s'
-auto_heartbeat_interval = '10s'
-auto_refresh_interval = '10s'
-enable_auto_extend_client_session = true
-enable_auto_heartbeat = false
-enable_auto_refresh = true
-mgmtd_server_addresses = []
-work_queue_size = 100

asian_ai/chinese/3FS/configs/hf3fs_fuse_main.toml

@@ -1,450 +0,0 @@
-attr_timeout = 30.0
-batch_io_coros = 128
-check_rmrf = true
-chunk_size_limit = '0'
-dryrun_bench_mode = false
-enable_interrupt = false
-enable_priority = false
-enable_read_cache = true
-enable_writeback_cache = false
-entry_timeout = 30.0
-fdatasync_update_length = false
-flush_on_stat = true
-fsync_length_hint = false
-io_job_deq_timeout = '1ms'
-io_jobq_size = 1024
-iov_limit = '1MB'
-max_background = 32
-max_idle_threads = -1
-max_jobs_per_ioring = 32
-max_readahead = '16MB'
-max_threads = 256
-max_uid = '1M'
-memset_before_read = false
-negative_timeout = 5.0
-notify_inval_threads = 32
-rdma_buf_pool_size = 1024
-readonly = false
-submit_wait_jitter = '1ms'
-symlink_timeout = 5.0
-sync_on_stat = true
-time_granularity = '1s'
-
-[client]
-default_compression_level = 0
-default_compression_threshold = '128KB'
-default_log_long_running_threshold = '0ns'
-default_report_metrics = false
-default_send_retry_times = 1
-default_timeout = '1s'
-enable_rdma_control = false
-force_use_tcp = false
-
-[client.io_worker]
-num_event_loop = 1
-rdma_connect_timeout = '5s'
-read_write_rdma_in_event_thread = false
-read_write_tcp_in_event_thread = false
-tcp_connect_timeout = '1s'
-wait_to_retry_send = '100ms'
-
-[client.io_worker.connect_concurrency_limiter]
-max_concurrency = 4
-
-[client.io_worker.ibsocket]
-buf_ack_batch = 8
-buf_signal_batch = 8
-buf_size = 16384
-drain_timeout = '5s'
-drop_connections = 0
-event_ack_batch = 128
-max_rd_atomic = 16
-max_rdma_wr = 128
-max_rdma_wr_per_post = 32
-max_sge = 16
-min_rnr_timer = 1
-record_bytes_per_peer = false
-record_latency_per_peer = false
-retry_cnt = 7
-rnr_retry = 0
-send_buf_cnt = 32
-sl = 0
-start_psn = 0
-timeout = 14
-
-[client.io_worker.transport_pool]
-max_connections = 1
-
-[client.processor]
-enable_coroutines_pool = true
-max_coroutines_num = 256
-max_processing_requests_num = 4096
-response_compression_level = 1
-response_compression_threshold = '128KB'
-
-[client.rdma_control]
-max_concurrent_transmission = 64
-
-[client.thread_pool]
-bg_thread_pool_stratetry = 'SHARED_QUEUE'
-collect_stats = false
-enable_work_stealing = false
-io_thread_pool_stratetry = 'SHARED_QUEUE'
-num_bg_threads = 2
-num_connect_threads = 2
-num_io_threads = 2
-num_proc_threads = 2
-proc_thread_pool_stratetry = 'SHARED_QUEUE'
-
-[[common.log.categories]]
-categories = [ '.' ]
-handlers = [ 'normal', 'err', 'fatal' ]
-inherit = true
-level = 'INFO'
-propagate = 'NONE'
-
-[[common.log.handlers]]
-async = true
-file_path = '/var/log/3fs/hf3fs_fuse_main.log'
-max_file_size = '100MB'
-max_files = 10
-name = 'normal'
-rotate = true
-rotate_on_open = false
-start_level = 'NONE'
-stream_type = 'STDERR'
-writer_type = 'FILE'
-
-[[common.log.handlers]]
-async = false
-file_path = '/var/log/3fs/hf3fs_fuse_main-err.log'
-max_file_size = '100MB'
-max_files = 10
-name = 'err'
-rotate = true
-rotate_on_open = false
-start_level = 'ERR'
-stream_type = 'STDERR'
-writer_type = 'FILE'
-
-[[common.log.handlers]]
-async = false
-file_path = '/var/log/3fs/hf3fs_fuse_main-fatal.log'
-max_file_size = '100MB'
-max_files = 10
-name = 'fatal'
-rotate = true
-rotate_on_open = false
-start_level = 'FATAL'
-stream_type = 'STDERR'
-writer_type = 'STREAM'
-
-[common.memory]
-prof_active = false
-prof_prefix = ''
-
-[common.monitor]
-collect_period = '1s'
-num_collectors = 1
-
-[[common.monitor.reporters]]
-type = 'monitor_collector'
-
-[common.monitor.reporters.monitor_collector]
-remote_ip = ''
-
-[io_bufs]
-max_buf_size = '1MB'
-max_readahead = '256KB'
-write_buf_size = '1MB'
-
-[io_jobq_sizes]
-hi = 32
-lo = 4096
-
-[io_worker_coros]
-hi = 8
-lo = 8
-
-[meta]
-check_server_interval = '5s'
-dynamic_stripe = true
-max_concurrent_requests = 128
-network_type = 'RDMA'
-remove_chunks_batch_size = 32
-remove_chunks_max_iters = 1024
-selection_mode = 'RandomFollow'
-
-[meta.background_closer]
-prune_session_batch_count = 128
-prune_session_batch_interval = '10s'
-retry_first_wait = '100ms'
-retry_max_wait = '10s'
-task_scan = '50ms'
-
-[meta.background_closer.coroutine_pool]
-coroutines_num = 8
-enable_work_stealing = false
-queue_size = 128
-
-[meta.retry_default]
-max_failures_before_failover = 1
-retry_fast = '1s'
-retry_init_wait = '500ms'
-retry_max_wait = '5s'
-retry_send = 1
-retry_total_time = '1min'
-rpc_timeout = '5s'
-
-[mgmtd]
-accept_incomplete_routing_info_during_mgmtd_bootstrapping = true
-auto_extend_client_session_interval = '10s'
-auto_heartbeat_interval = '10s'
-auto_refresh_interval = '10s'
-enable_auto_extend_client_session = true
-enable_auto_heartbeat = false
-enable_auto_refresh = true
-mgmtd_server_addresses = []
-work_queue_size = 100
-
-[periodic_sync]
-enable = true
-flush_write_buf = true
-interval = '30s'
-limit = 1000
-
-[periodic_sync.worker]
-coroutines_num = 4
-enable_work_stealing = false
-queue_size = 1024
-
-[storage]
-check_overlapping_read_buffers = true
-check_overlapping_write_buffers = false
-chunk_checksum_type = 'CRC32C'
-create_net_client_for_updates = false
-implementation_type = 'RPC'
-max_inline_read_bytes = '0'
-max_inline_write_bytes = '0'
-max_read_io_bytes = '0'
-
-[storage.net_client]
-default_compression_level = 0
-default_compression_threshold = '128KB'
-default_log_long_running_threshold = '0ns'
-default_report_metrics = false
-default_send_retry_times = 1
-default_timeout = '1s'
-enable_rdma_control = false
-force_use_tcp = false
-
-[storage.net_client.io_worker]
-num_event_loop = 1
-rdma_connect_timeout = '5s'
-read_write_rdma_in_event_thread = false
-read_write_tcp_in_event_thread = false
-tcp_connect_timeout = '1s'
-wait_to_retry_send = '100ms'
-
-[storage.net_client.io_worker.connect_concurrency_limiter]
-max_concurrency = 4
-
-[storage.net_client.io_worker.ibsocket]
-buf_ack_batch = 8
-buf_signal_batch = 8
-buf_size = 16384
-drain_timeout = '5s'
-drop_connections = 0
-event_ack_batch = 128
-max_rd_atomic = 16
-max_rdma_wr = 128
-max_rdma_wr_per_post = 32
-max_sge = 16
-min_rnr_timer = 1
-record_bytes_per_peer = false
-record_latency_per_peer = false
-retry_cnt = 7
-rnr_retry = 0
-send_buf_cnt = 32
-sl = 0
-start_psn = 0
-timeout = 14
-
-[storage.net_client.io_worker.transport_pool]
-max_connections = 1
-
-[storage.net_client.processor]
-enable_coroutines_pool = true
-max_coroutines_num = 256
-max_processing_requests_num = 4096
-response_compression_level = 1
-response_compression_threshold = '128KB'
-
-[storage.net_client.rdma_control]
-max_concurrent_transmission = 64
-
-[storage.net_client.thread_pool]
-bg_thread_pool_stratetry = 'SHARED_QUEUE'
-collect_stats = false
-enable_work_stealing = false
-io_thread_pool_stratetry = 'SHARED_QUEUE'
-num_bg_threads = 2
-num_connect_threads = 2
-num_io_threads = 2
-num_proc_threads = 2
-proc_thread_pool_stratetry = 'SHARED_QUEUE'
-
-[storage.net_client_for_updates]
-default_compression_level = 0
-default_compression_threshold = '128KB'
-default_log_long_running_threshold = '0ns'
-default_report_metrics = false
-default_send_retry_times = 1
-default_timeout = '1s'
-enable_rdma_control = false
-force_use_tcp = false
-
-[storage.net_client_for_updates.io_worker]
-num_event_loop = 1
-rdma_connect_timeout = '5s'
-read_write_rdma_in_event_thread = false
-read_write_tcp_in_event_thread = false
-tcp_connect_timeout = '1s'
-wait_to_retry_send = '100ms'
-
-[storage.net_client_for_updates.io_worker.connect_concurrency_limiter]
-max_concurrency = 4
-
-[storage.net_client_for_updates.io_worker.ibsocket]
-buf_ack_batch = 8
-buf_signal_batch = 8
-buf_size = 16384
-drain_timeout = '5s'
-drop_connections = 0
-event_ack_batch = 128
-max_rd_atomic = 16
-max_rdma_wr = 128
-max_rdma_wr_per_post = 32
-max_sge = 16
-min_rnr_timer = 1
-record_bytes_per_peer = false
-record_latency_per_peer = false
-retry_cnt = 7
-rnr_retry = 0
-send_buf_cnt = 32
-sl = 0
-start_psn = 0
-timeout = 14
-
-[storage.net_client_for_updates.io_worker.transport_pool]
-max_connections = 1
-
-[storage.net_client_for_updates.processor]
-enable_coroutines_pool = true
-max_coroutines_num = 256
-max_processing_requests_num = 4096
-response_compression_level = 1
-response_compression_threshold = '128KB'
-
-[storage.net_client_for_updates.rdma_control]
-max_concurrent_transmission = 64
-
-[storage.net_client_for_updates.thread_pool]
-bg_thread_pool_stratetry = 'SHARED_QUEUE'
-collect_stats = false
-enable_work_stealing = false
-io_thread_pool_stratetry = 'SHARED_QUEUE'
-num_bg_threads = 2
-num_connect_threads = 2
-num_io_threads = 2
-num_proc_threads = 2
-proc_thread_pool_stratetry = 'SHARED_QUEUE'
-
-[storage.retry]
-init_wait_time = '10s'
-max_failures_before_failover = 1
-max_retry_time = '1min'
-max_wait_time = '30s'
-
-[storage.traffic_control.query]
-max_batch_bytes = '4MB'
-max_batch_size = 128
-max_concurrent_requests = 32
-max_concurrent_requests_per_server = 8
-process_batches_in_parallel = true
-random_shuffle_requests = true
-
-[storage.traffic_control.read]
-max_batch_bytes = '4MB'
-max_batch_size = 128
-max_concurrent_requests = 32
-max_concurrent_requests_per_server = 8
-process_batches_in_parallel = true
-random_shuffle_requests = true
-
-[storage.traffic_control.remove]
-max_batch_bytes = '4MB'
-max_batch_size = 128
-max_concurrent_requests = 32
-max_concurrent_requests_per_server = 8
-process_batches_in_parallel = true
-random_shuffle_requests = true
-
-[storage.traffic_control.truncate]
-max_batch_bytes = '4MB'
-max_batch_size = 128
-max_concurrent_requests = 32
-max_concurrent_requests_per_server = 8
-process_batches_in_parallel = true
-random_shuffle_requests = true
-
-[storage.traffic_control.write]
-max_batch_bytes = '4MB'
-max_batch_size = 128
-max_concurrent_requests = 32
-max_concurrent_requests_per_server = 8
-process_batches_in_parallel = true
-random_shuffle_requests = true
-
-[storage_io.read]
-allowReadUncommitted = false
-enableChecksum = false
-
-[storage_io.read.debug]
-bypass_disk_io = false
-bypass_rdma_xmit = false
-inject_random_client_error = false
-inject_random_server_error = false
-max_num_of_injection_points = 100
-
-[storage_io.read.retry]
-init_wait_time = '0ns'
-max_retry_time = '0ns'
-max_wait_time = '0ns'
-retry_permanent_error = false
-
-[storage_io.read.targetSelection]
-mode = 'Default'
-targetIndex = 0
-trafficZone = ''
-
-[storage_io.write]
-enableChecksum = true
-
-[storage_io.write.debug]
-bypass_disk_io = false
-bypass_rdma_xmit = false
-inject_random_client_error = false
-inject_random_server_error = false
-max_num_of_injection_points = 100
-
-[storage_io.write.retry]
-init_wait_time = '0ns'
-max_retry_time = '0ns'
-max_wait_time = '0ns'
-retry_permanent_error = false
-
-[storage_io.write.targetSelection]
-mode = 'Default'
-targetIndex = 0
-trafficZone = ''

asian_ai/chinese/3FS/configs/hf3fs_fuse_main_app.toml

@@ -1 +0,0 @@
-

asian_ai/chinese/3FS/configs/hf3fs_fuse_main_launcher.toml

@@ -1,95 +0,0 @@
-allow_other = true
-cluster_id = ''
-mountpoint = ''
-token_file = ''
-
-[client]
-default_compression_level = 0
-default_compression_threshold = '128KB'
-default_log_long_running_threshold = '0ns'
-default_report_metrics = false
-default_send_retry_times = 1
-default_timeout = '1s'
-enable_rdma_control = false
-force_use_tcp = false
-
-[client.io_worker]
-num_event_loop = 1
-rdma_connect_timeout = '5s'
-read_write_rdma_in_event_thread = false
-read_write_tcp_in_event_thread = false
-tcp_connect_timeout = '1s'
-wait_to_retry_send = '100ms'
-
-[client.io_worker.connect_concurrency_limiter]
-max_concurrency = 4
-
-[client.io_worker.ibsocket]
-buf_ack_batch = 8
-buf_signal_batch = 8
-buf_size = 16384
-drain_timeout = '5s'
-drop_connections = 0
-event_ack_batch = 128
-max_rd_atomic = 16
-max_rdma_wr = 128
-max_rdma_wr_per_post = 32
-max_sge = 16
-min_rnr_timer = 1
-record_bytes_per_peer = false
-record_latency_per_peer = false
-retry_cnt = 7
-rnr_retry = 0
-send_buf_cnt = 32
-sl = 0
-start_psn = 0
-timeout = 14
-
-[client.io_worker.transport_pool]
-max_connections = 1
-
-[client.processor]
-enable_coroutines_pool = true
-max_coroutines_num = 256
-max_processing_requests_num = 4096
-response_compression_level = 1
-response_compression_threshold = '128KB'
-
-[client.rdma_control]
-max_concurrent_transmission = 64
-
-[client.thread_pool]
-bg_thread_pool_stratetry = 'SHARED_QUEUE'
-collect_stats = false
-enable_work_stealing = false
-io_thread_pool_stratetry = 'SHARED_QUEUE'
-num_bg_threads = 2
-num_connect_threads = 2
-num_io_threads = 2
-num_proc_threads = 2
-proc_thread_pool_stratetry = 'SHARED_QUEUE'
-
-[ib_devices]
-allow_no_usable_devices = false
-allow_unknown_zone = true
-default_network_zone = 'UNKNOWN'
-default_pkey_index = 0
-default_roce_pkey_index = 0
-default_traffic_class = 0
-device_filter = []
-fork_safe = true
-prefer_ibdevice = true
-skip_inactive_ports = true
-skip_unusable_device = true
-subnets = []
-
-[mgmtd_client]
-accept_incomplete_routing_info_during_mgmtd_bootstrapping = true
-auto_extend_client_session_interval = '10s'
-auto_heartbeat_interval = '10s'
-auto_refresh_interval = '10s'
-enable_auto_extend_client_session = true
-enable_auto_heartbeat = false
-enable_auto_refresh = true
-mgmtd_server_addresses = []
-work_queue_size = 100

asian_ai/chinese/3FS/configs/meta_main.toml

@@ -1,633 +0,0 @@
-[[common.log.categories]]
-categories = [ '.' ]
-handlers = [ 'normal', 'err', 'fatal' ]
-inherit = true
-level = 'INFO'
-propagate = 'NONE'
-
-[[common.log.categories]]
-categories = [ 'eventlog' ]
-handlers = [ 'event' ]
-inherit = false
-level = 'INFO'
-propagate = 'ERR'
-
-[[common.log.handlers]]
-async = true
-file_path = '/var/log/3fs/hf3fs_meta_main.log'
-max_file_size = '100MB'
-max_files = 10
-name = 'normal'
-rotate = true
-rotate_on_open = false
-start_level = 'NONE'
-stream_type = 'STDERR'
-writer_type = 'FILE'
-
-[[common.log.handlers]]
-async = false
-file_path = '/var/log/3fs/hf3fs_meta_main-err.log'
-max_file_size = '100MB'
-max_files = 10
-name = 'err'
-rotate = true
-rotate_on_open = false
-start_level = 'ERR'
-stream_type = 'STDERR'
-writer_type = 'FILE'
-
-[[common.log.handlers]]
-async = false
-file_path = '/var/log/3fs/hf3fs_meta_main-fatal.log'
-max_file_size = '100MB'
-max_files = 10
-name = 'fatal'
-rotate = true
-rotate_on_open = false
-start_level = 'FATAL'
-stream_type = 'STDERR'
-writer_type = 'STREAM'
-
-[[common.log.handlers]]
-async = true
-file_path = '/var/log/3fs/hf3fs_meta_main-event.log'
-max_file_size = '100MB'
-max_files = 10
-name = 'event'
-rotate = true
-rotate_on_open = false
-start_level = 'INFO'
-stream_type = 'STDERR'
-writer_type = 'EVENT'
-
-[common.memory]
-prof_active = false
-prof_prefix = ''
-
-[common.monitor]
-collect_period = '1s'
-num_collectors = 1
-
-[[common.monitor.reporters]]
-type = 'monitor_collector'
-
-[common.monitor.reporters.monitor_collector]
-remote_ip = ''
-
-[server]
-use_memkv = false
-
-[server.background_client]
-default_compression_level = 0
-default_compression_threshold = '128KB'
-default_log_long_running_threshold = '0ns'
-default_report_metrics = false
-default_send_retry_times = 1
-default_timeout = '1s'
-enable_rdma_control = false
-force_use_tcp = false
-
-[server.background_client.io_worker]
-num_event_loop = 1
-rdma_connect_timeout = '5s'
-read_write_rdma_in_event_thread = false
-read_write_tcp_in_event_thread = false
-tcp_connect_timeout = '1s'
-wait_to_retry_send = '100ms'
-
-[server.background_client.io_worker.connect_concurrency_limiter]
-max_concurrency = 4
-
-[server.background_client.io_worker.ibsocket]
-buf_ack_batch = 8
-buf_signal_batch = 8
-buf_size = 16384
-drain_timeout = '5s'
-drop_connections = 0
-event_ack_batch = 128
-max_rd_atomic = 16
-max_rdma_wr = 128
-max_rdma_wr_per_post = 32
-max_sge = 16
-min_rnr_timer = 1
-record_bytes_per_peer = false
-record_latency_per_peer = false
-retry_cnt = 7
-rnr_retry = 0
-send_buf_cnt = 32
-sl = 0
-start_psn = 0
-timeout = 14
-
-[server.background_client.io_worker.transport_pool]
-max_connections = 1
-
-[server.background_client.processor]
-enable_coroutines_pool = true
-max_coroutines_num = 256
-max_processing_requests_num = 4096
-response_compression_level = 1
-response_compression_threshold = '128KB'
-
-[server.background_client.rdma_control]
-max_concurrent_transmission = 64
-
-[server.background_client.thread_pool]
-bg_thread_pool_stratetry = 'SHARED_QUEUE'
-collect_stats = false
-enable_work_stealing = false
-io_thread_pool_stratetry = 'SHARED_QUEUE'
-num_bg_threads = 2
-num_connect_threads = 2
-num_io_threads = 2
-num_proc_threads = 2
-proc_thread_pool_stratetry = 'SHARED_QUEUE'
-
-[server.base.independent_thread_pool]
-bg_thread_pool_stratetry = 'SHARED_QUEUE'
-collect_stats = false
-enable_work_stealing = false
-io_thread_pool_stratetry = 'SHARED_QUEUE'
-num_bg_threads = 2
-num_connect_threads = 2
-num_io_threads = 2
-num_proc_threads = 2
-proc_thread_pool_stratetry = 'SHARED_QUEUE'
-
-[server.base.thread_pool]
-bg_thread_pool_stratetry = 'SHARED_QUEUE'
-collect_stats = false
-enable_work_stealing = false
-io_thread_pool_stratetry = 'SHARED_QUEUE'
-num_bg_threads = 2
-num_connect_threads = 2
-num_io_threads = 2
-num_proc_threads = 2
-proc_thread_pool_stratetry = 'SHARED_QUEUE'
-
-[[server.base.groups]]
-check_connections_interval = '1min'
-connection_expiration_time = '1day'
-network_type = 'RDMA'
-services = [ 'MetaSerde' ]
-use_independent_thread_pool = false
-
-[server.base.groups.io_worker]
-num_event_loop = 1
-rdma_connect_timeout = '5s'
-read_write_rdma_in_event_thread = false
-read_write_tcp_in_event_thread = false
-tcp_connect_timeout = '1s'
-wait_to_retry_send = '100ms'
-
-[server.base.groups.io_worker.connect_concurrency_limiter]
-max_concurrency = 4
-
-[server.base.groups.io_worker.ibsocket]
-buf_ack_batch = 8
-buf_signal_batch = 8
-buf_size = 16384
-drain_timeout = '5s'
-drop_connections = 0
-event_ack_batch = 128
-max_rd_atomic = 16
-max_rdma_wr = 128
-max_rdma_wr_per_post = 32
-max_sge = 16
-min_rnr_timer = 1
-record_bytes_per_peer = false
-record_latency_per_peer = false
-retry_cnt = 7
-rnr_retry = 0
-send_buf_cnt = 32
-sl = 0
-start_psn = 0
-timeout = 14
-
-[server.base.groups.io_worker.transport_pool]
-max_connections = 1
-
-[server.base.groups.listener]
-domain_socket_index = 1
-filter_list = []
-listen_port = 8001
-listen_queue_depth = 4096
-rdma_accept_timeout = '15s'
-rdma_listen_ethernet = true
-reuse_port = false
-
-[server.base.groups.processor]
-enable_coroutines_pool = true
-max_coroutines_num = 256
-max_processing_requests_num = 4096
-response_compression_level = 1
-response_compression_threshold = '128KB'
-
-[[server.base.groups]]
-check_connections_interval = '1min'
-connection_expiration_time = '1day'
-network_type = 'TCP'
-services = [ 'Core' ]
-use_independent_thread_pool = true
-
-[server.base.groups.io_worker]
-num_event_loop = 1
-rdma_connect_timeout = '5s'
-read_write_rdma_in_event_thread = false
-read_write_tcp_in_event_thread = false
-tcp_connect_timeout = '1s'
-wait_to_retry_send = '100ms'
-
-[server.base.groups.io_worker.connect_concurrency_limiter]
-max_concurrency = 4
-
-[server.base.groups.io_worker.ibsocket]
-buf_ack_batch = 8
-buf_signal_batch = 8
-buf_size = 16384
-drain_timeout = '5s'
-drop_connections = 0
-event_ack_batch = 128
-max_rd_atomic = 16
-max_rdma_wr = 128
-max_rdma_wr_per_post = 32
-max_sge = 16
-min_rnr_timer = 1
-record_bytes_per_peer = false
-record_latency_per_peer = false
-retry_cnt = 7
-rnr_retry = 0
-send_buf_cnt = 32
-sl = 0
-start_psn = 0
-timeout = 14
-
-[server.base.groups.io_worker.transport_pool]
-max_connections = 1
-
-[server.base.groups.listener]
-domain_socket_index = 1
-filter_list = []
-listen_port = 9001
-listen_queue_depth = 4096
-rdma_accept_timeout = '15s'
-rdma_listen_ethernet = true
-reuse_port = false
-
-[server.base.groups.processor]
-enable_coroutines_pool = true
-max_coroutines_num = 256
-max_processing_requests_num = 4096
-response_compression_level = 1
-response_compression_threshold = '128KB'
-
-[server.fdb]
-casual_read_risky = false
-clusterFile = ''
-default_backoff = 0
-enableMultipleClient = false
-externalClientDir = ''
-externalClientPath = ''
-multipleClientThreadNum = 4
-readonly = false
-trace_file = ''
-trace_format = 'json'
-
-[server.kv_engine]
-use_memkv = false
-
-[server.kv_engine.fdb]
-casual_read_risky = false
-clusterFile = ''
-default_backoff = 0
-enableMultipleClient = false
-externalClientDir = ''
-externalClientPath = ''
-multipleClientThreadNum = 4
-readonly = false
-trace_file = ''
-trace_format = 'json'
-
-[server.meta]
-acl_cache_time = '15s'
-allow_directly_move_to_trash = false
-allow_owner_change_immutable = false
-allow_stat_deleted_inodes = true
-authenticate = false
-batch_stat_by_path_concurrent = 4
-batch_stat_concurrent = 8
-check_file_hole = false
-dynamic_stripe = false
-dynamic_stripe_growth = 2
-dynamic_stripe_initial = 16
-enable_new_chunk_engine = false
-grv_cache = false
-idempotent_record_clean = '1min'
-idempotent_record_expire = '30min'
-idempotent_remove = true
-idempotent_rename = false
-iflags_chain_allocation = false
-iflags_chunk_engine = true
-ignore_length_hint = false
-inodeId_abort_on_duplicate = false
-inodeId_check_unique = true
-list_default_limit = 128
-max_batch_operations = 4096
-max_directory_depth = 64
-max_remove_chunks_per_request = 32
-max_symlink_count = 10
-max_symlink_depth = 4
-operation_timeout = '5s'
-otrunc_replace_file = true
-otrunc_replace_file_threshold = '1GB'
-readonly = false
-recursive_remove_check_owner = true
-recursive_remove_perm_check = 1024
-statfs_cache_time = '1min'
-statfs_space_imbalance_threshold = 5
-statfs_update_interval = '5s'
-sync_on_prune_session = false
-time_granularity = '1s'
-
-[server.meta.background_hole_checker]
-coroutines_num = 16
-enable_work_stealing = false
-queue_size = 4096
-
-[server.meta.distributor]
-timeout = '30s'
-update_interval = '1s'
-
-[server.meta.event_trace_log]
-dump_interval = '30s'
-enabled = true
-max_num_writers = 1
-max_row_group_length = 100000
-trace_file_dir = '.'
-
-[server.meta.forward]
-addr_type = 'RDMA'
-debug = true
-timeout = '10s'
-
-[server.meta.gc]
-check_session = true
-distributed_gc = true
-enable = true
-gc_delay_free_space_threshold = 5
-gc_directory_concurrent = 4
-gc_directory_delay = '0ns'
-gc_directory_entry_batch = 32
-gc_directory_entry_concurrent = 4
-gc_file_concurrent = 32
-gc_file_delay = '5min'
-large_file_chunks = 128
-recursive_perm_check = true
-remove_chunks_batch_size = 32
-retry_delay = '10min'
-scan_batch = 4096
-scan_interval = '200ms'
-small_file_chunks = 32
-txn_low_priority = false
-
-[server.meta.gc.retry_remove_chunks]
-init_wait_time = '10s'
-max_retry_time = '30s'
-max_wait_time = '10s'
-retry_permanent_error = false
-
-[server.meta.gc.workers]
-coroutines_num = 8
-enable_work_stealing = false
-queue_size = 1024
-
-[server.meta.retry_remove_chunks]
-init_wait_time = '10s'
-max_retry_time = '30s'
-max_wait_time = '10s'
-retry_permanent_error = false
-
-[server.meta.retry_transaction]
-max_backoff = '1s'
-max_retry_count = 10
-
-[server.meta.session_manager]
-enable = true
-scan_batch = 1024
-scan_interval = '5min'
-session_timeout = '5min'
-sync_on_prune_session = false
-
-[server.meta.session_manager.close_workers]
-coroutines_num = 32
-enable_work_stealing = false
-queue_size = 1024
-
-[server.meta.session_manager.scan_workers]
-coroutines_num = 8
-enable_work_stealing = false
-queue_size = 128
-
-[server.meta.user_cache]
-buckets = 127
-exist_ttl = '5min'
-inexist_ttl = '10s'
-
-[server.mgmtd_client]
-accept_incomplete_routing_info_during_mgmtd_bootstrapping = true
-auto_extend_client_session_interval = '10s'
-auto_heartbeat_interval = '10s'
-auto_refresh_interval = '10s'
-enable_auto_extend_client_session = false
-enable_auto_heartbeat = true
-enable_auto_refresh = true
-mgmtd_server_addresses = []
-work_queue_size = 100
-
-[server.storage_client]
-check_overlapping_read_buffers = true
-check_overlapping_write_buffers = false
-chunk_checksum_type = 'CRC32C'
-create_net_client_for_updates = false
-implementation_type = 'RPC'
-max_inline_read_bytes = '0'
-max_inline_write_bytes = '0'
-max_read_io_bytes = '0'
-
-[server.storage_client.net_client]
-default_compression_level = 0
-default_compression_threshold = '128KB'
-default_log_long_running_threshold = '0ns'
-default_report_metrics = false
-default_send_retry_times = 1
-default_timeout = '1s'
-enable_rdma_control = false
-force_use_tcp = false
-
-[server.storage_client.net_client.io_worker]
-num_event_loop = 1
-rdma_connect_timeout = '5s'
-read_write_rdma_in_event_thread = false
-read_write_tcp_in_event_thread = false
-tcp_connect_timeout = '1s'
-wait_to_retry_send = '100ms'
-
-[server.storage_client.net_client.io_worker.connect_concurrency_limiter]
-max_concurrency = 4
-
-[server.storage_client.net_client.io_worker.ibsocket]
-buf_ack_batch = 8
-buf_signal_batch = 8
-buf_size = 16384
-drain_timeout = '5s'
-drop_connections = 0
-event_ack_batch = 128
-max_rd_atomic = 16
-max_rdma_wr = 128
-max_rdma_wr_per_post = 32
-max_sge = 16
-min_rnr_timer = 1
-record_bytes_per_peer = false
-record_latency_per_peer = false
-retry_cnt = 7
-rnr_retry = 0
-send_buf_cnt = 32
-sl = 0
-start_psn = 0
-timeout = 14
-
-[server.storage_client.net_client.io_worker.transport_pool]
-max_connections = 1
-
-[server.storage_client.net_client.processor]
-enable_coroutines_pool = true
-max_coroutines_num = 256
-max_processing_requests_num = 4096
-response_compression_level = 1
-response_compression_threshold = '128KB'
-
-[server.storage_client.net_client.rdma_control]
-max_concurrent_transmission = 64
-
-[server.storage_client.net_client.thread_pool]
-bg_thread_pool_stratetry = 'SHARED_QUEUE'
-collect_stats = false
-enable_work_stealing = false
-io_thread_pool_stratetry = 'SHARED_QUEUE'
-num_bg_threads = 2
-num_connect_threads = 2
-num_io_threads = 2
-num_proc_threads = 2
-proc_thread_pool_stratetry = 'SHARED_QUEUE'
-
-[server.storage_client.net_client_for_updates]
-default_compression_level = 0
-default_compression_threshold = '128KB'
-default_log_long_running_threshold = '0ns'
-default_report_metrics = false
-default_send_retry_times = 1
-default_timeout = '1s'
-enable_rdma_control = false
-force_use_tcp = false
-
-[server.storage_client.net_client_for_updates.io_worker]
-num_event_loop = 1
-rdma_connect_timeout = '5s'
-read_write_rdma_in_event_thread = false
-read_write_tcp_in_event_thread = false
-tcp_connect_timeout = '1s'
-wait_to_retry_send = '100ms'
-
-[server.storage_client.net_client_for_updates.io_worker.connect_concurrency_limiter]
-max_concurrency = 4
-
-[server.storage_client.net_client_for_updates.io_worker.ibsocket]
-buf_ack_batch = 8
-buf_signal_batch = 8
-buf_size = 16384
-drain_timeout = '5s'
-drop_connections = 0
-event_ack_batch = 128
-max_rd_atomic = 16
-max_rdma_wr = 128
-max_rdma_wr_per_post = 32
-max_sge = 16
-min_rnr_timer = 1
-record_bytes_per_peer = false
-record_latency_per_peer = false
-retry_cnt = 7
-rnr_retry = 0
-send_buf_cnt = 32
-sl = 0
-start_psn = 0
-timeout = 14
-
-[server.storage_client.net_client_for_updates.io_worker.transport_pool]
-max_connections = 1
-
-[server.storage_client.net_client_for_updates.processor]
-enable_coroutines_pool = true
-max_coroutines_num = 256
-max_processing_requests_num = 4096
-response_compression_level = 1
-response_compression_threshold = '128KB'
-
-[server.storage_client.net_client_for_updates.rdma_control]
-max_concurrent_transmission = 64
-
-[server.storage_client.net_client_for_updates.thread_pool]
-bg_thread_pool_stratetry = 'SHARED_QUEUE'
-collect_stats = false
-enable_work_stealing = false
-io_thread_pool_stratetry = 'SHARED_QUEUE'
-num_bg_threads = 2
-num_connect_threads = 2
-num_io_threads = 2
-num_proc_threads = 2
-proc_thread_pool_stratetry = 'SHARED_QUEUE'
-
-[server.storage_client.retry]
-init_wait_time = '2s'
-max_failures_before_failover = 1
-max_retry_time = '5s'
-max_wait_time = '5s'
-
-[server.storage_client.traffic_control.query]
-max_batch_bytes = '4MB'
-max_batch_size = 128
-max_concurrent_requests = 32
-max_concurrent_requests_per_server = 8
-process_batches_in_parallel = true
-random_shuffle_requests = true
-
-[server.storage_client.traffic_control.read]
-max_batch_bytes = '4MB'
-max_batch_size = 128
-max_concurrent_requests = 32
-max_concurrent_requests_per_server = 8
-process_batches_in_parallel = true
-random_shuffle_requests = true
-
-[server.storage_client.traffic_control.remove]
-max_batch_bytes = '4MB'
-max_batch_size = 128
-max_concurrent_requests = 32
-max_concurrent_requests_per_server = 8
-process_batches_in_parallel = true
-random_shuffle_requests = true
-
-[server.storage_client.traffic_control.truncate]
-max_batch_bytes = '4MB'
-max_batch_size = 128
-max_concurrent_requests = 32
-max_concurrent_requests_per_server = 8
-process_batches_in_parallel = true
-random_shuffle_requests = true
-
-[server.storage_client.traffic_control.write]
-max_batch_bytes = '4MB'
-max_batch_size = 128
-max_concurrent_requests = 32
-max_concurrent_requests_per_server = 8
-process_batches_in_parallel = true
-random_shuffle_requests = true

asian_ai/chinese/3FS/configs/meta_main_app.toml

@@ -1,2 +0,0 @@
-allow_empty_node_id = true
-node_id = 0

asian_ai/chinese/3FS/configs/meta_main_launcher.toml

@@ -1,93 +0,0 @@
-allow_dev_version = true
-cluster_id = ''
-
-[client]
-default_compression_level = 0
-default_compression_threshold = '128KB'
-default_log_long_running_threshold = '0ns'
-default_report_metrics = false
-default_send_retry_times = 1
-default_timeout = '1s'
-enable_rdma_control = false
-force_use_tcp = false
-
-[client.io_worker]
-num_event_loop = 1
-rdma_connect_timeout = '5s'
-read_write_rdma_in_event_thread = false
-read_write_tcp_in_event_thread = false
-tcp_connect_timeout = '1s'
-wait_to_retry_send = '100ms'
-
-[client.io_worker.connect_concurrency_limiter]
-max_concurrency = 4
-
-[client.io_worker.ibsocket]
-buf_ack_batch = 8
-buf_signal_batch = 8
-buf_size = 16384
-drain_timeout = '5s'
-drop_connections = 0
-event_ack_batch = 128
-max_rd_atomic = 16
-max_rdma_wr = 128
-max_rdma_wr_per_post = 32
-max_sge = 16
-min_rnr_timer = 1
-record_bytes_per_peer = false
-record_latency_per_peer = false
-retry_cnt = 7
-rnr_retry = 0
-send_buf_cnt = 32
-sl = 0
-start_psn = 0
-timeout = 14
-
-[client.io_worker.transport_pool]
-max_connections = 1
-
-[client.processor]
-enable_coroutines_pool = true
-max_coroutines_num = 256
-max_processing_requests_num = 4096
-response_compression_level = 1
-response_compression_threshold = '128KB'
-
-[client.rdma_control]
-max_concurrent_transmission = 64
-
-[client.thread_pool]
-bg_thread_pool_stratetry = 'SHARED_QUEUE'
-collect_stats = false
-enable_work_stealing = false
-io_thread_pool_stratetry = 'SHARED_QUEUE'
-num_bg_threads = 2
-num_connect_threads = 2
-num_io_threads = 2
-num_proc_threads = 2
-proc_thread_pool_stratetry = 'SHARED_QUEUE'
-
-[ib_devices]
-allow_no_usable_devices = false
-allow_unknown_zone = true
-default_network_zone = 'UNKNOWN'
-default_pkey_index = 0
-default_roce_pkey_index = 0
-default_traffic_class = 0
-device_filter = []
-fork_safe = true
-prefer_ibdevice = true
-skip_inactive_ports = true
-skip_unusable_device = true
-subnets = []
-
-[mgmtd_client]
-accept_incomplete_routing_info_during_mgmtd_bootstrapping = true
-auto_extend_client_session_interval = '10s'
-auto_heartbeat_interval = '10s'
-auto_refresh_interval = '10s'
-enable_auto_extend_client_session = false
-enable_auto_heartbeat = true
-enable_auto_refresh = true
-mgmtd_server_addresses = []
-work_queue_size = 100

asian_ai/chinese/3FS/configs/mgmtd_main.toml

@@ -1,232 +0,0 @@
-[[common.log.categories]]
-categories = [ '.' ]
-handlers = [ 'normal', 'err', 'fatal' ]
-inherit = true
-level = 'INFO'
-propagate = 'NONE'
-
-[[common.log.handlers]]
-async = true
-file_path = '/var/log/3fs/mgmtd_main.log'
-max_file_size = '100MB'
-max_files = 10
-name = 'normal'
-rotate = true
-rotate_on_open = false
-start_level = 'NONE'
-stream_type = 'STDERR'
-writer_type = 'FILE'
-
-[[common.log.handlers]]
-async = false
-file_path = '/var/log/3fs/mgmtd_main-err.log'
-max_file_size = '100MB'
-max_files = 10
-name = 'err'
-rotate = true
-rotate_on_open = false
-start_level = 'ERR'
-stream_type = 'STDERR'
-writer_type = 'FILE'
-
-[[common.log.handlers]]
-async = false
-file_path = '/var/log/3fs/mgmtd_main-fatal.log'
-max_file_size = '100MB'
-max_files = 10
-name = 'fatal'
-rotate = true
-rotate_on_open = false
-start_level = 'FATAL'
-stream_type = 'STDERR'
-writer_type = 'STREAM'
-
-[common.memory]
-prof_active = false
-prof_prefix = ''
-
-[common.monitor]
-collect_period = '1s'
-num_collectors = 1
-
-[[common.monitor.reporters]]
-type = 'monitor_collector'
-
-[common.monitor.reporters.monitor_collector]
-remote_ip = ""
-
-[server.base.independent_thread_pool]
-bg_thread_pool_stratetry = 'SHARED_QUEUE'
-collect_stats = false
-enable_work_stealing = false
-io_thread_pool_stratetry = 'SHARED_QUEUE'
-num_bg_threads = 2
-num_connect_threads = 2
-num_io_threads = 2
-num_proc_threads = 2
-proc_thread_pool_stratetry = 'SHARED_QUEUE'
-
-[server.base.thread_pool]
-bg_thread_pool_stratetry = 'SHARED_QUEUE'
-collect_stats = false
-enable_work_stealing = false
-io_thread_pool_stratetry = 'SHARED_QUEUE'
-num_bg_threads = 2
-num_connect_threads = 2
-num_io_threads = 2
-num_proc_threads = 2
-proc_thread_pool_stratetry = 'SHARED_QUEUE'
-
-[[server.base.groups]]
-check_connections_interval = '1min'
-connection_expiration_time = '1day'
-network_type = 'RDMA'
-services = [ 'Mgmtd' ]
-use_independent_thread_pool = false
-
-[server.base.groups.io_worker]
-num_event_loop = 1
-rdma_connect_timeout = '5s'
-read_write_rdma_in_event_thread = false
-read_write_tcp_in_event_thread = false
-tcp_connect_timeout = '1s'
-wait_to_retry_send = '100ms'
-
-[server.base.groups.io_worker.connect_concurrency_limiter]
-max_concurrency = 4
-
-[server.base.groups.io_worker.ibsocket]
-buf_ack_batch = 8
-buf_signal_batch = 8
-buf_size = 16384
-drain_timeout = '5s'
-drop_connections = 0
-event_ack_batch = 128
-max_rd_atomic = 16
-max_rdma_wr = 128
-max_rdma_wr_per_post = 32
-max_sge = 16
-min_rnr_timer = 1
-record_bytes_per_peer = false
-record_latency_per_peer = false
-retry_cnt = 7
-rnr_retry = 0
-send_buf_cnt = 32
-sl = 0
-start_psn = 0
-timeout = 14
-
-[server.base.groups.io_worker.transport_pool]
-max_connections = 1
-
-[server.base.groups.listener]
-domain_socket_index = 1
-filter_list = []
-listen_port = 8000
-listen_queue_depth = 4096
-rdma_accept_timeout = '15s'
-rdma_listen_ethernet = true
-reuse_port = false
-
-[server.base.groups.processor]
-enable_coroutines_pool = true
-max_coroutines_num = 256
-max_processing_requests_num = 4096
-response_compression_level = 1
-response_compression_threshold = '128KB'
-
-[[server.base.groups]]
-check_connections_interval = '1min'
-connection_expiration_time = '1day'
-network_type = 'TCP'
-services = [ 'Core' ]
-use_independent_thread_pool = true
-
-[server.base.groups.io_worker]
-num_event_loop = 1
-rdma_connect_timeout = '5s'
-read_write_rdma_in_event_thread = false
-read_write_tcp_in_event_thread = false
-tcp_connect_timeout = '1s'
-wait_to_retry_send = '100ms'
-
-[server.base.groups.io_worker.connect_concurrency_limiter]
-max_concurrency = 4
-
-[server.base.groups.io_worker.ibsocket]
-buf_ack_batch = 8
-buf_signal_batch = 8
-buf_size = 16384
-drain_timeout = '5s'
-drop_connections = 0
-event_ack_batch = 128
-max_rd_atomic = 16
-max_rdma_wr = 128
-max_rdma_wr_per_post = 32
-max_sge = 16
-min_rnr_timer = 1
-record_bytes_per_peer = false
-record_latency_per_peer = false
-retry_cnt = 7
-rnr_retry = 0
-send_buf_cnt = 32
-sl = 0
-start_psn = 0
-timeout = 14
-
-[server.base.groups.io_worker.transport_pool]
-max_connections = 1
-
-[server.base.groups.listener]
-domain_socket_index = 1
-filter_list = []
-listen_port = 9000
-listen_queue_depth = 4096
-rdma_accept_timeout = '15s'
-rdma_listen_ethernet = true
-reuse_port = false
-
-[server.base.groups.processor]
-enable_coroutines_pool = true
-max_coroutines_num = 256
-max_processing_requests_num = 4096
-response_compression_level = 1
-response_compression_threshold = '128KB'
-
-[server.service]
-allow_heartbeat_from_unregistered = true
-authenticate = false
-bootstrapping_length = '2min'
-bump_routing_info_version_interval = '5s'
-check_status_interval = '10s'
-client_session_timeout = '20min'
-enable_routinginfo_cache = true
-extend_lease_check_release_version = true
-extend_lease_interval = '10s'
-heartbeat_fail_interval = '1min'
-heartbeat_ignore_stale_targets = true
-heartbeat_ignore_unknown_targets = false
-heartbeat_timestamp_valid_window = '30s'
-lease_length = '1min'
-new_chain_bootstrap_interval = '2min'
-only_accept_client_uuid = false
-retry_times_on_txn_errors = -1
-send_heartbeat = true
-send_heartbeat_interval = '10s'
-suspicious_lease_interval = '20s'
-target_info_load_interval = '1s'
-target_info_persist_batch = 1000
-target_info_persist_interval = '1s'
-try_adjust_target_order_as_preferred = false
-update_chains_interval = '1s'
-update_metrics_interval = '1s'
-validate_lease_on_write = true
-
-[server.service.retry_transaction]
-max_backoff = '1s'
-max_retry_count = 10
-
-[server.service.user_cache]
-buckets = 127
-exist_ttl = '5min'
-inexist_ttl = '10s'

asian_ai/chinese/3FS/configs/mgmtd_main_app.toml

@@ -1,2 +0,0 @@
-allow_empty_node_id = true
-node_id = 0

asian_ai/chinese/3FS/configs/mgmtd_main_launcher.toml

@@ -1,44 +0,0 @@
-allow_dev_version = true
-cluster_id = ''
-use_memkv = false
-
-[fdb]
-casual_read_risky = false
-clusterFile = ''
-default_backoff = 0
-enableMultipleClient = false
-externalClientDir = ''
-externalClientPath = ''
-multipleClientThreadNum = 4
-readonly = false
-trace_file = ''
-trace_format = 'json'
-
-[ib_devices]
-allow_no_usable_devices = false
-allow_unknown_zone = true
-default_network_zone = 'UNKNOWN'
-default_pkey_index = 0
-default_roce_pkey_index = 0
-default_traffic_class = 0
-device_filter = []
-fork_safe = true
-prefer_ibdevice = true
-skip_inactive_ports = true
-skip_unusable_device = true
-subnets = []
-
-[kv_engine]
-use_memkv = false
-
-[kv_engine.fdb]
-casual_read_risky = false
-clusterFile = ''
-default_backoff = 0
-enableMultipleClient = false
-externalClientDir = ''
-externalClientPath = ''
-multipleClientThreadNum = 4
-readonly = false
-trace_file = ''
-trace_format = 'json'

asian_ai/chinese/3FS/configs/monitor_collector_main.toml

@@ -1,141 +0,0 @@
-[common]
-cluster_id = ''
-
-[common.ib_devices]
-allow_unknown_zone = true
-default_network_zone = 'UNKNOWN'
-device_filter = []
-subnets = []
-
-[[common.log.categories]]
-categories = [ '.' ]
-handlers = [ 'normal', 'err', 'fatal' ]
-inherit = true
-level = 'INFO'
-propagate = 'NONE'
-
-[[common.log.handlers]]
-async = true
-file_path = '/var/log/3fs/monitor_collector_main.log'
-max_file_size = '100MB'
-max_files = 10
-name = 'normal'
-rotate = true
-rotate_on_open = false
-start_level = 'NONE'
-stream_type = 'STDERR'
-writer_type = 'FILE'
-
-[[common.log.handlers]]
-async = false
-file_path = '/var/log/3fs/monitor_collector_main-err.log'
-max_file_size = '100MB'
-max_files = 10
-name = 'err'
-rotate = true
-rotate_on_open = false
-start_level = 'ERR'
-stream_type = 'STDERR'
-writer_type = 'FILE'
-
-[[common.log.handlers]]
-async = false
-file_path = '/var/log/3fs/monitor_collector_main-fatal.log'
-max_file_size = '100MB'
-max_files = 10
-name = 'fatal'
-rotate = true
-rotate_on_open = false
-start_level = 'FATAL'
-stream_type = 'STDERR'
-writer_type = 'STREAM'
-
-[server.base.independent_thread_pool]
-bg_thread_pool_stratetry = 'SHARED_QUEUE'
-collect_stats = false
-enable_work_stealing = false
-io_thread_pool_stratetry = 'SHARED_QUEUE'
-num_bg_threads = 2
-num_connect_threads = 2
-num_io_threads = 2
-num_proc_threads = 2
-proc_thread_pool_stratetry = 'SHARED_QUEUE'
-
-[server.base.thread_pool]
-bg_thread_pool_stratetry = 'SHARED_QUEUE'
-collect_stats = false
-enable_work_stealing = false
-io_thread_pool_stratetry = 'SHARED_QUEUE'
-num_bg_threads = 2
-num_connect_threads = 2
-num_io_threads = 2
-num_proc_threads = 2
-proc_thread_pool_stratetry = 'SHARED_QUEUE'
-
-[[server.base.groups]]
-#default_timeout = '1s'
-#drop_connections_interval = '1h'
-network_type = 'TCP'
-services = [ 'MonitorCollector' ]
-use_independent_thread_pool = false
-
-[server.base.groups.io_worker]
-num_event_loop = 1
-rdma_connect_timeout = '5s'
-read_write_rdma_in_event_thread = false
-read_write_tcp_in_event_thread = false
-tcp_connect_timeout = '1s'
-wait_to_retry_send = '100ms'
-
-[server.base.groups.io_worker.ibsocket]
-buf_ack_batch = 8
-buf_signal_batch = 8
-buf_size = 16384
-drop_connections = 0
-event_ack_batch = 128
-#gid_index = 0
-max_rd_atomic = 16
-max_rdma_wr = 128
-max_rdma_wr_per_post = 32
-max_sge = 16
-min_rnr_timer = 1
-pkey_index = 0
-record_bytes_per_peer = false
-record_latency_per_peer = false
-retry_cnt = 7
-rnr_retry = 0
-send_buf_cnt = 32
-sl = 0
-start_psn = 0
-timeout = 14
-traffic_class = 0
-
-[server.base.groups.io_worker.transport_pool]
-max_connections = 1
-
-[server.base.groups.listener]
-filter_list = []
-listen_port = 10000
-listen_queue_depth = 4096
-rdma_listen_ethernet = true
-reuse_port = false
-
-[server.base.groups.processor]
-enable_coroutines_pool = true
-max_coroutines_num = 256
-max_processing_requests_num = 4096
-
-[server.monitor_collector]
-batch_commit_size = 4096
-conn_threads = 32
-queue_capacity = 204800
-
-[server.monitor_collector.reporter]
-type = 'clickhouse'
-
-[server.monitor_collector.reporter.clickhouse]
-db = ''
-host = ''
-passwd = ''
-port = ''
-user = ''

asian_ai/chinese/3FS/configs/storage_main.toml

@@ -1,510 +0,0 @@
-[[common.log.categories]]
-categories = [ '.' ]
-handlers = [ 'normal', 'err', 'fatal' ]
-inherit = true
-level = 'INFO'
-propagate = 'NONE'
-
-[[common.log.handlers]]
-async = true
-file_path = '/var/log/3fs/storage_main.log'
-max_file_size = '100MB'
-max_files = 10
-name = 'normal'
-rotate = true
-rotate_on_open = false
-start_level = 'NONE'
-stream_type = 'STDERR'
-writer_type = 'FILE'
-
-[[common.log.handlers]]
-async = false
-file_path = '/var/log/3fs/storage_main-err.log'
-max_file_size = '100MB'
-max_files = 10
-name = 'err'
-rotate = true
-rotate_on_open = false
-start_level = 'ERR'
-stream_type = 'STDERR'
-writer_type = 'FILE'
-
-[[common.log.handlers]]
-async = false
-file_path = '/var/log/3fs/storage_main-fatal.log'
-max_file_size = '100MB'
-max_files = 10
-name = 'fatal'
-rotate = true
-rotate_on_open = false
-start_level = 'FATAL'
-stream_type = 'STDERR'
-writer_type = 'STREAM'
-
-[common.memory]
-prof_active = false
-prof_prefix = ''
-
-[common.monitor]
-collect_period = '1s'
-num_collectors = 1
-
-[[common.monitor.reporters]]
-type = 'monitor_collector'
-
-[common.monitor.reporters.monitor_collector]
-remote_ip = ""
-
-[server]
-speed_up_quit = true
-use_coroutines_pool_read = true
-use_coroutines_pool_update = true
-
-[server.aio_read_worker]
-enable_io_uring = true
-inflight_control_offset = 128
-ioengine = 'libaio'
-max_events = 512
-min_complete = 128
-num_threads = 32
-queue_size = 4096
-wait_all_inflight = false
-
-[server.allocate_worker]
-max_remain_groups = 8
-max_remain_ultra_groups = 4
-max_reserved_chunks = '1GB'
-min_remain_groups = 4
-min_remain_ultra_groups = 0
-
-[server.base.independent_thread_pool]
-bg_thread_pool_stratetry = 'SHARED_QUEUE'
-collect_stats = false
-enable_work_stealing = false
-io_thread_pool_stratetry = 'SHARED_QUEUE'
-num_bg_threads = 2
-num_connect_threads = 2
-num_io_threads = 2
-num_proc_threads = 2
-proc_thread_pool_stratetry = 'SHARED_QUEUE'
-
-[server.base.thread_pool]
-bg_thread_pool_stratetry = 'SHARED_QUEUE'
-collect_stats = false
-enable_work_stealing = false
-io_thread_pool_stratetry = 'SHARED_QUEUE'
-num_bg_threads = 2
-num_connect_threads = 2
-num_io_threads = 32
-num_proc_threads = 32
-proc_thread_pool_stratetry = 'SHARED_QUEUE'
-
-[[server.base.groups]]
-check_connections_interval = '1min'
-connection_expiration_time = '1day'
-network_type = 'RDMA'
-services = [ 'StorageSerde' ]
-use_independent_thread_pool = false
-
-[server.base.groups.io_worker]
-num_event_loop = 1
-rdma_connect_timeout = '5s'
-read_write_rdma_in_event_thread = false
-read_write_tcp_in_event_thread = false
-tcp_connect_timeout = '1s'
-wait_to_retry_send = '100ms'
-
-[server.base.groups.io_worker.connect_concurrency_limiter]
-max_concurrency = 4
-
-[server.base.groups.io_worker.ibsocket]
-buf_ack_batch = 8
-buf_signal_batch = 8
-buf_size = 16384
-drain_timeout = '5s'
-drop_connections = 0
-event_ack_batch = 128
-max_rd_atomic = 16
-max_rdma_wr = 128
-max_rdma_wr_per_post = 32
-max_sge = 16
-min_rnr_timer = 1
-record_bytes_per_peer = false
-record_latency_per_peer = false
-retry_cnt = 7
-rnr_retry = 0
-send_buf_cnt = 32
-sl = 0
-start_psn = 0
-timeout = 14
-
-[server.base.groups.io_worker.transport_pool]
-max_connections = 1
-
-[server.base.groups.listener]
-custom_tcp_nic_prefix = ''
-domain_socket_index = 1
-filter_list = []
-listen_port = 8000
-listen_queue_depth = 4096
-rdma_accept_timeout = '15s'
-rdma_listen_ethernet = true
-reuse_port = false
-
-[server.base.groups.processor]
-enable_coroutines_pool = true
-max_coroutines_num = 256
-max_processing_requests_num = 4096
-response_compression_level = 1
-response_compression_threshold = '128KB'
-
-[[server.base.groups]]
-check_connections_interval = '1min'
-connection_expiration_time = '1day'
-network_type = 'TCP'
-services = [ 'Core' ]
-use_independent_thread_pool = true
-
-[server.base.groups.io_worker]
-num_event_loop = 1
-rdma_connect_timeout = '5s'
-read_write_rdma_in_event_thread = false
-read_write_tcp_in_event_thread = false
-tcp_connect_timeout = '1s'
-wait_to_retry_send = '100ms'
-
-[server.base.groups.io_worker.connect_concurrency_limiter]
-max_concurrency = 4
-
-[server.base.groups.io_worker.ibsocket]
-buf_ack_batch = 8
-buf_signal_batch = 8
-buf_size = 16384
-drain_timeout = '5s'
-drop_connections = 0
-event_ack_batch = 128
-max_rd_atomic = 16
-max_rdma_wr = 128
-max_rdma_wr_per_post = 32
-max_sge = 16
-min_rnr_timer = 1
-record_bytes_per_peer = false
-record_latency_per_peer = false
-retry_cnt = 7
-rnr_retry = 0
-send_buf_cnt = 32
-sl = 0
-start_psn = 0
-timeout = 14
-
-[server.base.groups.io_worker.transport_pool]
-max_connections = 1
-
-[server.base.groups.listener]
-domain_socket_index = 1
-filter_list = []
-listen_port = 9000
-listen_queue_depth = 4096
-rdma_accept_timeout = '15s'
-rdma_listen_ethernet = true
-reuse_port = false
-
-[server.base.groups.processor]
-enable_coroutines_pool = true
-max_coroutines_num = 256
-max_processing_requests_num = 4096
-response_compression_level = 1
-response_compression_threshold = '128KB'
-
-[server.buffer_pool]
-big_rdmabuf_count = 64
-big_rdmabuf_size = '64MB'
-rdmabuf_count = 1024
-rdmabuf_size = '4MB'
-
-[server.check_worker]
-disk_low_space_threshold = 0.95999999999999996
-disk_reject_create_chunk_threshold = 0.97999999999999998
-emergency_recycling_ratio = 0.94999999999999996
-update_target_size_interval = '10s'
-
-[server.client]
-default_compression_level = 0
-default_compression_threshold = '128KB'
-default_log_long_running_threshold = '0ns'
-default_report_metrics = false
-default_send_retry_times = 1
-default_timeout = '1s'
-enable_rdma_control = false
-force_use_tcp = false
-
-[server.client.io_worker]
-num_event_loop = 1
-rdma_connect_timeout = '5s'
-read_write_rdma_in_event_thread = false
-read_write_tcp_in_event_thread = false
-tcp_connect_timeout = '1s'
-wait_to_retry_send = '100ms'
-
-[server.client.io_worker.connect_concurrency_limiter]
-max_concurrency = 4
-
-[server.client.io_worker.ibsocket]
-buf_ack_batch = 8
-buf_signal_batch = 8
-buf_size = 16384
-drain_timeout = '5s'
-drop_connections = 0
-event_ack_batch = 128
-max_rd_atomic = 16
-max_rdma_wr = 128
-max_rdma_wr_per_post = 32
-max_sge = 16
-min_rnr_timer = 1
-record_bytes_per_peer = false
-record_latency_per_peer = false
-retry_cnt = 7
-rnr_retry = 0
-send_buf_cnt = 32
-sl = 0
-start_psn = 0
-timeout = 14
-
-[server.client.io_worker.transport_pool]
-max_connections = 1
-
-[server.client.processor]
-enable_coroutines_pool = true
-max_coroutines_num = 256
-max_processing_requests_num = 4096
-response_compression_level = 1
-response_compression_threshold = '128KB'
-
-[server.client.rdma_control]
-max_concurrent_transmission = 64
-
-[server.client.thread_pool]
-bg_thread_pool_stratetry = 'SHARED_QUEUE'
-collect_stats = false
-enable_work_stealing = false
-io_thread_pool_stratetry = 'SHARED_QUEUE'
-num_bg_threads = 2
-num_connect_threads = 2
-num_io_threads = 2
-num_proc_threads = 2
-proc_thread_pool_stratetry = 'SHARED_QUEUE'
-
-[server.coroutines_pool_default]
-coroutines_num = 64
-queue_size = 1024
-threads_num = 8
-
-[server.coroutines_pool_read]
-coroutines_num = 64
-queue_size = 1024
-threads_num = 8
-
-[server.coroutines_pool_sync]
-coroutines_num = 64
-queue_size = 1024
-threads_num = 8
-
-[server.coroutines_pool_update]
-coroutines_num = 64
-queue_size = 1024
-threads_num = 8
-
-[server.dump_worker]
-dump_interval = '1day'
-dump_root_path = ''
-high_cpu_usage_threshold = 100
-
-[server.forward_client]
-default_compression_level = 0
-default_compression_threshold = '128KB'
-default_log_long_running_threshold = '0ns'
-default_report_metrics = false
-default_send_retry_times = 1
-default_timeout = '1s'
-enable_rdma_control = false
-force_use_tcp = false
-
-[server.forward_client.io_worker]
-num_event_loop = 1
-rdma_connect_timeout = '5s'
-read_write_rdma_in_event_thread = false
-read_write_tcp_in_event_thread = false
-tcp_connect_timeout = '1s'
-wait_to_retry_send = '100ms'
-
-[server.forward_client.io_worker.connect_concurrency_limiter]
-max_concurrency = 4
-
-[server.forward_client.io_worker.ibsocket]
-buf_ack_batch = 8
-buf_signal_batch = 8
-buf_size = 16384
-drain_timeout = '5s'
-drop_connections = 0
-event_ack_batch = 128
-max_rd_atomic = 16
-max_rdma_wr = 128
-max_rdma_wr_per_post = 32
-max_sge = 16
-min_rnr_timer = 1
-record_bytes_per_peer = false
-record_latency_per_peer = false
-retry_cnt = 7
-rnr_retry = 0
-send_buf_cnt = 32
-sl = 0
-start_psn = 0
-timeout = 14
-
-[server.forward_client.io_worker.transport_pool]
-max_connections = 1
-
-[server.forward_client.processor]
-enable_coroutines_pool = true
-max_coroutines_num = 256
-max_processing_requests_num = 4096
-response_compression_level = 1
-response_compression_threshold = '128KB'
-
-[server.forward_client.rdma_control]
-max_concurrent_transmission = 64
-
-[server.forward_client.thread_pool]
-bg_thread_pool_stratetry = 'SHARED_QUEUE'
-collect_stats = false
-enable_work_stealing = false
-io_thread_pool_stratetry = 'SHARED_QUEUE'
-num_bg_threads = 2
-num_connect_threads = 2
-num_io_threads = 2
-num_proc_threads = 2
-proc_thread_pool_stratetry = 'SHARED_QUEUE'
-
-[server.mgmtd]
-accept_incomplete_routing_info_during_mgmtd_bootstrapping = true
-auto_extend_client_session_interval = '10s'
-auto_heartbeat_interval = '10s'
-auto_refresh_interval = '10s'
-enable_auto_extend_client_session = false
-enable_auto_heartbeat = true
-enable_auto_refresh = true
-mgmtd_server_addresses = []
-work_queue_size = 100
-
-[server.reliable_forwarding]
-max_inline_forward_bytes = '0'
-retry_first_wait = '100ms'
-retry_max_wait = '1s'
-retry_total_time = '1min'
-
-[server.reliable_update]
-clean_up_expired_clients = false
-expired_clients_timeout = '1h'
-
-[server.storage]
-apply_transmission_before_getting_semaphore = true
-batch_read_ignore_chain_version = false
-batch_read_job_split_size = 1024
-max_concurrent_rdma_reads = 256
-max_concurrent_rdma_writes = 256
-max_num_results_per_query = 100
-post_buffer_per_bytes = '64KB'
-rdma_transmission_req_timeout = '0ns'
-read_only = false
-
-[server.storage.event_trace_log]
-dump_interval = '30s'
-enabled = true
-max_num_writers = 1
-max_row_group_length = 100000
-trace_file_dir = '.'
-
-[server.storage.write_worker]
-bg_num_threads = 8
-num_threads = 32
-queue_size = 4096
-
-[server.sync_meta_kv_worker]
-sync_meta_kv_interval = '1min'
-
-[server.sync_worker]
-batch_size = 16
-full_sync_chains = []
-full_sync_level = 'NONE'
-num_channels = 1024
-num_threads = 16
-sync_start_timeout = '10s'
-
-[server.sync_worker.batch_concurrency_limiter]
-max_concurrency = 64
-
-[server.sync_worker.pool]
-coroutines_num = 64
-enable_work_stealing = false
-queue_size = 1024
-
-[server.targets]
-allow_disk_without_uuid = false
-collect_all_fds = true
-create_engine_path = true
-space_info_cache_timeout = '5s'
-target_num_per_path = 0
-target_paths = []
-
-[server.targets.storage_target]
-force_persist = true
-kv_path = ''
-migrate_kv_store = false
-mutex_num = 257
-point_query_strategy = 'NONE'
-
-[server.targets.storage_target.file_store]
-preopen_chunk_size_list = []
-
-[server.targets.storage_target.kv_store]
-create_if_missing = false
-integrate_leveldb_log = false
-leveldb_block_cache_size = '8GB'
-leveldb_iterator_fill_cache = true
-leveldb_shared_block_cache = true
-leveldb_sst_file_size = '16MB'
-leveldb_write_buffer_size = '16MB'
-rocksdb_avoid_flush_during_recovery = false
-rocksdb_avoid_flush_during_shutdown = false
-rocksdb_avoid_unnecessary_blocking_io = false
-rocksdb_block_cache_size = '8GB'
-rocksdb_block_size = '4KB'
-rocksdb_bloom_filter_bits_per_key = 10
-rocksdb_compression = 'kNoCompression'
-rocksdb_enable_bloom_filter = true
-rocksdb_enable_pipelined_write = false
-rocksdb_enable_prefix_transform = true
-rocksdb_keep_log_file_num = 10
-rocksdb_level0_file_num_compaction_trigger = 4
-rocksdb_lowest_used_cache_tier = 'kNonVolatileBlockTier'
-rocksdb_max_manifest_file_size = '64MB'
-rocksdb_num_levels = 7
-rocksdb_prepopulate_block_cache = 'kDisable'
-rocksdb_readahead_size = '2MB'
-rocksdb_shared_block_cache = true
-rocksdb_stats_dump_period = '2min'
-rocksdb_target_file_size_base = '64MB'
-rocksdb_target_file_size_multiplier = 1
-rocksdb_threads_num = 8
-rocksdb_unordered_write = false
-rocksdb_wal_recovery_mode = 'kTolerateCorruptedTailRecords'
-rocksdb_write_buffer_size = '16MB'
-sync_when_write = true
-type = 'LevelDB'
-
-[server.targets.storage_target.meta_store]
-allocate_size = '256MB'
-punch_hole_batch_size = 16
-recycle_batch_size = 256
-removed_chunk_expiration_time = '3day'
-removed_chunk_force_recycled_time = '1h'

asian_ai/chinese/3FS/configs/storage_main_app.toml

@@ -1,2 +0,0 @@
-allow_empty_node_id = true
-node_id = 0