File size: 7,914 Bytes
f8b5d42 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 | import React, { useEffect, useState } from "react";
import Sidebar from "@/components/SettingsSidebar";
import { isMobile } from "react-device-detect";
import PreLoader from "@/components/Preloader";
import CTAButton from "@/components/lib/CTAButton";
import Admin from "@/models/admin";
import showToast from "@/utils/toast";
import { numberWithCommas } from "@/utils/numbers";
import { useTranslation } from "react-i18next";
import { useModal } from "@/hooks/useModal";
import ModalWrapper from "@/components/ModalWrapper";
import ChangeWarningModal from "@/components/ChangeWarning";
function isNullOrNaN(value) {
if (value === null) return true;
return isNaN(value);
}
export default function EmbeddingTextSplitterPreference() {
const [settings, setSettings] = useState({});
const [loading, setLoading] = useState(true);
const [saving, setSaving] = useState(false);
const [hasChanges, setHasChanges] = useState(false);
const { isOpen, openModal, closeModal } = useModal();
const { t } = useTranslation();
const handleSubmit = async (e) => {
e.preventDefault();
const form = new FormData(e.target);
if (
Number(form.get("text_splitter_chunk_overlap")) >=
Number(form.get("text_splitter_chunk_size"))
) {
showToast(
"Chunk overlap cannot be larger or equal to chunk size.",
"error"
);
return;
}
openModal();
};
const handleSaveSettings = async () => {
setSaving(true);
try {
const form = new FormData(
document.getElementById("text-splitter-chunking-form")
);
await Admin.updateSystemPreferences({
text_splitter_chunk_size: isNullOrNaN(
form.get("text_splitter_chunk_size")
)
? 1000
: Number(form.get("text_splitter_chunk_size")),
text_splitter_chunk_overlap: isNullOrNaN(
form.get("text_splitter_chunk_overlap")
)
? 1000
: Number(form.get("text_splitter_chunk_overlap")),
});
setHasChanges(false);
closeModal();
showToast("Text chunking strategy settings saved.", "success");
} catch (error) {
showToast("Failed to save text chunking strategy settings.", "error");
} finally {
setSaving(false);
}
};
useEffect(() => {
async function fetchSettings() {
const _settings = (await Admin.systemPreferences())?.settings;
setSettings(_settings ?? {});
setLoading(false);
}
fetchSettings();
}, []);
return (
<div className="w-screen h-screen overflow-hidden bg-theme-bg-container flex">
<Sidebar />
{loading ? (
<div
style={{ height: isMobile ? "100%" : "calc(100% - 32px)" }}
className="relative md:ml-[2px] md:mr-[16px] md:my-[16px] md:rounded-[16px] bg-theme-bg-secondary w-full h-full overflow-y-scroll p-4 md:p-0"
>
<div className="w-full h-full flex justify-center items-center">
<PreLoader />
</div>
</div>
) : (
<div
style={{ height: isMobile ? "100%" : "calc(100% - 32px)" }}
className="relative md:ml-[2px] md:mr-[16px] md:my-[16px] md:rounded-[16px] bg-theme-bg-secondary w-full h-full overflow-y-scroll p-4 md:p-0"
>
<form
onSubmit={handleSubmit}
onChange={() => setHasChanges(true)}
className="flex w-full"
id="text-splitter-chunking-form"
>
<div className="flex flex-col w-full px-1 md:pl-6 md:pr-[50px] md:py-6 py-16">
<div className="w-full flex flex-col gap-y-1 pb-4 border-white light:border-theme-sidebar-border border-b-2 border-opacity-10">
<div className="flex gap-x-4 items-center">
<p className="text-lg leading-6 font-bold text-white">
{t("text.title")}
</p>
</div>
<p className="text-xs leading-[18px] font-base text-white text-opacity-60">
{t("text.desc-start")} <br />
{t("text.desc-end")}
</p>
</div>
<div className="w-full justify-end flex">
{hasChanges && (
<CTAButton className="mt-3 mr-0 -mb-14 z-10">
{saving ? t("common.saving") : t("common.save")}
</CTAButton>
)}
</div>
<div className="flex flex-col gap-y-4 mt-8">
<div className="flex flex-col max-w-[300px]">
<div className="flex flex-col gap-y-2 mb-4">
<label className="text-white text-sm font-semibold block">
{t("text.size.title")}
</label>
<p className="text-xs text-white/60">
{t("text.size.description")}
</p>
</div>
<input
type="number"
name="text_splitter_chunk_size"
min={1}
max={settings?.max_embed_chunk_size || 1000}
onWheel={(e) => e?.currentTarget?.blur()}
className="border-none bg-theme-settings-input-bg text-white placeholder:text-theme-settings-input-placeholder text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
placeholder="maximum length of vectorized text"
defaultValue={
isNullOrNaN(settings?.text_splitter_chunk_size)
? 1000
: Number(settings?.text_splitter_chunk_size)
}
required={true}
autoComplete="off"
/>
<p className="text-xs text-white/40 mt-2">
{t("text.size.recommend")}{" "}
{numberWithCommas(settings?.max_embed_chunk_size || 1000)}.
</p>
</div>
</div>
<div className="flex flex-col gap-y-4 mt-8">
<div className="flex flex-col max-w-[300px]">
<div className="flex flex-col gap-y-2 mb-4">
<label className="text-white text-sm font-semibold block">
{t("text.overlap.title")}
</label>
<p className="text-xs text-white/60">
{t("text.overlap.description")}
</p>
</div>
<input
type="number"
name="text_splitter_chunk_overlap"
min={0}
onWheel={(e) => e?.currentTarget?.blur()}
className="border-none bg-theme-settings-input-bg text-white placeholder:text-theme-settings-input-placeholder text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
placeholder="maximum length of vectorized text"
defaultValue={
isNullOrNaN(settings?.text_splitter_chunk_overlap)
? 20
: Number(settings?.text_splitter_chunk_overlap)
}
required={true}
autoComplete="off"
/>
</div>
</div>
</div>
</form>
</div>
)}
<ModalWrapper isOpen={isOpen}>
<ChangeWarningModal
warningText="Changing text splitter settings will clear any previously cached documents.\n\nThese new settings will be applied to all documents when embedding them into a workspace."
onClose={closeModal}
onConfirm={handleSaveSettings}
/>
</ModalWrapper>
</div>
);
}
|