Update indeksator.cmd

indeksator.cmd

@@ -1,43 +1,83 @@
-:; # Indeksator Prosty RAG v0.1 - Jerzy Głowacki na licencji Apache 2.0
-@echo off
-setlocal enabledelayedexpansion
-set embedfile=bge-m3.embedfile
-set inputDir=baza
-set chunksFile=chunks.txt
-set dbFile=prosty-rag.db
-set chunkWords=200
-set overlapWords=10
-for /l %%i in (1,1,%overlapWords%) do set buf[%%i]=
-break>%chunksFile%
-if not exist %inputDir% echo Pobieranie przykładowego pliku %inputDir%\wikipedia.txt... && curl --create-dirs -Lo %inputDir%\wikipedia.txt https://huggingface.co/jglowa/prosty-rag/resolve/main/baza/wikipedia.txt?download=true  && echo Gotowe!
-if not exist %embedfile% echo Pobieranie %embedfile%... && curl -Lo %embedfile% https://huggingface.co/jglowa/prosty-rag/resolve/main/bge-m3.embedfile?download=true  && echo Gotowe!
-if not exist pdftotext.exe echo Pobieranie pdftotext.exe... && curl -LO https://dl.xpdfreader.com/xpdf-tools-win-4.05.zip && tar --strip-components 2 -xf xpdf-tools-win-4.05.zip xpdf-tools-win-4.05/bin64/pdftotext.exe && del xpdf-tools-win-4.05.zip && echo Gotowe!
-echo Indeksowanie plików PDF/TXT/MD w folderze %inputDir%...
-for %%F in ("%inputDir%\*.pdf") do if not exist "%%~dpnF.txt" echo Konwertowanie %%~nxF... && pdftotext -nopgbrk -enc UTF-8 "%%~F"
-for %%F in ("%inputDir%\*.txt" "%inputDir%\*.md") do (
-  echo Przetwarzanie %%~nxF...
-  (
-    set wordCount=0
-    set /p =%%~nxF: <nul
-    for /f "usebackq delims=" %%L in ("%%F") do (
-      for %%W in (%%L) do (
-        set /p =%%W <nul
-        for /l %%i in (2,1,%overlapWords%) do (set /a j=%%i-1 && set buf[!j!]=!buf[%%i]!)
-        set "buf[%overlapWords%]=%%W"
-        set /a wordCount+=1
-        if !wordCount! geq !chunkWords! (
-          echo.
-          set /p =%%~nxF: <nul
-          for /l %%i in (1,1,%overlapWords%) do set /p =!buf[%%i]! <nul
-          set /a wordCount=0
-        )
-      )
-    )
-    echo.
-  )>>%chunksFile%
-)
-echo Osadzanie plików...
-if exist %dbFile% del %dbFile%
-%embedfile% import %chunksFile% %dbFile% && echo Gotowe! Po każdej zmianie w folderze %inputDir% należy uruchomić ponownie indeksator.
-del %chunksFile%
-endlocal
+:; # Indeksator Prosty RAG v0.1 - Jerzy Głowacki na licencji Apache 2.0
+:; # *NIX:
+:; embedfile="bge-m3.embedfile"
+:; inputDir="baza"
+:; chunksFile="chunks.txt"
+:; dbFile="prosty-rag.db"
+:; chunkWords=200
+:; overlapWords=10
+:; declare -a buf
+:; for ((i = 0; i < overlapWords; i++)); do buf[i]=""; done
+:; > $chunksFile
+:; [ ! -f $inputDir ] && echo Pobieranie przykładowego pliku $inputDir/wikipedia.txt... && curl --create-dirs -Lo $inputDir/wikipedia.txt https://huggingface.co/jglowa/prosty-rag/resolve/main/baza/wikipedia.txt?download=true && echo Gotowe!
+:; [ ! -f $embedfile ] && echo Pobieranie $embedfile... && curl -Lo $embedfile https://huggingface.co/jglowa/prosty-rag/resolve/main/bge-m3.llamafile?download=true && chmod +x $embedfile && echo Gotowe!
+:; [ ! -f pdftotext ] && echo Pobieranie pdftotext... && curl -LO https://dl.xpdfreader.com/xpdf-tools-linux-4.05.tar.gz && tar --strip-components 2 -xzf xpdf-tools-linux-4.05.tar.gz xpdf-tools-linux-4.05/bin64/pdftotext && del xpdf-tools-linux-4.05.tar.gz && echo Gotowe!
+:; echo "Indeksowanie plików PDF/TXT/MD w folderze $inputDir..."
+:; for pdf in $inputDir/*.pdf; do echo "Konwertowanie $(basename "$pdf")..." && pdftotext -nopgbrk -enc UTF-8 "$pdf"; done
+:; for file in $inputDir/*.txt $inputDir/*.md; do
+:;   echo "Przetwarzanie $(basename "$file")..."
+:;   wordCount=0
+:;   while IFS= read -r line || [ -n "$line" ]; do
+:;     for word in $line; do
+:;       for ((i = 0; i < overlapWords - 1; i++)); do buf[i]="${buf[i+1]}"; done
+:;       buf[overlapWords-1]="$word"
+:;       ((wordCount++))
+:;       if (( wordCount >= chunkWords )); then
+:;         {
+:;           printf "%s: " $(basename "$file")
+:;           for ((i = 0; i < overlapWords; i++)); do printf "%s " "${buf[i]}"; done
+:;           printf "\n"
+:;         } >> $chunksFile
+:;         wordCount=0
+:;       fi
+:;     done
+:;   done < "$file"
+:;   echo >> $chunksFile
+:; done
+:; echo "Osadzanie plików..."
+:; [ -f $dbFile ] && rm $dbFile
+:; ./$embedfile import $chunksFile $dbFile && echo "Gotowe! Po każdej zmianie w folderze $inputDir należy uruchomić ponownie indeksator."
+:; rm "$chunksFile"; exit $?
+:; # Windows:
+@echo off
+setlocal enabledelayedexpansion
+set embedfile=bge-m3.embedfile
+set inputDir=baza
+set chunksFile=chunks.txt
+set dbFile=prosty-rag.db
+set chunkWords=200
+set overlapWords=10
+for /l %%i in (1,1,%overlapWords%) do set buf[%%i]=
+break>%chunksFile%
+if not exist %inputDir% echo Pobieranie przykładowego pliku %inputDir%\wikipedia.txt... && curl --create-dirs -Lo %inputDir%\wikipedia.txt https://huggingface.co/jglowa/prosty-rag/resolve/main/baza/wikipedia.txt?download=true  && echo Gotowe^!
+if not exist %embedfile% echo Pobieranie %embedfile%... && curl -Lo %embedfile% https://huggingface.co/jglowa/prosty-rag/resolve/main/bge-m3.embedfile?download=true  && echo Gotowe^!
+if not exist pdftotext.exe echo Pobieranie pdftotext.exe... && curl -LO https://dl.xpdfreader.com/xpdf-tools-win-4.05.zip && tar --strip-components 2 -xf xpdf-tools-win-4.05.zip xpdf-tools-win-4.05/bin64/pdftotext.exe && del xpdf-tools-win-4.05.zip && echo Gotowe^!
+echo Indeksowanie plików PDF/TXT/MD w folderze %inputDir%...
+for %%F in ("%inputDir%\*.pdf") do if not exist "%%~dpnF.txt" echo Konwertowanie %%~nxF... && pdftotext -nopgbrk -enc UTF-8 "%%~F"
+for %%F in ("%inputDir%\*.txt" "%inputDir%\*.md") do (
+  echo Przetwarzanie %%~nxF...
+  (
+    set wordCount=0
+    set /p =%%~nxF: <nul
+    for /f "usebackq delims=" %%L in ("%%F") do (
+      for %%W in (%%L) do (
+        set /p =%%W <nul
+        for /l %%i in (2,1,%overlapWords%) do (set /a j=%%i-1 && set buf[!j!]=!buf[%%i]!)
+        set "buf[%overlapWords%]=%%W"
+        set /a wordCount+=1
+        if !wordCount! geq !chunkWords! (
+          echo.
+          set /p =%%~nxF: <nul
+          for /l %%i in (1,1,%overlapWords%) do set /p =!buf[%%i]! <nul
+          set /a wordCount=0
+        )
+      )
+    )
+    echo.
+  )>>%chunksFile%
+)
+echo Osadzanie plików...
+if exist %dbFile% del %dbFile%
+%embedfile% import %chunksFile% %dbFile% && echo Gotowe^! Po każdej zmianie w folderze %inputDir% należy uruchomić ponownie indeksator.
+del %chunksFile%
+endlocal