Update indeksator.cmd

indeksator.cmd

@@ -1,45 +1,35 @@
 :; # Indeksator Prosty RAG v0.1 - Jerzy Głowacki na licencji Apache 2.0
 :; # *NIX:
+:; OS=$(uname -s | sed -e 's/^Linux$/linux/' -e 's/^Darwin$/mac/')
 :; embedfile="bge-m3.embedfile"
 :; inputDir="baza"
 :; chunksFile="chunks.txt"
 :; dbFile="prosty-rag.db"
 :; chunkWords=200
 :; overlapWords=10
-:; buf=()
-:; for ((i = 0; i < overlapWords; i++)); do buf[i]=""; done
 :; > $chunksFile
 :; [ ! -d $inputDir ] && echo Pobieranie przykładowego pliku $inputDir/wikipedia.txt... && curl --create-dirs -Lo $inputDir/wikipedia.txt https://huggingface.co/jglowa/prosty-rag/resolve/main/baza/wikipedia.txt?download=true && echo Gotowe!
 :; [ ! -f $embedfile ] && echo Pobieranie $embedfile... && curl -Lo $embedfile https://huggingface.co/asg017/embedfile/resolve/refs%2Fpr%2F2/bge-m3.embedfile?download=true && chmod +x $embedfile && echo Gotowe!
-:; [ ! -f pdftotext ] && echo Pobieranie pdftotext... && curl -LO https://dl.xpdfreader.com/xpdf-tools-linux-4.05.tar.gz && tar --strip-components 2 -xzf xpdf-tools-linux-4.05.tar.gz xpdf-tools-linux-4.05/bin64/pdftotext && del xpdf-tools-linux-4.05.tar.gz && echo Gotowe!
+:; [ ! -f pdftotext ] && echo Pobieranie pdftotext... && curl -LO https://dl.xpdfreader.com/xpdf-tools-$OS-4.05.tar.gz && tar --strip-components 2 -xzf xpdf-tools-$OS-4.05.tar.gz xpdf-tools-$OS-4.05/bin64/pdftotext && del xpdf-tools-$OS-4.05.tar.gz && echo Gotowe!
 :; echo "Indeksowanie plików PDF/TXT/MD w folderze $inputDir..."
 :; shopt -s nullglob
 :; for pdf in $inputDir/*.pdf; do echo "Konwertowanie $(basename "$pdf")..." && pdftotext -nopgbrk -enc UTF-8 "$pdf"; done
 :; for file in $inputDir/*.txt $inputDir/*.md; do
-:;   echo "Przetwarzanie $(basename "$file")..."
-:;   wordCount=0
-:;   while IFS= read -r line || [ -n "$line" ]; do
-:;     for word in $line; do
-:;       for ((i = 0; i < overlapWords - 1; i++)); do buf[i]="${buf[i+1]}"; done
-:;       buf[overlapWords-1]="$word"
-:;       ((wordCount++))
-:;       if (( wordCount >= chunkWords )); then
-:;         {
-:;           printf "%s: " $(basename "$file")
-:;           for ((i = 0; i < overlapWords; i++)); do printf "%s " "${buf[i]}"; done
-:;           printf "\n"
-:;         } >> $chunksFile
-:;         wordCount=0
-:;       fi
-:;     done
-:;   done < "$file"
-:;   echo >> $chunksFile
+:;   filename=$(basename "$file")
+:;   echo "Przetwarzanie $filename..."
+:;   mapfile -t words < <(tr -s '[:space:]' '\n' < "$file" | grep -v '^$')
+:;   totalWords=${#words[@]}
+:;   start=0
+:;   while ((start < totalWords)); do
+:;     chunk=("${words[@]:start:chunkWords}")
+:;     echo "$filename: ${chunk[*]}" >> $chunksFile
+:;     ((start += chunkWords - overlapWords))
+:;   done
 :; done
 :; echo "Osadzanie plików..."
 :; [ -f $dbFile ] && rm $dbFile
 :; ./$embedfile import $chunksFile $dbFile && echo "Gotowe! Po każdej zmianie w folderze $inputDir należy uruchomić ponownie indeksator."
-:; # rm $chunksFile
-:; exit $?
+:; rm $chunksFile; exit $?
 :; # Windows:
 @echo off
 setlocal enabledelayedexpansion