app/cli/lib/context_shared.go

package lib

import (
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"io"
	"os"
	"plandex-cli/api"
	"strings"
	"sync"

	shared "plandex-shared"

	"github.com/sashabaranov/go-openai"
)

const ContextMapMaxClientConcurrency = 250

type filePathWithSize struct {
	Path string
	Size int64
}

type mapFileDetails struct {
	size                          int64
	tokens                        int
	shaVal                        string
	mapFilesSkippedAfterSizeLimit []string
	mapFilesTruncatedTooLarge     []filePathWithSize
	mapContent                    string
}

func getMapFileDetails(path string, size, mapSize int64) (mapFileDetails, error) {
	var isImage bool
	var totalMapSizeExceeded bool

	res := mapFileDetails{
		size:                          size,
		mapFilesSkippedAfterSizeLimit: []string{},
		mapFilesTruncatedTooLarge:     []filePathWithSize{},
	}

	if !shared.HasFileMapSupport(path) {
		if shared.IsImageFile(path) {
			isImage = true

			var err error
			res.tokens, err = readImageTokensForDefsOnly(path, size, openai.ImageURLDetailHigh, 8*1024)
			if err != nil {
				return mapFileDetails{}, fmt.Errorf("failed to read image tokens for %s: %v", path, err)
			}
		} else {
			res.tokens = shared.GetBytesToTokensEstimate(size)
		}
	} else {
		var truncated bool
		if size > shared.MaxContextMapSingleInputSize {
			size = shared.MaxContextMapSingleInputSize
			truncated = true
			res.tokens = shared.GetBytesToTokensEstimate(size)
		}

		// should go in either skip list *or* truncated list, not both
		if mapSize+size > shared.MaxContextMapTotalInputSize {
			totalMapSizeExceeded = true
			res.mapFilesSkippedAfterSizeLimit = append(res.mapFilesSkippedAfterSizeLimit, path)
			res.tokens = shared.GetBytesToTokensEstimate(size)
		} else if truncated {
			res.mapFilesTruncatedTooLarge = append(res.mapFilesTruncatedTooLarge, filePathWithSize{Path: path, Size: size})
		}
	}

	if totalMapSizeExceeded || !shared.HasFileMapSupport(path) || isImage {
		shaVal := sha256.Sum256([]byte(fmt.Sprintf("%d", res.tokens)))
		res.shaVal = hex.EncodeToString(shaVal[:])

		res.mapContent = ""
		res.size = 0
	} else {
		// partial read for the map
		contentRes, err := getMapFileContent(path)
		if err != nil {
			return mapFileDetails{}, fmt.Errorf("failed to read file %s: %v", path, err)
		}

		res.mapContent = contentRes.content
		res.shaVal = contentRes.shaVal

		if contentRes.truncated {
			res.mapFilesTruncatedTooLarge = append(res.mapFilesTruncatedTooLarge, filePathWithSize{Path: path, Size: shared.MaxContextMapSingleInputSize})
			res.size = shared.MaxContextMapSingleInputSize
			res.tokens = shared.GetBytesToTokensEstimate(shared.MaxContextMapSingleInputSize)
		} else {
			// do the actual token count if we didn't truncate
			res.tokens = shared.GetNumTokensEstimate(res.mapContent)
		}
	}

	return res, nil
}

type mapFileContent struct {
	mapData   []byte
	content   string
	shaVal    string
	truncated bool
}

func getMapFileContent(path string) (mapFileContent, error) {
	f, err := os.Open(path)
	if err != nil {
		return mapFileContent{}, err
	}
	defer f.Close()

	info, err := f.Stat()
	if err != nil {
		return mapFileContent{}, err
	}
	size := info.Size()

	limit := int64(shared.MaxContextMapSingleInputSize)
	truncated := size > limit

	limitReader := io.LimitReader(f, limit)
	bytes, err := io.ReadAll(limitReader)
	if err != nil {
		return mapFileContent{}, err
	}

	sum := sha256.Sum256(bytes)
	shaVal := hex.EncodeToString(sum[:])

	return mapFileContent{mapData: bytes, content: string(bytes), shaVal: shaVal, truncated: truncated}, nil
}

func processMapBatches(mapInputBatches []shared.FileMapInputs) (shared.FileMapBodies, error) {
	allMapBodies := shared.FileMapBodies{}

	var mapMu sync.Mutex
	errCh := make(chan error, len(mapInputBatches))

	for _, batch := range mapInputBatches {
		if len(batch) == 0 {
			errCh <- nil
			continue
		}

		go func(batch shared.FileMapInputs) {
			mapRes, apiErr := api.Client.GetFileMap(shared.GetFileMapRequest{
				MapInputs: batch,
			})
			if apiErr != nil {
				errCh <- fmt.Errorf("failed to get file map: %v", apiErr)
				return
			}
			mapMu.Lock()
			for path, bodies := range mapRes.MapBodies {
				allMapBodies[path] = bodies
			}
			mapMu.Unlock()
			errCh <- nil
		}(batch)
	}

	for i := 0; i < len(mapInputBatches); i++ {
		err := <-errCh
		if err != nil {
			return nil, err
		}
	}

	return allMapBodies, nil
}

func readImageTokensForDefsOnly(path string, size int64, detail openai.ImageURLDetail, headerBytes int64) (int, error) {
	file, err := os.Open(path)
	if err != nil {
		return 0, fmt.Errorf("failed to open file %s: %w", path, err)
	}
	defer file.Close()

	tokens, err := shared.GetImageTokensFromHeader(file, detail, headerBytes)
	if err != nil {
		tokens = shared.GetImageTokensEstimateFromBytes(size)
	}
	return tokens, nil
}

func printSkippedFilesMsg(
	filesSkippedTooLarge []filePathWithSize,
	filesSkippedAfterSizeLimit []string,
	mapFilesTruncatedTooLarge []filePathWithSize,
	mapFilesSkippedAfterSizeLimit []string,
) {
	fmt.Println()
	fmt.Println(getSkippedFilesMsg(filesSkippedTooLarge, filesSkippedAfterSizeLimit, mapFilesTruncatedTooLarge, mapFilesSkippedAfterSizeLimit))
}

func getSkippedFilesMsg(
	filesSkippedTooLarge []filePathWithSize,
	filesSkippedAfterSizeLimit []string,
	mapFilesTruncatedTooLarge []filePathWithSize,
	mapFilesSkippedAfterSizeLimit []string,
) string {
	var builder strings.Builder

	if len(filesSkippedTooLarge) > 0 {
		fmt.Fprintf(&builder, "ℹ️  These files were skipped because they're too large:\n")
		for i, file := range filesSkippedTooLarge {
			if i >= maxSkippedFileList {
				fmt.Fprintf(&builder, "  • and %d more\n", len(filesSkippedTooLarge)-maxSkippedFileList)
				break
			}
			fmt.Fprintf(&builder, "  • %s - %d MB\n", file.Path, file.Size/1024/1024)
		}
	}
	if len(mapFilesTruncatedTooLarge) > 0 {
		fmt.Fprintf(&builder, "ℹ️  These files were truncated because they're too large to map fully:\n")
		for i, file := range mapFilesTruncatedTooLarge {
			if i >= maxSkippedFileList {
				fmt.Fprintf(&builder, "  • and %d more\n", len(mapFilesTruncatedTooLarge)-maxSkippedFileList)
				break
			}
			if file.Size > 1024*1024 {
				fmt.Fprintf(&builder, "  • %s - %d MB\n", file.Path, file.Size/1024/1024)
			} else {
				fmt.Fprintf(&builder, "  • %s - %d KB\n", file.Path, file.Size/1024)
			}
		}
		if len(mapFilesTruncatedTooLarge) > 0 {
			fmt.Fprintf(&builder, "They will still be included in the map, but only the first %d KB will be mapped.\n", shared.MaxContextMapSingleInputSize/1024)
		}
	}
	if len(filesSkippedAfterSizeLimit) > 0 {
		fmt.Fprintf(&builder, "ℹ️  These files were skipped because the total size limit was exceeded:\n")
		for i, file := range filesSkippedAfterSizeLimit {
			if i >= maxSkippedFileList {
				fmt.Fprintf(&builder, "  • and %d more\n", len(filesSkippedAfterSizeLimit)-maxSkippedFileList)
				break
			}
			fmt.Fprintf(&builder, "  • %s\n", file)
		}
	}
	if len(mapFilesSkippedAfterSizeLimit) > 0 {
		fmt.Fprintf(&builder, "ℹ️  These files were skipped because the total map size limit was exceeded:\n")
		for i, file := range mapFilesSkippedAfterSizeLimit {
			if i >= maxSkippedFileList {
				fmt.Fprintf(&builder, "  • and %d more\n", len(mapFilesSkippedAfterSizeLimit)-maxSkippedFileList)
				break
			}
			fmt.Fprintf(&builder, "  • %s\n", file)
		}
		if len(mapFilesSkippedAfterSizeLimit) > 0 {
			fmt.Fprintf(&builder, "They will still be included in the map as paths in the project, but no maps will be generated for them.\n")
		}
	}
	return builder.String()
}