source/pkg/compactscan/scan.go

// Package compactscan adapts the existing whole-bundle scanner to the frozen
// compactmodel package contract without invoking any legacy classifier.
package compactscan

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"huggingface.co/turenlabs/Vigil/source/pkg/bundle"
	"huggingface.co/turenlabs/Vigil/source/pkg/compactmodel"
)

// Stats contains only aggregate package accounting; no member names or content
// are exposed to machine-readable scoring output.
type Stats struct {
	DiscoveredFiles int `json:"discovered_files"`
	ModeledFiles    int `json:"modeled_files"`
	BytesRead       int `json:"bytes_read"`
}

type member struct {
	rel  string
	info os.FileInfo
}

// Scan rejects incomplete or ambiguous filesystem trees, invokes bundle.Scan
// to enumerate the whole package, and materializes only members selected by
// the compact preprocessing contract.
func Scan(target string) (compactmodel.Package, Stats, error) {
	if target == "" {
		return compactmodel.Package{}, Stats{}, errors.New("compact scan target is required")
	}
	if hasTraversal(target) {
		return compactmodel.Package{}, Stats{}, errors.New("compact scan target contains traversal")
	}
	abs, err := filepath.Abs(target)
	if err != nil {
		return compactmodel.Package{}, Stats{}, fmt.Errorf("resolve compact scan target: %w", err)
	}
	targetInfo, err := os.Lstat(abs)
	if err != nil {
		return compactmodel.Package{}, Stats{}, fmt.Errorf("inspect compact scan target: %w", err)
	}
	if targetInfo.Mode()&os.ModeSymlink != 0 {
		return compactmodel.Package{}, Stats{}, errors.New("compact scan target must not be a symlink")
	}
	rootPath := abs
	if !targetInfo.IsDir() {
		if !targetInfo.Mode().IsRegular() || !strings.EqualFold(filepath.Base(abs), "SKILL.md") {
			return compactmodel.Package{}, Stats{}, errors.New("compact scan requires a package directory or its SKILL.md")
		}
		rootPath = filepath.Dir(abs)
	}
	rootPath, err = filepath.EvalSymlinks(rootPath)
	if err != nil {
		return compactmodel.Package{}, Stats{}, fmt.Errorf("resolve compact package root: %w", err)
	}
	if err := preflightPackage(rootPath); err != nil {
		return compactmodel.Package{}, Stats{}, fmt.Errorf("compact package preflight failed: %w", err)
	}

	scanned, err := bundle.Scan(rootPath)
	if err != nil {
		return compactmodel.Package{}, Stats{}, fmt.Errorf("whole-package scanner failed: %w", err)
	}
	if scanned.SkillMdPath == "" {
		return compactmodel.Package{}, Stats{}, errors.New("whole-package scanner found no SKILL.md")
	}
	if scanned.ParseErr != nil {
		return compactmodel.Package{}, Stats{}, errors.New("whole-package scanner could not parse SKILL.md")
	}
	if len(scanned.Notes) != 0 {
		return compactmodel.Package{}, Stats{}, errors.New("whole-package scanner reported an incomplete or ambiguous scan")
	}

	root, err := os.OpenRoot(rootPath)
	if err != nil {
		return compactmodel.Package{}, Stats{}, fmt.Errorf("open compact package root: %w", err)
	}
	defer root.Close()

	members := make([]member, 0, len(scanned.Files)+1)
	skillRel, err := filepath.Rel(rootPath, scanned.SkillMdPath)
	if err != nil {
		return compactmodel.Package{}, Stats{}, errors.New("SKILL.md is outside the package root")
	}
	members = append(members, member{rel: filepath.ToSlash(skillRel)})
	for _, file := range scanned.Files {
		if file == nil {
			return compactmodel.Package{}, Stats{}, errors.New("whole-package scanner returned an empty member")
		}
		if file.IsSymlink {
			return compactmodel.Package{}, Stats{}, errors.New("compact package contains a symlink")
		}
		members = append(members, member{rel: filepath.ToSlash(file.RelPath)})
	}

	lightweight := make([]compactmodel.File, len(members))
	seen := make(map[string]struct{}, len(members))
	for index := range members {
		rel := members[index].rel
		if rel == "" || rel == "." || strings.Contains(rel, "\\") || hasTraversal(rel) || filepath.IsAbs(rel) {
			return compactmodel.Package{}, Stats{}, errors.New("whole-package scanner returned a non-canonical member")
		}
		if _, exists := seen[rel]; exists {
			return compactmodel.Package{}, Stats{}, errors.New("whole-package scanner returned a duplicate member")
		}
		seen[rel] = struct{}{}
		info, err := root.Lstat(filepath.FromSlash(rel))
		if err != nil {
			return compactmodel.Package{}, Stats{}, errors.New("package member changed after whole-package scan")
		}
		if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
			return compactmodel.Package{}, Stats{}, errors.New("compact package member is not an unlinked regular file")
		}
		members[index].info = info
		lightweight[index] = compactmodel.File{Path: rel, Executable: info.Mode().Perm()&0o111 != 0}
	}

	// Check every discovered member for hard-link ambiguity, including members
	// beyond the preprocessing cap.
	for _, item := range members {
		file, err := root.Open(filepath.FromSlash(item.rel))
		if err != nil {
			return compactmodel.Package{}, Stats{}, errors.New("open package member after whole-package scan")
		}
		opened, statErr := file.Stat()
		if statErr != nil {
			file.Close()
			return compactmodel.Package{}, Stats{}, errors.New("stat package member after whole-package scan")
		}
		links, linkErr := openedHardLinkCount(file, opened)
		closeErr := file.Close()
		if linkErr != nil || closeErr != nil || links != 1 || !os.SameFile(item.info, opened) {
			return compactmodel.Package{}, Stats{}, errors.New("compact package contains a changed or hard-linked member")
		}
	}

	selected, err := compactmodel.SelectFiles(lightweight)
	if err != nil {
		return compactmodel.Package{}, Stats{}, err
	}
	byPath := make(map[string]member, len(members))
	for _, item := range members {
		byPath[item.rel] = item
	}
	result := compactmodel.Package{Files: make([]compactmodel.File, 0, len(selected))}
	for _, selectedFile := range selected {
		item := byPath[selectedFile.Path]
		file, err := root.Open(filepath.FromSlash(item.rel))
		if err != nil {
			return compactmodel.Package{}, Stats{}, errors.New("open selected compact package member")
		}
		opened, err := file.Stat()
		if err != nil || !opened.Mode().IsRegular() || !os.SameFile(item.info, opened) {
			file.Close()
			return compactmodel.Package{}, Stats{}, errors.New("selected package member changed while opening")
		}
		links, err := openedHardLinkCount(file, opened)
		if err != nil || links != 1 {
			file.Close()
			return compactmodel.Package{}, Stats{}, errors.New("selected package member has ambiguous links")
		}
		content, err := sampleFileContent(file, opened.Size())
		if err != nil {
			file.Close()
			return compactmodel.Package{}, Stats{}, err
		}
		final, statErr := file.Stat()
		if statErr != nil {
			file.Close()
			return compactmodel.Package{}, Stats{}, errors.New("stat selected package member after reading")
		}
		links, linkErr := openedHardLinkCount(file, final)
		closeErr := file.Close()
		if linkErr != nil || closeErr != nil || links != 1 || final.Size() != opened.Size() || !os.SameFile(opened, final) {
			return compactmodel.Package{}, Stats{}, errors.New("selected package member changed while reading")
		}
		selectedFile.Content = content
		selectedFile.SizeBytes = opened.Size()
		result.Files = append(result.Files, selectedFile)
	}
	document, err := compactmodel.Serialize(result)
	if err != nil {
		return compactmodel.Package{}, Stats{}, err
	}
	return result, Stats{
		DiscoveredFiles: len(members),
		ModeledFiles:    document.FileCount,
		BytesRead:       document.BytesRead,
	}, nil
}

func sampleFileContent(file *os.File, size int64) ([]byte, error) {
	if size < 0 {
		return nil, errors.New("negative package member size")
	}
	if size <= compactmodel.MaxFileBytes {
		data := make([]byte, int(size))
		if _, err := io.ReadFull(file, data); err != nil {
			return nil, errors.New("read complete package member")
		}
		return data, nil
	}
	half := compactmodel.MaxFileBytes / 2
	data := make([]byte, compactmodel.MaxFileBytes)
	if _, err := file.ReadAt(data[:half], 0); err != nil {
		return nil, errors.New("read package member head")
	}
	if _, err := file.ReadAt(data[half:], size-int64(half)); err != nil {
		return nil, errors.New("read package member tail")
	}
	return data, nil
}

func hasTraversal(value string) bool {
	value = strings.ReplaceAll(value, "\\", "/")
	for _, part := range strings.Split(value, "/") {
		if part == ".." {
			return true
		}
	}
	return false
}