source/pkg/bundle/scan.go

package bundle

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"huggingface.co/turenlabs/Vigil/source/pkg/parser"
)

const (
	// maxSniffBytes is the per-file read cap (1 MiB). Large enough that ordinary
	// scripts read fully (closing the front-padding tail-hiding hole); anything
	// larger sets File.Truncated so analyzers can FLAG ON TRUNCATE rather than
	// silently scanning only the prefix.
	maxSniffBytes = 1048576

	// maxBundleFiles bounds the number of members walked, guarding against a
	// pathological directory (or a symlink loop that the inode set somehow misses)
	// turning the scan into an unbounded crawl.
	maxBundleFiles = 4096
)

// skillMdName is the canonical entry-point filename (matched case-insensitively).
const skillMdName = "skill.md"

// IsBundlePath reports whether path should be scanned as a bundle: true if path
// is a directory, or a SKILL.md (case-insensitive) file inside a directory.
// A plain .md file that is not named SKILL.md is NOT a bundle (preserving
// byte-for-byte single-file behavior).
func IsBundlePath(path string) bool {
	info, err := os.Stat(path)
	if err != nil {
		return false
	}
	if info.IsDir() {
		return true
	}
	return strings.EqualFold(filepath.Base(path), skillMdName)
}

// ScanPath is the backward-compatible entry point.
//
//   - A directory                  -> full Bundle of that directory.
//   - A SKILL.md path              -> full Bundle of its parent directory.
//   - Any other regular file (.md) -> degenerate single-file Bundle with no
//     siblings, so legacy single-file scanning is unchanged.
func ScanPath(path string) (*Bundle, error) {
	info, err := os.Stat(path)
	if err != nil {
		return nil, fmt.Errorf("stat %s: %w", path, err)
	}

	if info.IsDir() {
		return Scan(path)
	}

	if strings.EqualFold(filepath.Base(path), skillMdName) {
		return Scan(filepath.Dir(path))
	}

	// Degenerate single-file bundle: parse only this file, no siblings.
	return scanSingleFile(path)
}

// scanSingleFile builds a degenerate Bundle from one markdown file (legacy
// single-file mode). No directory walk, no siblings.
func scanSingleFile(path string) (*Bundle, error) {
	abs, err := filepath.Abs(path)
	if err != nil {
		abs = path
	}
	b := &Bundle{
		Dir:         filepath.Dir(abs),
		SkillMdPath: abs,
		Files:       nil,
	}
	skill, perr := parser.Parse(path)
	if perr != nil {
		b.ParseErr = perr
	} else {
		b.Skill = skill
		b.AllowedTools = parseAllowedTools(skill.RawContent)
	}
	if info, serr := os.Stat(path); serr == nil {
		b.SkillMdBytes = info.Size()
	}
	return b, nil
}

// Scan walks the WHOLE skill directory (hidden/dot files included), locates
// SKILL.md case-insensitively, parses it via parser.Parse (unchanged), and
// populates Bundle.Files with magic-sniffed kinds, capped sniff prefixes,
// truncation/newline-ratio flags, and resolved symlink targets. It never panics
// on unreadable or looping entries — such cases append a Note and continue.
func Scan(dir string) (*Bundle, error) {
	absDir, err := filepath.Abs(dir)
	if err != nil {
		absDir = dir
	}

	b := &Bundle{Dir: absDir}

	// Locate SKILL.md (case-insensitive) at the top level of the directory.
	skillMdAbs := findSkillMd(absDir)
	b.SkillMdPath = skillMdAbs

	visited := make(map[uint64]bool) // inode set for symlink-loop safety
	fileCount := 0

	walkErr := filepath.WalkDir(absDir, func(path string, d os.DirEntry, walkErr error) error {
		if walkErr != nil {
			// Record and skip an unreadable entry; never abort the whole walk.
			b.Notes = append(b.Notes, fmt.Sprintf("walk error at %s: %v", relOrPath(absDir, path), walkErr))
			if d != nil && d.IsDir() {
				return filepath.SkipDir
			}
			return nil
		}

		if path == absDir {
			return nil // the root directory itself
		}

		if fileCount >= maxBundleFiles {
			b.Notes = append(b.Notes, fmt.Sprintf("file cap %d reached; remaining entries skipped", maxBundleFiles))
			return filepath.SkipAll
		}

		rel := relOrPath(absDir, path)

		if d.IsDir() {
			// Loop-safety for directory symlinks is handled below in the symlink
			// branch; a plain directory is just descended into.
			return nil
		}

		// SKILL.md itself is held on the Bundle, not in Files.
		if skillMdAbs != "" && sameAbs(path, skillMdAbs) {
			return nil
		}

		f := &File{
			RelPath: rel,
			AbsPath: path,
			Hidden:  pathHasHiddenSegment(rel),
		}

		// Detect symlinks via the entry type (WalkDir does NOT follow them, so a
		// symlinked directory arrives here as a non-dir entry — we never crawl
		// into it, only resolve and record its target).
		isSymlink := d.Type()&os.ModeSymlink != 0
		f.IsSymlink = isSymlink

		// readPath is the path we sniff bytes from: the resolved target for a
		// symlink (so a symlinked executable reference is first-class), else the
		// path itself.
		readPath := path

		if isSymlink {
			target, escapes, isLoop := resolveSymlink(path, absDir)
			f.SymlinkTarget = target
			f.SymlinkEscapes = escapes
			if isLoop {
				b.Notes = append(b.Notes, fmt.Sprintf("symlink loop broken at %s", rel))
				// Still record the entry (kind unknown) so a looping symlinked
				// reference is never silently dropped.
				b.Files = append(b.Files, f)
				fileCount++
				return nil
			}
			if target != "" {
				readPath = target
			}
			// Inode-dedupe the resolved target so two symlinks to the same file,
			// or a symlink pointing back at a real sibling, do not double-count and
			// cannot drive an unbounded loop.
			if ino, ok := inodeOf(readPath); ok {
				if visited[ino] {
					b.Notes = append(b.Notes, fmt.Sprintf("symlink target already visited at %s", rel))
				} else {
					visited[ino] = true
				}
			}
		} else if ino, ok := inodeOf(path); ok {
			visited[ino] = true
		}

		// Size from the resolved target (Lstat-size of a symlink is just the link
		// length, which is useless for payload accounting).
		if info, serr := os.Stat(readPath); serr == nil {
			f.SizeBytes = info.Size()
		}

		sniff, _, truncated, newlineRatio, rerr := readSniff(readPath)
		if rerr != nil {
			b.Notes = append(b.Notes, fmt.Sprintf("read error at %s: %v", rel, rerr))
		}
		f.Sniff = sniff
		f.Truncated = truncated
		f.NewlineRatio = newlineRatio

		f.Kind = classifyKind(filepath.Base(path), sniff)
		f.ScriptLang = scriptLangFor(filepath.Base(path), sniff)

		b.Files = append(b.Files, f)
		fileCount++
		b.PayloadBytes += f.SizeBytes
		return nil
	})
	if walkErr != nil {
		b.Notes = append(b.Notes, fmt.Sprintf("walk terminated: %v", walkErr))
	}

	// Parse SKILL.md via the unchanged parser.
	if skillMdAbs != "" {
		skill, perr := parser.Parse(skillMdAbs)
		if perr != nil {
			b.ParseErr = perr
		} else {
			b.Skill = skill
			b.AllowedTools = parseAllowedTools(skill.RawContent)
		}
		if info, serr := os.Stat(skillMdAbs); serr == nil {
			b.SkillMdBytes = info.Size()
		}
	} else {
		b.Notes = append(b.Notes, "no SKILL.md found in bundle directory")
	}

	resolveReferences(b)
	return b, nil
}

// findSkillMd returns the absolute path of the top-level SKILL.md
// (case-insensitive) in dir, or "" if none exists. Only the immediate directory
// is consulted; a SKILL.md nested in a subdirectory is treated as a sibling
// markdown payload, not the entry point.
func findSkillMd(dir string) string {
	entries, err := os.ReadDir(dir)
	if err != nil {
		return ""
	}
	for _, e := range entries {
		if e.IsDir() {
			continue
		}
		if strings.EqualFold(e.Name(), skillMdName) {
			return filepath.Join(dir, e.Name())
		}
	}
	return ""
}

// readSniff reads up to maxSniffBytes from absPath. It reports the full file
// size, whether the file exceeded the cap (Truncated => the tail is NOT in
// data, the flag-on-truncate signal), and the newline ratio of the read prefix
// (newline bytes / len, used for padding-evasion detection). It never reads
// unbounded content, so a symlink to a huge file is safe.
func readSniff(absPath string) (data []byte, size int64, truncated bool, newlineRatio float64, err error) {
	f, oerr := os.Open(absPath) // #nosec G304 -- absPath is a bundle member discovered by the walk; reads are capped at maxSniffBytes and never executed. batou:ignore injection -- defensive scanner reads attacker-controlled skill files by design; bounded read, no exec.
	if oerr != nil {
		return nil, 0, false, 0, oerr
	}
	defer func() { _ = f.Close() }()

	if info, serr := f.Stat(); serr == nil {
		size = info.Size()
	}

	// Read one byte past the cap to detect truncation deterministically even when
	// Stat size is unavailable or lies (e.g. a growing/proc-like file).
	buf := make([]byte, maxSniffBytes+1)
	n, rerr := io.ReadFull(f, buf)
	if rerr != nil && rerr != io.EOF && rerr != io.ErrUnexpectedEOF {
		return nil, size, false, 0, rerr
	}

	if n > maxSniffBytes {
		truncated = true
		n = maxSniffBytes
	}
	data = buf[:n]

	if n > 0 {
		newlines := 0
		for _, b := range data {
			if b == '\n' {
				newlines++
			}
		}
		newlineRatio = float64(newlines) / float64(n)
	}

	return data, size, truncated, newlineRatio, nil
}

// resolveSymlink resolves a symlink at absPath. It returns the fully-resolved
// target (via EvalSymlinks), whether that target lies OUTSIDE bundleRoot
// (SymlinkEscapes), and whether resolution failed in a way consistent with a
// loop (isLoop). A loop or unresolvable target never causes a panic; the caller
// records a Note and keeps the entry so a symlinked executable reference is
// never silently dropped.
func resolveSymlink(absPath, bundleRoot string) (target string, escapes bool, isLoop bool) {
	resolved, err := filepath.EvalSymlinks(absPath)
	if err != nil {
		// EvalSymlinks fails on a loop ("too many links") and on a dangling
		// target. Fall back to a single os.Readlink so we can still record where
		// the link points (its declared target) without following it.
		if link, lerr := os.Readlink(absPath); lerr == nil {
			t := link
			if !filepath.IsAbs(t) {
				t = filepath.Join(filepath.Dir(absPath), t)
			}
			return filepath.Clean(t), !withinRoot(filepath.Clean(t), bundleRoot), true
		}
		return "", false, true
	}
	return resolved, !withinRoot(resolved, bundleRoot), false
}

// withinRoot reports whether target is inside root (or equal to it).
func withinRoot(target, root string) bool {
	rel, err := filepath.Rel(root, target)
	if err != nil {
		return false
	}
	return rel != ".." && !strings.HasPrefix(rel, ".."+string(os.PathSeparator))
}

// relOrPath returns path relative to base, falling back to the absolute path if
// the relative form cannot be computed.
func relOrPath(base, path string) string {
	if rel, err := filepath.Rel(base, path); err == nil {
		return rel
	}
	return path
}

// sameAbs reports whether two paths refer to the same absolute location.
func sameAbs(a, b string) bool {
	ca, _ := filepath.Abs(a)
	cb, _ := filepath.Abs(b)
	return filepath.Clean(ca) == filepath.Clean(cb)
}