use anyhow::{anyhow, Result};
use bytesize::ByteSize;
use clap_config_file::ClapConfigFile;
use sha2::{Digest, Sha256};
use std::io::IsTerminal;
use std::{fs, path::Path, str::FromStr, time::UNIX_EPOCH};

use crate::{
    defaults::{BINARY_FILE_EXTENSIONS, DEFAULT_IGNORE_PATTERNS, DEFAULT_OUTPUT_TEMPLATE},
    priority::PriorityRule,
};

#[derive(Clone, Debug, Default, clap::ValueEnum, serde::Serialize, serde::Deserialize)]
pub enum ConfigFormat {
    #[default]
    Toml,
    Yaml,
    Json,
}

#[derive(ClapConfigFile, Clone)]
#[config_file_name = "yek"]
#[config_file_formats = "toml,yaml,json"]
pub struct YekConfig {
    /// Input files and/or directories to process
    #[config_arg(positional)]
    pub input_paths: Vec<String>,

    /// Print version of yek
    #[config_arg(long = "version", short = 'V')]
    pub version: bool,

    /// Max size per chunk. e.g. "10MB" or "128K" or when using token counting mode, "100" or "128K"
    #[config_arg(default_value = "10MB")]
    pub max_size: String,

    /// Use token mode instead of byte mode
    #[config_arg()]
    pub tokens: String,

    /// Enable JSON output
    #[config_arg()]
    pub json: bool,

    /// Enable debug output
    #[config_arg()]
    pub debug: bool,

    /// Output directory. If none is provided & stdout is a TTY, we pick a temp dir
    #[config_arg()]
    pub output_dir: Option<String>,

    /// Output template. Defaults to ">>>> FILE_PATH\nFILE_CONTENT"
    #[config_arg(default_value = ">>>> FILE_PATH\nFILE_CONTENT")]
    pub output_template: String,

    /// Ignore patterns
    #[config_arg(long = "ignore-patterns", multi_value_behavior = "extend")]
    pub ignore_patterns: Vec<String>,

    /// Unignore patterns. Yek has some built-in ignore patterns, but you can override them here.
    #[config_arg(long = "unignore-patterns", multi_value_behavior = "extend")]
    pub unignore_patterns: Vec<String>,

    /// Priority rules
    #[config_arg(accept_from = "config_only")]
    pub priority_rules: Vec<PriorityRule>,

    /// Binary file extensions to ignore
    #[config_arg(accept_from = "config_only", default_value = BINARY_FILE_EXTENSIONS)]
    pub binary_extensions: Vec<String>,

    /// Maximum additional boost from Git commit times (0..1000)
    #[config_arg(accept_from = "config_only")]
    pub git_boost_max: Option<i32>,

    /// True if we should stream output to stdout (computed)
    pub stream: bool,

    /// True if we should count tokens, not bytes (computed)
    pub token_mode: bool,

    /// Final resolved output file path (only used if not streaming)
    pub output_file_full_path: Option<String>,

    /// Maximum depth to search for Git commit times
    #[config_arg(accept_from = "config_only", default_value = "100")]
    pub max_git_depth: i32,
}

/// Provide defaults so tests or other callers can create a baseline YekConfig easily.
impl Default for YekConfig {
    fn default() -> Self {
        Self {
            input_paths: Vec::new(),
            version: false,
            max_size: "10MB".to_string(),
            tokens: String::new(),
            json: false,
            debug: false,
            output_dir: None,
            output_template: DEFAULT_OUTPUT_TEMPLATE.to_string(),
            ignore_patterns: Vec::new(),
            unignore_patterns: Vec::new(),
            priority_rules: Vec::new(),
            binary_extensions: BINARY_FILE_EXTENSIONS
                .iter()
                .map(|s| s.to_string())
                .collect(),
            git_boost_max: Some(100),

            // computed fields
            stream: false,
            token_mode: false,
            output_file_full_path: None,
            max_git_depth: 100,
        }
    }
}

impl YekConfig {
    pub fn extend_config_with_defaults(input_paths: Vec<String>, output_dir: String) -> Self {
        YekConfig {
            input_paths,
            output_dir: Some(output_dir),
            ..Default::default()
        }
    }
}

impl YekConfig {
    /// Ensure output directory exists and is valid. Returns the resolved output directory path.
    pub fn ensure_output_dir(&self) -> Result<String> {
        if self.stream {
            return Ok(String::new());
        }

        let output_dir = if let Some(dir) = &self.output_dir {
            dir.clone()
        } else {
            let temp_dir = std::env::temp_dir().join("yek-output");
            temp_dir.to_string_lossy().to_string()
        };

        let path = Path::new(&output_dir);
        if path.exists() && !path.is_dir() {
            return Err(anyhow!(
                "output_dir: '{}' exists but is not a directory",
                output_dir
            ));
        }

        std::fs::create_dir_all(path)
            .map_err(|e| anyhow!("output_dir: cannot create '{}': {}", output_dir, e))?;

        Ok(output_dir)
    }

    /// Parse from CLI + config file, fill in computed fields, and validate.
    pub fn init_config() -> Self {
        // 1) parse from CLI and optional config file:
        let mut cfg = YekConfig::parse();

        // Handle version flag
        if cfg.version {
            println!("{}", env!("CARGO_PKG_VERSION"));
            std::process::exit(0);
        }

        // 2) compute derived fields:
        cfg.token_mode = !cfg.tokens.is_empty();
        let force_tty = std::env::var("FORCE_TTY").is_ok();

        cfg.stream = !std::io::stdout().is_terminal() && !force_tty;

        // default input dirs to current dir if none:
        if cfg.input_paths.is_empty() {
            cfg.input_paths.push(".".to_string());
        }

        // Extend binary extensions with the built-in list:
        let mut merged_bins = BINARY_FILE_EXTENSIONS
            .iter()
            .map(|s| s.to_string())
            .collect::<Vec<_>>();
        merged_bins.append(&mut cfg.binary_extensions);
        cfg.binary_extensions = merged_bins
            .into_iter()
            .collect::<std::collections::HashSet<_>>()
            .into_iter()
            .collect();

        // Always start with default ignore patterns, then add user's:
        let mut ignore = DEFAULT_IGNORE_PATTERNS
            .iter()
            .map(|s| s.to_string())
            .collect::<Vec<_>>();
        ignore.extend(cfg.ignore_patterns);
        cfg.ignore_patterns = ignore;

        // Apply unignore patterns (turn them into negative globs "!…")
        cfg.ignore_patterns
            .extend(cfg.unignore_patterns.iter().map(|pat| format!("!{}", pat)));

        // Handle output directory setup
        if !cfg.stream {
            match cfg.ensure_output_dir() {
                Ok(dir) => cfg.output_dir = Some(dir),
                Err(e) => {
                    eprintln!("Warning: Failed to create output directory: {}", e);
                    cfg.stream = true; // Fall back to streaming mode
                }
            }
        }

        // By default, we start with no final output_file_full_path:
        cfg.output_file_full_path = None;

        // 3) Validate
        if let Err(e) = cfg.validate() {
            eprintln!("Error: {}", e);
            std::process::exit(1);
        }

        cfg
    }

    /// Compute a quick checksum for the input paths (files and directories).
    /// For directories, it uses the top-level listing. For files, it uses the file metadata.
    pub fn get_checksum(input_paths: &[String]) -> String {
        let mut hasher = Sha256::new();
        for path_str in input_paths {
            let base_path = Path::new(path_str);
            if !base_path.exists() {
                continue;
            }

            // If it's a file, hash the file metadata directly
            if base_path.is_file() {
                if let Ok(meta) = fs::metadata(base_path) {
                    hasher.update(path_str.as_bytes());
                    hasher.update(meta.len().to_le_bytes());

                    if let Ok(mod_time) = meta.modified() {
                        if let Ok(dur) = mod_time.duration_since(UNIX_EPOCH) {
                            hasher.update(dur.as_secs().to_le_bytes());
                            hasher.update(dur.subsec_nanos().to_le_bytes());
                        }
                    }
                }
                continue;
            }

            // If it's a directory, hash its contents
            let entries = match fs::read_dir(base_path) {
                Ok(iter) => iter.filter_map(|e| e.ok()).collect::<Vec<_>>(),
                Err(_) => continue,
            };

            // Sort deterministically by path name
            let mut sorted = entries;
            sorted.sort_by_key(|a| a.path());

            for entry in sorted {
                let p = entry.path();
                if let Ok(meta) = fs::metadata(&p) {
                    let path_str = p.to_string_lossy();
                    hasher.update(path_str.as_bytes());
                    hasher.update(meta.len().to_le_bytes());

                    if let Ok(mod_time) = meta.modified() {
                        if let Ok(dur) = mod_time.duration_since(UNIX_EPOCH) {
                            hasher.update(dur.as_secs().to_le_bytes());
                            hasher.update(dur.subsec_nanos().to_le_bytes());
                        }
                    }
                }
            }
        }
        let result = hasher.finalize();
        // Convert the 32-byte result to hex, but only keep the first 8 characters
        let hex = format!("{:x}", result);
        hex[..8].to_owned()
    }

    /// Validate the final config.

pub fn validate(&self) -> Result<()> {
        if !self.output_template.contains("FILE_PATH")
            || !self.output_template.contains("FILE_CONTENT")
        {
            return Err(anyhow!(
                "output_template: must contain FILE_PATH and FILE_CONTENT"
            ));
        }
        if self.max_size == "0" {
            return Err(anyhow!("max_size: cannot be 0"));
        }
        if !self.token_mode {
            ByteSize::from_str(&self.max_size)
                .map_err(|e| anyhow!("max_size: Invalid size format: {}", e))?;
        } else if self.tokens.to_lowercase().ends_with('k') {
            let val = self.tokens[..self.tokens.len() - 1]
                .trim()
                .parse::<usize>()
                .map_err(|e| anyhow!("tokens: Invalid token size: {}", e))?;
            if val == 0 {
                return Err(anyhow!("tokens: cannot be 0"));
            }
        } else if !self.tokens.is_empty() {
            let val = self
                .tokens
                .parse::<usize>()
                .map_err(|e| anyhow!("tokens: Invalid token size: {}", e))?;
            if val == 0 {
                return Err(anyhow!("tokens: cannot be 0"));
            }
        }
        if !self.stream {
            self.ensure_output_dir()?;
        }
        for pattern in &self.ignore_patterns {
            glob::Pattern::new(pattern)
                .map_err(|e| anyhow!("ignore_patterns: Invalid pattern '{}': {}", pattern, e))?;
        }
        for rule in &self.priority_rules {
            if rule.score < 0 || rule.score > 1000 {
                return Err(anyhow!(
                    "priority_rules: Priority score {} must be between 0 and 1000",
                    rule.score
                ));
            }
            glob::Pattern::new(&rule.pattern).map_err(|e| {
                anyhow!("priority_rules: Invalid pattern '{}': {}", rule.pattern, e)
            })?;
        }
        Ok(())
    }

use anyhow::anyhow;
use anyhow::Result;
use bytesize::ByteSize;
use content_inspector::{inspect, ContentType};
use rayon::prelude::*;
use std::{
    collections::HashMap,
    fs::File,
    io::{self, Read},
    path::Path,
    str::FromStr,
    sync::OnceLock,
};
use tiktoken_rs::CoreBPE;

pub mod config;
pub mod defaults;
pub mod parallel;
pub mod priority;

use config::YekConfig;
use parallel::{process_files_parallel, ProcessedFile};
use priority::compute_recentness_boost;

// Add a static BPE encoder for reuse
static TOKENIZER: OnceLock<CoreBPE> = OnceLock::new();

fn get_tokenizer() -> &'static CoreBPE {
    TOKENIZER.get_or_init(|| {
        tiktoken_rs::get_bpe_from_model("gpt-3.5-turbo").expect("Failed to load tokenizer")
    })
}

/// Check if a file is likely text or binary by reading only a small chunk.
/// This avoids reading large files fully just to detect their type.
pub fn is_text_file(path: &Path, user_binary_extensions: &[String]) -> io::Result<bool> {
    // If extension is known to be binary, skip quickly
    if let Some(ext) = path.extension().and_then(|e| e.to_str()) {
        if user_binary_extensions.iter().any(|bin_ext| bin_ext == ext) {
            return Ok(false);
        }
    }

    // Short partial read to check if it's binary or text
    const INSPECTION_BYTES: usize = 8192;
    let mut file = File::open(path)?;
    let mut buf = vec![0u8; INSPECTION_BYTES];
    let n = file.read(&mut buf)?;
    buf.truncate(n);

    Ok(inspect(&buf) != ContentType::BINARY)
}

/// Main entrypoint for serialization, used by CLI and tests
pub fn serialize_repo(config: &YekConfig) -> Result<(String, Vec<ProcessedFile>)> {
    // Gather commit times from each input path that is a directory
    let combined_commit_times = config
        .input_paths
        .par_iter()
        .filter_map(|path_str| {
            let repo_path = Path::new(path_str);
            if repo_path.is_dir() {
                priority::get_recent_commit_times_git2(
                    repo_path,
                    config.max_git_depth.try_into().unwrap_or(0),
                )
            } else {
                None
            }
        })
        .flatten()
        .collect::<HashMap<String, u64>>();

    // Compute a recentness-based boost
    let recentness_boost =
        compute_recentness_boost(&combined_commit_times, config.git_boost_max.unwrap_or(100));

    // Process files in parallel for each input path
    let merged_files = config
        .input_paths
        .par_iter()
        .map(|path_str| {
            let path = Path::new(path_str);
            process_files_parallel(path, config, &recentness_boost)
        })
        .collect::<Result<Vec<Vec<ProcessedFile>>>>()?
        .into_iter()
        .flatten()
        .collect::<Vec<ProcessedFile>>();

    let mut files = merged_files;

    // Sort final (priority asc, then file_index asc)
    files.par_sort_by(|a, b| {
        a.priority
            .cmp(&b.priority)
            .then_with(|| a.rel_path.cmp(&b.rel_path))
    });

    // Build the final output string
    let output_string = concat_files(&files, config)?;

    // Only count tokens if debug logging is enabled
    if tracing::Level::DEBUG <= tracing::level_filters::STATIC_MAX_LEVEL {
        tracing::debug!("{} tokens generated", count_tokens(&output_string));
    }

    Ok((output_string, files))
}

pub fn concat_files(files: &[ProcessedFile], config: &YekConfig) -> anyhow::Result<String> {
    let mut accumulated = 0_usize;
    let cap = if config.token_mode {
        parse_token_limit(&config.tokens)?
    } else {
        ByteSize::from_str(&config.max_size)
            .map_err(|e| anyhow!("max_size: Invalid size format: {}", e))?
            .as_u64() as usize
    };

    // Sort by priority (asc) and file_index (asc)
    let mut sorted_files: Vec<_> = files.iter().collect();
    sorted_files.sort_by(|a, b| {
        a.priority
            .cmp(&b.priority)
            .then_with(|| a.rel_path.cmp(&b.rel_path))
    });

    let mut files_to_include = Vec::new();
    for file in sorted_files {
        let content_size = if config.token_mode {
            // Format the file content with template first, then count tokens
            let formatted = if config.json {
                serde_json::to_string(&serde_json::json!({
                    "filename": &file.rel_path,
                    "content": &file.content,
                }))
                .map_err(|e| anyhow!("Failed to serialize JSON: {}", e))?
            } else {
                config
                    .output_template
                    .replace("FILE_PATH", &file.rel_path)
                    .replace("FILE_CONTENT", &file.content)
            };
            count_tokens(&formatted)
        } else {
            file.content.len()
        };

        if accumulated + content_size <= cap {
            accumulated += content_size;
            files_to_include.push(file);
        } else {
            break;
        }
    }

    if config.json {
        // JSON array of objects
        Ok(serde_json::to_string_pretty(
            &files_to_include
                .iter()
                .map(|f| {
                    serde_json::json!({
                        "filename": &f.rel_path,
                        "content": &f.content,
                    })
                })
                .collect::<Vec<_>>(),
        )?)
    } else {
        // Use the user-defined template
        Ok(files_to_include
            .iter()
            .map(|f| {
                config
                    .output_template
                    .replace("FILE_PATH", &f.rel_path)
                    .replace("FILE_CONTENT", &f.content)
                    // Handle both literal "\n" and escaped "\\n"
                    .replace("\\\\\n", "\n") // First handle escaped newline
                    .replace("\\\\n", "\n") // Then handle escaped \n sequence
            })
            .collect::<Vec<_>>()
            .join("\n"))
    }
}

/// Parse a token limit string like "800k" or "1000" into a number
pub fn parse_token_limit(limit: &str) -> anyhow::Result<usize> {
    if limit.to_lowercase().ends_with('k') {
        limit[..limit.len() - 1]
            .trim()
            .parse::<usize>()
            .map(|n| n * 1000)
            .map_err(|e| anyhow!("tokens: Invalid token size: {}", e))
    } else {
        limit
            .parse::<usize>()
            .map_err(|e| anyhow!("tokens: Invalid token size: {}", e))
    }
}

/// Count tokens using tiktoken's GPT-3.5-Turbo tokenizer for accuracy
pub fn count_tokens(text: &str) -> usize {
    get_tokenizer().encode_with_special_tokens(text).len()
}

use crate::{config::YekConfig, priority::get_file_priority, Result};
use content_inspector::{inspect, ContentType};
use glob::glob;
use ignore::gitignore::GitignoreBuilder;
use path_slash::PathBufExt;
use rayon::prelude::*;
use std::{
    collections::HashMap,
    fs,
    path::Path,
    sync::{mpsc, Arc},
};
use tracing::debug;

#[derive(Debug, Clone)]
pub struct ProcessedFile {
    pub priority: i32,
    pub file_index: usize,
    pub rel_path: String,
    pub content: String,
}

/// Process a single file, checking ignore patterns and reading its contents.
fn process_single_file(
    file_path: &Path,
    config: &YekConfig,
    boost_map: &HashMap<String, i32>,
) -> Result<Vec<ProcessedFile>> {
    let base_dir = file_path.parent().unwrap_or(Path::new(""));
    let rel_path = normalize_path(file_path, base_dir);

    // Build the gitignore
    let mut gitignore_builder = GitignoreBuilder::new(base_dir);
    for pattern in &config.ignore_patterns {
        gitignore_builder.add_line(None, pattern)?;
    }

    // If there is a .gitignore in this folder, add it last so its "!" lines override prior patterns
    let gitignore_file = base_dir.join(".gitignore");
    if gitignore_file.exists() {
        gitignore_builder.add(&gitignore_file);
    }

    let gitignore = gitignore_builder.build()?;
    if gitignore.matched(file_path, false).is_ignore() {
        debug!("Skipping ignored file: {rel_path}");
        return Ok(Vec::new());
    }

    let mut processed_files = Vec::new();

    match fs::read(file_path) {
        Ok(content) => {
            if inspect(&content) == ContentType::BINARY {
                debug!("Skipping binary file: {rel_path}");
            } else {
                let rule_priority = get_file_priority(&rel_path, &config.priority_rules);
                let boost = boost_map.get(&rel_path).copied().unwrap_or(0);
                let combined_priority = rule_priority + boost;

                processed_files.push(ProcessedFile {
                    priority: combined_priority,
                    file_index: 0, // For a single file, the index is always 0
                    rel_path,
                    content: String::from_utf8_lossy(&content).to_string(),
                });
            }
        }
        Err(e) => {
            debug!("Failed to read {rel_path}: {e}");
        }
    }

    Ok(processed_files)
}

/// Walk files in parallel (if a directory is given), skipping ignored paths,
/// then read each file's contents in a separate thread.
/// Return the resulting `ProcessedFile` objects.
pub fn process_files_parallel(
    base_path: &Path,
    config: &YekConfig,
    boost_map: &HashMap<String, i32>,
) -> Result<Vec<ProcessedFile>> {
    // Expand globs into a list of paths
    let mut expanded_paths = Vec::new();
    let path_str = base_path.to_string_lossy();
    for entry in glob(&path_str)? {
        match entry {
            Ok(path) => expanded_paths.push(path),
            Err(e) => debug!("Glob entry error: {:?}", e),
        }
    }

    // If it's a single file (no glob expansion or single file result), process it directly
    if expanded_paths.len() == 1 && expanded_paths[0].is_file() {
        return process_single_file(&expanded_paths[0], config, boost_map);
    }

    // Iterate over expanded paths, handling files and directories
    let mut all_processed_files = Vec::new();
    for path in expanded_paths {
        if path.is_file() {
            all_processed_files.extend(process_single_file(&path, config, boost_map)?);
        } else if path.is_dir() {
            // For directories, use the original recursive logic
            all_processed_files.extend(process_files_parallel_internal(&path, config, boost_map)?);
        }
    }

    Ok(all_processed_files)
}

/// Internal function to handle directory recursion (separated for clarity)
fn process_files_parallel_internal(
    base_path: &Path,
    config: &YekConfig,
    boost_map: &HashMap<String, i32>,
) -> Result<Vec<ProcessedFile>> {
    // It's a directory, so walk it
    let mut walk_builder = ignore::WalkBuilder::new(base_path);

    // Standard filters + no follow symlinks
    walk_builder
        .follow_links(false)
        .standard_filters(true)
        .require_git(false);

    // Build the gitignore
    let mut gitignore_builder = GitignoreBuilder::new(base_path);
    // Add our custom patterns first
    for pattern in &config.ignore_patterns {
        gitignore_builder.add_line(None, pattern)?;
    }

    // If there is a .gitignore in this folder, add it last so its "!" lines override prior patterns
    let gitignore_file = base_path.join(".gitignore");
    if gitignore_file.exists() {
        gitignore_builder.add(&gitignore_file);
    }

    let gitignore = Arc::new(gitignore_builder.build()?); // Propagate error here

    // This channel will carry (path, rel_path) to the processing thread
    let (processed_files_tx, processed_files_rx) = mpsc::channel::<(std::path::PathBuf, String)>();

    // Processing happens on a dedicated thread, to keep from blocking the main walker
    let process_thread = std::thread::spawn({
        let priority_rules = config.priority_rules.clone();
        let boost_map = boost_map.clone();
        move || {
            let mut processed = Vec::new();
            for (path, rel_path) in processed_files_rx {
                // Read entire file
                match fs::read(&path) {
                    Ok(content) => {
                        // Check if it's binary quickly
                        if inspect(&content) == ContentType::BINARY {
                            debug!("Skipping binary file: {rel_path}");
                            continue;
                        }
                        // Compute priority
                        let rule_priority = get_file_priority(&rel_path, &priority_rules);
                        let boost = boost_map.get(&rel_path).copied().unwrap_or(0);
                        let combined = rule_priority + boost;
                        processed.push(ProcessedFile {
                            priority: combined,
                            file_index: 0, // assigned later
                            rel_path,
                            content: String::from_utf8_lossy(&content).to_string(),
                        });
                    }
                    Err(e) => {
                        debug!("Failed to read {rel_path}: {e}");
                    }
                }
            }
            processed
        }
    });

    // Use ignore's parallel walker to skip ignored files
    let base_cloned = base_path.to_owned();
    let walker_tx = processed_files_tx.clone();

    // Now build the walker (no .gitignore custom filename)
    walk_builder.build_parallel().run(move || {
        let base_dir = base_cloned.clone();
        let processed_files_tx = walker_tx.clone();
        let gitignore = Arc::clone(&gitignore);

        Box::new(move |entry| {
            let entry = match entry {
                Ok(e) => e,
                Err(_) => return ignore::WalkState::Continue,
            };
            // Only process files
            if !entry.file_type().is_some_and(|ft| ft.is_file()) {
                return ignore::WalkState::Continue;
            }

            let path = entry.path().to_path_buf();
            let rel_path = normalize_path(&path, &base_dir);

            // If gitignore says skip, we do not even read
            if gitignore.matched(&path, false).is_ignore() {
                debug!("Skipping ignored file: {rel_path}");
                return ignore::WalkState::Continue;
            }

            // Otherwise we send to processing thread
            processed_files_tx.send((path, rel_path)).ok();
            ignore::WalkState::Continue
        })
    });

    // Drop the sender so the thread can end
    drop(processed_files_tx);

    // Join the processing thread
    let mut processed_files = process_thread.join().unwrap();

    // Now assign file_index within each priority group
    let mut counters = HashMap::new();
    for f in &mut processed_files {
        let ctr = counters.entry(f.priority).or_insert(0);
        f.file_index = *ctr;
        *ctr += 1;
    }

    if config.debug {
        debug!(
            "Processed {} files in parallel for base_path: {}",
            processed_files.len(),
            base_path.display()
        );
    }

    // Sort by priority desc, then file_index
    processed_files.par_sort_by(|a, b| {
        a.priority
            .cmp(&b.priority)
            .reverse()
            .then_with(|| a.file_index.cmp(&b.file_index))
    });

    Ok(processed_files)
}

/// Create a relative, slash-normalized path
pub fn normalize_path(path: &Path, base: &Path) -> String {
    path.strip_prefix(base)
        .unwrap_or(path)
        .to_path_buf()
        .to_slash()
        .unwrap_or_default()
        .to_string()
}

use git2;
use regex;
use serde::{Deserialize, Serialize};
use std::{collections::HashMap, path::Path};
use tracing::debug;

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct PriorityRule {
    pub pattern: String,
    pub score: i32,
}

/// Determine final priority of a file by scanning the priority list
/// in descending order of score.
pub fn get_file_priority(path: &str, rules: &[PriorityRule]) -> i32 {
    let mut priority = 0;
    for rule in rules {
        if let Ok(re) = regex::Regex::new(&rule.pattern) {
            if re.is_match(path) {
                priority += rule.score;
            }
        }
    }
    priority
}

/// Rank-based approach to compute how "recent" each file is (0=oldest, 1=newest).
/// Then scale it to a user-defined or default max boost.
pub fn compute_recentness_boost(
    commit_times: &HashMap<String, u64>,
    max_boost: i32,
) -> HashMap<String, i32> {
    if commit_times.is_empty() {
        return HashMap::new();
    }

    // Sort by ascending commit time => first is oldest
    let mut sorted: Vec<(&String, &u64)> = commit_times.iter().collect();
    sorted.sort_by_key(|(_, t)| **t);

    // If there's only one file, or zero, no boosts make sense
    if sorted.len() <= 1 {
        let mut single = HashMap::new();
        for file in commit_times.keys() {
            single.insert(file.clone(), 0);
        }
        return single;
    }

    let mut result = HashMap::new();
    let oldest_time = *sorted.first().unwrap().1;
    let newest_time = *sorted.last().unwrap().1;
    let time_range = newest_time.saturating_sub(oldest_time) as f64;

    // If all files have the same timestamp, they should all get the same boost
    if time_range == 0.0 {
        for (path, _) in sorted {
            result.insert(path.clone(), 0);
        }
        return result;
    }

    // Calculate boost based on time difference from oldest file
    for (path, time) in sorted {
        let time_diff = (*time - oldest_time) as f64;
        let rank = time_diff / time_range; // 0.0..1.0 (older files get lower rank)
        let boost = (rank * max_boost as f64).round() as i32; // Newer files get higher boost
        result.insert(path.clone(), boost);
    }
    result
}

/// Get the commit time of the most recent change to each file using git2.
/// Returns a map from file path (relative to the repo root) → last commit Unix time.
/// If Git or .git folder is missing, returns None instead of erroring.
/// Only considers up to `max_commits` most recent commits.
pub fn get_recent_commit_times_git2(
    repo_path: &Path,
    max_commits: usize,
) -> Option<HashMap<String, u64>> {
    // Walk up until you find a .git folder but not higher than the base of the given repo_path
    let mut current_path = repo_path.to_path_buf();
    while current_path.components().count() > 1 {
        if current_path.join(".git").exists() {
            break;
        }
        current_path = current_path.parent()?.to_path_buf();
    }

    let repo = match git2::Repository::open(&current_path) {
        Ok(repo) => repo,
        Err(_) => {
            debug!("Not a Git repository or unable to open: {:?}", current_path);
            return None;
        }
    };

    let mut revwalk = match repo.revwalk() {
        Ok(revwalk) => revwalk,
        Err(_) => {
            debug!("Unable to get revwalk for: {:?}", current_path);
            return None;
        }
    };

    if let Err(e) = revwalk.push_head() {
        debug!(
            "Unable to push HEAD to revwalk: {:?} in {:?}",
            e, current_path
        );
        return None;
    }
    revwalk.set_sorting(git2::Sort::TIME).ok()?;

    let mut commit_times = HashMap::new();
    for oid_result in revwalk.take(max_commits) {
        let oid = match oid_result {
            Ok(oid) => oid,
            Err(e) => {
                debug!("Error during revwalk iteration: {:?}", e);
                continue;
            }
        };

        let commit = match repo.find_commit(oid) {
            Ok(commit) => commit,
            Err(e) => {
                debug!("Failed to find commit for OID {:?}: {:?}", oid, e);
                continue;
            }
        };
        let tree = match commit.tree() {
            Ok(tree) => tree,
            Err(e) => {
                debug!("Failed to get tree for commit {:?}: {:?}", oid, e);
                continue;
            }
        };

        let time = commit.time().seconds() as u64;
        tree.walk(git2::TreeWalkMode::PreOrder, |root, entry| {
            if let Some(name) = entry.name() {
                if entry.kind() == Some(git2::ObjectType::Blob) {
                    let full_path = format!("{}{}", root, name);
                    commit_times.entry(full_path).or_insert(time);
                }
            }
            git2::TreeWalkResult::Ok
        })
        .ok()?;
    }

    Some(commit_times)
}

use anyhow::Result;
use bytesize::ByteSize;
use rayon::join;
use std::path::Path;
use tracing::{debug, Level};
use tracing_subscriber::fmt;
use yek::{config::YekConfig, serialize_repo};

fn main() -> Result<()> {
    // 1) Parse CLI + config files:
    let mut full_config = YekConfig::init_config();

    let env_filter = if full_config.debug {
        "yek=debug,ignore=off"
    } else {
        "yek=info,ignore=off"
    };

    // 2) Initialize tracing:
    fmt::Subscriber::builder()
        .with_max_level(if full_config.debug {
            Level::DEBUG
        } else {
            Level::INFO
        })
        .with_target(false)
        .with_thread_ids(false)
        .with_thread_names(false)
        .with_file(false)
        .with_line_number(false)
        .with_level(true)
        .with_env_filter(env_filter)
        .compact()
        .init();

    if full_config.debug {
        let config_str = serde_json::to_string_pretty(&full_config)?;
        debug!("Configuration:\n{}", config_str);
    }

    // If streaming => skip checksum + read. Just do single-thread call to serialize_repo.
    // If not streaming => run checksum + repo serialization in parallel.
    if full_config.stream {
        let (output, files) = serialize_repo(&full_config)?;
        // We print actual text to stdout:
        println!("{}", output);

        if full_config.debug {
            debug!("{} files processed (streaming).", files.len());
            debug!("Output lines: {}", output.lines().count());
        }
    } else {
        // Not streaming => run repo serialization & checksum in parallel
        let (serialization_res, checksum_res) = join(
            || serialize_repo(&full_config),
            || YekConfig::get_checksum(&full_config.input_paths),
        );

        // Handle both results
        let (output_string, files) = serialization_res?;
        let checksum = checksum_res;

        // Now set the final output file with the computed checksum
        let extension = if full_config.json { "json" } else { "txt" };
        let output_dir = full_config.output_dir.as_ref().ok_or_else(|| {
            anyhow::anyhow!("Output directory is required when not in streaming mode. This may indicate a configuration validation error.")
        })?;

        let final_path = Path::new(output_dir)
            .join(format!("yek-output-{}.{}", checksum, extension))
            .to_string_lossy()
            .to_string();
        full_config.output_file_full_path = Some(final_path.clone());

        // If debug, show stats
        if full_config.debug {
            let size = ByteSize::b(output_string.len() as u64);
            debug!("{} files processed", files.len());
            debug!("{} generated", size);
            debug!("{} lines generated", output_string.lines().count());
        }

        // Actually write the final output file.
        // We'll do it right here (instead of inside `serialize_repo`) to ensure we use our new final_path:
        std::fs::write(&final_path, output_string.as_bytes())?;

        // Print path to stdout (like original code did)
        println!("{}", final_path);
    }

    Ok(())
}

/// Known binary file extensions that should be skipped
#[rustfmt::skip]
pub const BINARY_FILE_EXTENSIONS: &[&str] = &[
    // Executables, Libraries, Core Dumps
    "exe", "dll", "so", "dylib", "ocx", "ax", "drv", "sys", "msi", "app", "ipa", "apk",
    "bin", "out", "a", "lib", "ko", "elf", "o", "nro", "core", "img", "iso",

    // Java / .NET / Archives
    "class", "jar", "war", "ear",
    "resources", // sometimes included in Java archives
    "nupkg", // NuGet package
    "exe.config", // sometimes for .NET
    "dll.config",

    // Archives & Compressed
    "zip", "tar", "gz", "tgz", "bz2", "xz", "7z", "rar", "lz4", "lz", "zst", "lzma",
    "cab", "ar", "cpio", "rpm", "deb", "pkg", "crx", "bin", "dmg", "hfs", "img",
    "cso", // Compressed ISO
    "bz", "tbz", "tbz2", "tlz", "txz", "z", "Z", "apk", "xapk",

    // Disk & Container Images
    "iso", "img", "dmg", "vhd", "vhdx", "vmdk", "vdi", "qcow", "qcow2",
    "mdf", "mds", "nrg", "uif",

    // Documents & Office
    "pdf",
    "doc", "docx", "dot", "dotx", "docm", "dotm",
    "xls", "xlsx", "xlsm", "xlsb", "xlt", "xltx", "xltm", "xlc", "xlw",
    "ppt", "pptx", "pptm", "pps", "ppsx", "pot", "potx", "potm",
    "pub",  // Microsoft Publisher
    "vsd", "vsdx", // Visio
    "accdb", "accde", "mdb", "mde", // Access
    "odt", "ods", "odp", "odg", "odf", // OpenDocument
    "pages", "numbers", "key", // Apple iWork
    "rtf", // can be binary-like depending on usage

    // Spreadsheets, DB, and Misc Data
    "db", "sqlite", "db3", "s3db", "frm", "myd", "myi", // MySQL
    "mdb", "bak", "nsf", // Lotus Notes
    "gdb", "fdb", // Firebird
    "mdb", // Access DB
    "wdb", // Works DB

    // Images
    "jpg", "jpeg", "png", "gif", "bmp", "ico", "tiff", "tif", "webp", "jfif", "jp2",
    "psd", "psb", "xcf", "ai", "eps", "raw", "arw", "cr2", "nef", "dng", "raf", "orf",
    "sr2", "heic", "heif", "icns", "img", "bpg",

    // Audio
    "mp3", "mp2", "aac", "ac3", "wav", "ogg", "oga", "flac", "alac", "m4a", "mp4a",
    "wma", "ra", "ram", "ape", "opus", "amr", "awb",

    // Video
    "mp4", "m4v", "mov", "avi", "wmv", "mkv", "flv", "f4v", "f4p", "f4a", "f4b", "3gp",
    "3g2", "mpeg", "mpg", "mpe", "m1v", "m2v", "ts", "mts", "m2ts", "vob", "rm", "rmvb",
    "asf", "ogv", "ogm", "webm", "dv", "divx", "xvid",

    // Font Files
    "ttf", "otf", "woff", "woff2", "eot", "fon", "psf",

    // Firmware / BIOS / ROM / Game Data
    "rom", "iso", "bin", "gba", "gbc", "nds", "n64", "z64", "v64", "gcm", "ciso", "wbfs",
    "pak", "wad", "dat", "sav", "rpx",

    // Flash / Vector
    "swf", "fla", "svgz", // .svgz is compressed SVG (binary)

    // CAD / 3D
    "dwg", "dxf", "dwf", "skp", "ifc",
    "stl", "obj", "fbx", "dae", "blend", "3ds", "ase", "gltf", "glb",

    // E-Books
    "epub", "mobi", "azw", "azw3", "fb2", "lrf", "lit", "pdb",

    // Other
    "swp", "swo", // Vim swap files
    "pch", // Precompiled header
    "xex", "elf", // Console executables
    "dmp", "mdmp", // Memory dump
    "bkf", "bkp", // Backup
    "pak", // Common game data archives
    "idx", "dat", "vcd", // Various binary data
    "icns", // macOS icon
    "hlp", "chm", // Windows help
    "torrent", // BitTorrent
    "mar", // Mozilla archive
    "qcow", "qcow2", // QEMU disk
    "apk", "aab", // Android package/bundle
    "crx", // Chrome extension
    "appx", // Windows app package
    "xap", // Windows Phone app
];

/// Default sets of ignore patterns (separate from .gitignore)
pub const DEFAULT_IGNORE_PATTERNS: &[&str] = &[
    "LICENSE",
    ".git/**",
    ".next/**",
    "node_modules/**",
    "vendor/**",
    "dist/**",
    "build/**",
    "out/**",
    "target/**",
    "bin/**",
    "obj/**",
    ".idea/**",
    ".vscode/**",
    ".vs/**",
    ".settings/**",
    ".gradle/**",
    ".mvn/**",
    ".pytest_cache/**",
    "__pycache__/**",
    ".sass-cache/**",
    ".vercel/**",
    ".turbo/**",
    "coverage/**",
    "test-results/**",
    ".gitignore",
    "pnpm-lock.yaml",
    "yek.toml",
    "package-lock.json",
    "yarn.lock",
    "Cargo.lock",
    "Gemfile.lock",
    "composer.lock",
    "mix.lock",
    "poetry.lock",
    "Pipfile.lock",
    "packages.lock.json",
    "paket.lock",
    "*.pyc",
    "*.pyo",
    "*.pyd",
    "*.class",
    "*.o",
    "*.obj",
    "*.dll",
    "*.exe",
    "*.so",
    "*.dylib",
    "*.log",
    "*.tmp",
    "*.temp",
    "*.swp",
    "*.swo",
    ".DS_Store",
    "Thumbs.db",
    ".env*",
    "*.bak",
    "*~",
];

pub const DEFAULT_OUTPUT_TEMPLATE: &str = ">>>> FILE_PATH\nFILE_CONTENT";

pub fn validate (&self) -> Result<()> {}

This code segment functions to: Perform comprehensive validation checks on configuration parameters including template structure integrity, size format correctness, directory existence, and pattern syntax verification while enforcing constraints on numeric value ranges.

```rust
[TASK_BEGIN]
    pub fn validate(&self) -> Result<()> {
        // Validate max_size format
        if self.token_mode {
            self.tokens.parse::<usize>().map_err(|_| {
                anyhow!("Invalid token size format '{}' - must be a positive integer", self.tokens)
            })?;
        } else {
            ByteSize::from_str(&self.max_size).map_err(|_| {
                anyhow!(
                    "Invalid size format '{}' - must be like '10MB' or '128K'",
                    self.max_size
                )
            })?;
        }

        // Validate git_boost_max bounds
        if let Some(boost) = self.git_boost_max {
            if !(0..=1000).contains(&boost) {
                return Err(anyhow!("git_boost_max must be between 0 and 1000"));
            }
        }

        // Validate output template contains required placeholders
        if !self.output_template.contains("FILE_PATH") {
            return Err(anyhow!("output_template must contain FILE_PATH placeholder"));
        }
        if !self.output_template.contains("FILE_CONTENT") {
            return Err(anyhow!("output_template must contain FILE_CONTENT placeholder"));
        }

        // Validate max_git_depth
        if self.max_git_depth < 0 {
            return Err(anyhow!("max_git_depth cannot be negative"));
        }

        Ok(())
    }
[TASK_END]
```

pub fn validate(&self) -> Result<()> {
        if self.token_mode {
            self.tokens.parse::<usize>().map_err(|_| {
                anyhow!("Invalid token size format '{}' - must be a positive integer", self.tokens)
            })?;
        } else {
            ByteSize::from_str(&self.max_size).map_err(|_| {
                anyhow!(
                    "Invalid size format '{}' - must be like '10MB' or '128K'",
                    self.max_size
                )
            })?;
        }
        if let Some(boost) = self.git_boost_max {
            if !(0..=1000).contains(&boost) {
                return Err(anyhow!("git_boost_max must be between 0 and 1000"));
            }
        }
        if !self.output_template.contains("FILE_PATH") {
            return Err(anyhow!("output_template must contain FILE_PATH placeholder"));
        }
        if !self.output_template.contains("FILE_CONTENT") {
            return Err(anyhow!("output_template must contain FILE_CONTENT placeholder"));
        }
        if self.max_git_depth < 0 {
            return Err(anyhow!("max_git_depth cannot be negative"));
        }
        Ok(())
    }


use crate::{config::YekConfig, priority::get_file_priority, Result};
use content_inspector::{inspect, ContentType};
use glob::glob;
use ignore::gitignore::GitignoreBuilder;
use path_slash::PathBufExt;
use rayon::prelude::*;
use std::{
    collections::HashMap,
    fs,
    path::Path,
    sync::{mpsc, Arc},
};
use tracing::debug;

#[derive(Debug, Clone)]
pub struct ProcessedFile {
    pub priority: i32,
    pub file_index: usize,
    pub rel_path: String,
    pub content: String,
}

/// Process a single file, checking ignore patterns and reading its contents.
fn process_single_file(
    file_path: &Path,
    config: &YekConfig,
    boost_map: &HashMap<String, i32>,
) -> Result<Vec<ProcessedFile>> {
    let base_dir = file_path.parent().unwrap_or(Path::new(""));
    let rel_path = normalize_path(file_path, base_dir);

    // Build the gitignore
    let mut gitignore_builder = GitignoreBuilder::new(base_dir);
    for pattern in &config.ignore_patterns {
        gitignore_builder.add_line(None, pattern)?;
    }

    // If there is a .gitignore in this folder, add it last so its "!" lines override prior patterns
    let gitignore_file = base_dir.join(".gitignore");
    if gitignore_file.exists() {
        gitignore_builder.add(&gitignore_file);
    }

    let gitignore = gitignore_builder.build()?;
    if gitignore.matched(file_path, false).is_ignore() {
        debug!("Skipping ignored file: {rel_path}");
        return Ok(Vec::new());
    }

    let mut processed_files = Vec::new();

    match fs::read(file_path) {
        Ok(content) => {
            if inspect(&content) == ContentType::BINARY {
                debug!("Skipping binary file: {rel_path}");
            } else {
                let rule_priority = get_file_priority(&rel_path, &config.priority_rules);
                let boost = boost_map.get(&rel_path).copied().unwrap_or(0);
                let combined_priority = rule_priority + boost;

                processed_files.push(ProcessedFile {
                    priority: combined_priority,
                    file_index: 0, // For a single file, the index is always 0
                    rel_path,
                    content: String::from_utf8_lossy(&content).to_string(),
                });
            }
        }
        Err(e) => {
            debug!("Failed to read {rel_path}: {e}");
        }
    }

    Ok(processed_files)
}

/// Walk files in parallel (if a directory is given), skipping ignored paths,
/// then read each file's contents in a separate thread.
/// Return the resulting `ProcessedFile` objects.

/// Internal function to handle directory recursion (separated for clarity)
fn process_files_parallel_internal(
    base_path: &Path,
    config: &YekConfig,
    boost_map: &HashMap<String, i32>,
) -> Result<Vec<ProcessedFile>> {
    // It's a directory, so walk it
    let mut walk_builder = ignore::WalkBuilder::new(base_path);

    // Standard filters + no follow symlinks
    walk_builder
        .follow_links(false)
        .standard_filters(true)
        .require_git(false);

    // Build the gitignore
    let mut gitignore_builder = GitignoreBuilder::new(base_path);
    // Add our custom patterns first
    for pattern in &config.ignore_patterns {
        gitignore_builder.add_line(None, pattern)?;
    }

    // If there is a .gitignore in this folder, add it last so its "!" lines override prior patterns
    let gitignore_file = base_path.join(".gitignore");
    if gitignore_file.exists() {
        gitignore_builder.add(&gitignore_file);
    }

    let gitignore = Arc::new(gitignore_builder.build()?); // Propagate error here

    // This channel will carry (path, rel_path) to the processing thread
    let (processed_files_tx, processed_files_rx) = mpsc::channel::<(std::path::PathBuf, String)>();

    // Processing happens on a dedicated thread, to keep from blocking the main walker
    let process_thread = std::thread::spawn({
        let priority_rules = config.priority_rules.clone();
        let boost_map = boost_map.clone();
        move || {
            let mut processed = Vec::new();
            for (path, rel_path) in processed_files_rx {
                // Read entire file
                match fs::read(&path) {
                    Ok(content) => {
                        // Check if it's binary quickly
                        if inspect(&content) == ContentType::BINARY {
                            debug!("Skipping binary file: {rel_path}");
                            continue;
                        }
                        // Compute priority
                        let rule_priority = get_file_priority(&rel_path, &priority_rules);
                        let boost = boost_map.get(&rel_path).copied().unwrap_or(0);
                        let combined = rule_priority + boost;
                        processed.push(ProcessedFile {
                            priority: combined,
                            file_index: 0, // assigned later
                            rel_path,
                            content: String::from_utf8_lossy(&content).to_string(),
                        });
                    }
                    Err(e) => {
                        debug!("Failed to read {rel_path}: {e}");
                    }
                }
            }
            processed
        }
    });

    // Use ignore's parallel walker to skip ignored files
    let base_cloned = base_path.to_owned();
    let walker_tx = processed_files_tx.clone();

    // Now build the walker (no .gitignore custom filename)
    walk_builder.build_parallel().run(move || {
        let base_dir = base_cloned.clone();
        let processed_files_tx = walker_tx.clone();
        let gitignore = Arc::clone(&gitignore);

        Box::new(move |entry| {
            let entry = match entry {
                Ok(e) => e,
                Err(_) => return ignore::WalkState::Continue,
            };
            // Only process files
            if !entry.file_type().is_some_and(|ft| ft.is_file()) {
                return ignore::WalkState::Continue;
            }

            let path = entry.path().to_path_buf();
            let rel_path = normalize_path(&path, &base_dir);

            // If gitignore says skip, we do not even read
            if gitignore.matched(&path, false).is_ignore() {
                debug!("Skipping ignored file: {rel_path}");
                return ignore::WalkState::Continue;
            }

            // Otherwise we send to processing thread
            processed_files_tx.send((path, rel_path)).ok();
            ignore::WalkState::Continue
        })
    });

    // Drop the sender so the thread can end
    drop(processed_files_tx);

    // Join the processing thread
    let mut processed_files = process_thread.join().unwrap();

    // Now assign file_index within each priority group
    let mut counters = HashMap::new();
    for f in &mut processed_files {
        let ctr = counters.entry(f.priority).or_insert(0);
        f.file_index = *ctr;
        *ctr += 1;
    }

    if config.debug {
        debug!(
            "Processed {} files in parallel for base_path: {}",
            processed_files.len(),
            base_path.display()
        );
    }

    // Sort by priority desc, then file_index
    processed_files.par_sort_by(|a, b| {
        a.priority
            .cmp(&b.priority)
            .reverse()
            .then_with(|| a.file_index.cmp(&b.file_index))
    });

    Ok(processed_files)
}

/// Create a relative, slash-normalized path
pub fn normalize_path(path: &Path, base: &Path) -> String {
    path.strip_prefix(base)
        .unwrap_or(path)
        .to_path_buf()
        .to_slash()
        .unwrap_or_default()
        .to_string()
}

pub fn process_files_parallel(
    base_path: &Path,
    config: &YekConfig,
    boost_map: &HashMap<String, i32>,
) -> Result<Vec<ProcessedFile>> {
    let mut expanded_paths = Vec::new();
    let path_str = base_path.to_string_lossy();
    for entry in glob(&path_str)? {
        match entry {
            Ok(path) => expanded_paths.push(path),
            Err(e) => debug!("Glob entry error: {:?}", e),
        }
    }
    if expanded_paths.len() == 1 && expanded_paths[0].is_file() {
        return process_single_file(&expanded_paths[0], config, boost_map);
    }
    let mut all_processed_files = Vec::new();
    for path in expanded_paths {
        if path.is_file() {
            all_processed_files.extend(process_single_file(&path, config, boost_map)?);
        } else if path.is_dir() {
            all_processed_files.extend(process_files_parallel_internal(&path, config, boost_map)?);
        }
    }
    Ok(all_processed_files)
}

use anyhow::{anyhow, Result};
use bytesize::ByteSize;
use clap_config_file::ClapConfigFile;
use sha2::{Digest, Sha256};
use std::io::IsTerminal;
use std::{fs, path::Path, str::FromStr, time::UNIX_EPOCH};

use crate::{
    defaults::{BINARY_FILE_EXTENSIONS, DEFAULT_IGNORE_PATTERNS, DEFAULT_OUTPUT_TEMPLATE},
    priority::PriorityRule,
};

#[derive(Clone, Debug, Default, clap::ValueEnum, serde::Serialize, serde::Deserialize)]
pub enum ConfigFormat {
    #[default]
    Toml,
    Yaml,
    Json,
}

#[derive(ClapConfigFile, Clone)]
#[config_file_name = "yek"]
#[config_file_formats = "toml,yaml,json"]
pub struct YekConfig {
    /// Input files and/or directories to process
    #[config_arg(positional)]
    pub input_paths: Vec<String>,

    /// Print version of yek
    #[config_arg(long = "version", short = 'V')]
    pub version: bool,

    /// Max size per chunk. e.g. "10MB" or "128K" or when using token counting mode, "100" or "128K"
    #[config_arg(default_value = "10MB")]
    pub max_size: String,

    /// Use token mode instead of byte mode
    #[config_arg()]
    pub tokens: String,

    /// Enable JSON output
    #[config_arg()]
    pub json: bool,

    /// Enable debug output
    #[config_arg()]
    pub debug: bool,

    /// Output directory. If none is provided & stdout is a TTY, we pick a temp dir
    #[config_arg()]
    pub output_dir: Option<String>,

    /// Output template. Defaults to ">>>> FILE_PATH\nFILE_CONTENT"
    #[config_arg(default_value = ">>>> FILE_PATH\nFILE_CONTENT")]
    pub output_template: String,

    /// Ignore patterns
    #[config_arg(long = "ignore-patterns", multi_value_behavior = "extend")]
    pub ignore_patterns: Vec<String>,

    /// Unignore patterns. Yek has some built-in ignore patterns, but you can override them here.
    #[config_arg(long = "unignore-patterns", multi_value_behavior = "extend")]
    pub unignore_patterns: Vec<String>,

    /// Priority rules
    #[config_arg(accept_from = "config_only")]
    pub priority_rules: Vec<PriorityRule>,

    /// Binary file extensions to ignore
    #[config_arg(accept_from = "config_only", default_value = BINARY_FILE_EXTENSIONS)]
    pub binary_extensions: Vec<String>,

    /// Maximum additional boost from Git commit times (0..1000)
    #[config_arg(accept_from = "config_only")]
    pub git_boost_max: Option<i32>,

    /// True if we should stream output to stdout (computed)
    pub stream: bool,

    /// True if we should count tokens, not bytes (computed)
    pub token_mode: bool,

    /// Final resolved output file path (only used if not streaming)
    pub output_file_full_path: Option<String>,

    /// Maximum depth to search for Git commit times
    #[config_arg(accept_from = "config_only", default_value = "100")]
    pub max_git_depth: i32,
}

/// Provide defaults so tests or other callers can create a baseline YekConfig easily.
impl Default for YekConfig {
    fn default() -> Self {
        Self {
            input_paths: Vec::new(),
            version: false,
            max_size: "10MB".to_string(),
            tokens: String::new(),
            json: false,
            debug: false,
            output_dir: None,
            output_template: DEFAULT_OUTPUT_TEMPLATE.to_string(),
            ignore_patterns: Vec::new(),
            unignore_patterns: Vec::new(),
            priority_rules: Vec::new(),
            binary_extensions: BINARY_FILE_EXTENSIONS
                .iter()
                .map(|s| s.to_string())
                .collect(),
            git_boost_max: Some(100),

            // computed fields
            stream: false,
            token_mode: false,
            output_file_full_path: None,
            max_git_depth: 100,
        }
    }
}

impl YekConfig {
    pub fn extend_config_with_defaults(input_paths: Vec<String>, output_dir: String) -> Self {
        YekConfig {
            input_paths,
            output_dir: Some(output_dir),
            ..Default::default()
        }
    }
}

impl YekConfig {
    /// Ensure output directory exists and is valid. Returns the resolved output directory path.
    pub fn ensure_output_dir(&self) -> Result<String> {
        if self.stream {
            return Ok(String::new());
        }

        let output_dir = if let Some(dir) = &self.output_dir {
            dir.clone()
        } else {
            let temp_dir = std::env::temp_dir().join("yek-output");
            temp_dir.to_string_lossy().to_string()
        };

        let path = Path::new(&output_dir);
        if path.exists() && !path.is_dir() {
            return Err(anyhow!(
                "output_dir: '{}' exists but is not a directory",
                output_dir
            ));
        }

        std::fs::create_dir_all(path)
            .map_err(|e| anyhow!("output_dir: cannot create '{}': {}", output_dir, e))?;

        Ok(output_dir)
    }

    /// Parse from CLI + config file, fill in computed fields, and validate.
    pub fn init_config() -> Self {
        // 1) parse from CLI and optional config file:
        let mut cfg = YekConfig::parse();

        // Handle version flag
        if cfg.version {
            println!("{}", env!("CARGO_PKG_VERSION"));
            std::process::exit(0);
        }

        // 2) compute derived fields:
        cfg.token_mode = !cfg.tokens.is_empty();
        let force_tty = std::env::var("FORCE_TTY").is_ok();

        cfg.stream = !std::io::stdout().is_terminal() && !force_tty;

        // default input dirs to current dir if none:
        if cfg.input_paths.is_empty() {
            cfg.input_paths.push(".".to_string());
        }

        // Extend binary extensions with the built-in list:
        let mut merged_bins = BINARY_FILE_EXTENSIONS
            .iter()
            .map(|s| s.to_string())
            .collect::<Vec<_>>();
        merged_bins.append(&mut cfg.binary_extensions);
        cfg.binary_extensions = merged_bins
            .into_iter()
            .collect::<std::collections::HashSet<_>>()
            .into_iter()
            .collect();

        // Always start with default ignore patterns, then add user's:
        let mut ignore = DEFAULT_IGNORE_PATTERNS
            .iter()
            .map(|s| s.to_string())
            .collect::<Vec<_>>();
        ignore.extend(cfg.ignore_patterns);
        cfg.ignore_patterns = ignore;

        // Apply unignore patterns (turn them into negative globs "!…")
        cfg.ignore_patterns
            .extend(cfg.unignore_patterns.iter().map(|pat| format!("!{}", pat)));

        // Handle output directory setup
        if !cfg.stream {
            match cfg.ensure_output_dir() {
                Ok(dir) => cfg.output_dir = Some(dir),
                Err(e) => {
                    eprintln!("Warning: Failed to create output directory: {}", e);
                    cfg.stream = true; // Fall back to streaming mode
                }
            }
        }

        // By default, we start with no final output_file_full_path:
        cfg.output_file_full_path = None;

        // 3) Validate
        if let Err(e) = cfg.validate() {
            eprintln!("Error: {}", e);
            std::process::exit(1);
        }

        cfg
    }

    /// Compute a quick checksum for the input paths (files and directories).
    /// For directories, it uses the top-level listing. For files, it uses the file metadata.
    pub fn get_checksum(input_paths: &[String]) -> String {
        let mut hasher = Sha256::new();
        for path_str in input_paths {
            let base_path = Path::new(path_str);
            if !base_path.exists() {
                continue;
            }

            // If it's a file, hash the file metadata directly
            if base_path.is_file() {
                if let Ok(meta) = fs::metadata(base_path) {
                    hasher.update(path_str.as_bytes());
                    hasher.update(meta.len().to_le_bytes());

                    if let Ok(mod_time) = meta.modified() {
                        if let Ok(dur) = mod_time.duration_since(UNIX_EPOCH) {
                            hasher.update(dur.as_secs().to_le_bytes());
                            hasher.update(dur.subsec_nanos().to_le_bytes());
                        }
                    }
                }
                continue;
            }

            // If it's a directory, hash its contents
            let entries = match fs::read_dir(base_path) {
                Ok(iter) => iter.filter_map(|e| e.ok()).collect::<Vec<_>>(),
                Err(_) => continue,
            };

            // Sort deterministically by path name
            let mut sorted = entries;
            sorted.sort_by_key(|a| a.path());

            for entry in sorted {
                let p = entry.path();
                if let Ok(meta) = fs::metadata(&p) {
                    let path_str = p.to_string_lossy();
                    hasher.update(path_str.as_bytes());
                    hasher.update(meta.len().to_le_bytes());

                    if let Ok(mod_time) = meta.modified() {
                        if let Ok(dur) = mod_time.duration_since(UNIX_EPOCH) {
                            hasher.update(dur.as_secs().to_le_bytes());
                            hasher.update(dur.subsec_nanos().to_le_bytes());
                        }
                    }
                }
            }
        }
        let result = hasher.finalize();
        // Convert the 32-byte result to hex, but only keep the first 8 characters
        let hex = format!("{:x}", result);
        hex[..8].to_owned()
    }

    /// Validate the final config.
    pub fn validate(&self) -> Result<()> {
        if !self.output_template.contains("FILE_PATH")
            || !self.output_template.contains("FILE_CONTENT")
        {
            return Err(anyhow!(
                "output_template: must contain FILE_PATH and FILE_CONTENT"
            ));
        }

        if self.max_size == "0" {
            return Err(anyhow!("max_size: cannot be 0"));
        }

        if !self.token_mode {
            ByteSize::from_str(&self.max_size)
                .map_err(|e| anyhow!("max_size: Invalid size format: {}", e))?;
        } else if self.tokens.to_lowercase().ends_with('k') {
            let val = self.tokens[..self.tokens.len() - 1]
                .trim()
                .parse::<usize>()
                .map_err(|e| anyhow!("tokens: Invalid token size: {}", e))?;
            if val == 0 {
                return Err(anyhow!("tokens: cannot be 0"));
            }
        } else if !self.tokens.is_empty() {
            // parse as integer
            let val = self
                .tokens
                .parse::<usize>()
                .map_err(|e| anyhow!("tokens: Invalid token size: {}", e))?;
            if val == 0 {
                return Err(anyhow!("tokens: cannot be 0"));
            }
        }

        // If not streaming, validate output directory
        if !self.stream {
            self.ensure_output_dir()?;
        }

        // Validate ignore patterns
        for pattern in &self.ignore_patterns {
            glob::Pattern::new(pattern)
                .map_err(|e| anyhow!("ignore_patterns: Invalid pattern '{}': {}", pattern, e))?;
        }

        // Validate priority rules
        for rule in &self.priority_rules {
            if rule.score < 0 || rule.score > 1000 {
                return Err(anyhow!(
                    "priority_rules: Priority score {} must be between 0 and 1000",
                    rule.score
                ));
            }
            glob::Pattern::new(&rule.pattern).map_err(|e| {
                anyhow!("priority_rules: Invalid pattern '{}': {}", rule.pattern, e)
            })?;
        }

        Ok(())
    }
}

pub fn process_files_parallel (
    base_path: &Path,
    config: &YekConfig,
    boost_map: &HashMap<String, i32>,
) -> Result<Vec<ProcessedFile>> {}

This code segment functions to: Expand input path patterns into individual files and directories, then recursively process their contents while applying priority calculations and ignoring specified patterns.

```rust
[TASK_BEGIN]
pub fn process_files_parallel(
    base_path: &Path,
    config: &YekConfig,
    boost_map: &HashMap<String, i32>,
) -> Result<Vec<ProcessedFile>> {
    if base_path.is_file() {
        process_single_file(base_path, config, boost_map)
    } else if base_path.is_dir() {
        process_files_parallel_internal(base_path, config, boost_map)
    } else {
        Ok(Vec::new())
    }
}
[TASK_END]
```


use anyhow::{anyhow, Result};
use bytesize::ByteSize;
use clap_config_file::ClapConfigFile;
use sha2::{Digest, Sha256};
use std::io::IsTerminal;
use std::{fs, path::Path, str::FromStr, time::UNIX_EPOCH};

use crate::{
    defaults::{BINARY_FILE_EXTENSIONS, DEFAULT_IGNORE_PATTERNS, DEFAULT_OUTPUT_TEMPLATE},
    priority::PriorityRule,
};

#[derive(Clone, Debug, Default, clap::ValueEnum, serde::Serialize, serde::Deserialize)]
pub enum ConfigFormat {
    #[default]
    Toml,
    Yaml,
    Json,
}

#[derive(ClapConfigFile, Clone)]
#[config_file_name = "yek"]
#[config_file_formats = "toml,yaml,json"]
pub struct YekConfig {
    /// Input files and/or directories to process
    #[config_arg(positional)]
    pub input_paths: Vec<String>,

    /// Print version of yek
    #[config_arg(long = "version", short = 'V')]
    pub version: bool,

    /// Max size per chunk. e.g. "10MB" or "128K" or when using token counting mode, "100" or "128K"
    #[config_arg(default_value = "10MB")]
    pub max_size: String,

    /// Use token mode instead of byte mode
    #[config_arg()]
    pub tokens: String,

    /// Enable JSON output
    #[config_arg()]
    pub json: bool,

    /// Enable debug output
    #[config_arg()]
    pub debug: bool,

    /// Output directory. If none is provided & stdout is a TTY, we pick a temp dir
    #[config_arg()]
    pub output_dir: Option<String>,

    /// Output template. Defaults to ">>>> FILE_PATH\nFILE_CONTENT"
    #[config_arg(default_value = ">>>> FILE_PATH\nFILE_CONTENT")]
    pub output_template: String,

    /// Ignore patterns
    #[config_arg(long = "ignore-patterns", multi_value_behavior = "extend")]
    pub ignore_patterns: Vec<String>,

    /// Unignore patterns. Yek has some built-in ignore patterns, but you can override them here.
    #[config_arg(long = "unignore-patterns", multi_value_behavior = "extend")]
    pub unignore_patterns: Vec<String>,

    /// Priority rules
    #[config_arg(accept_from = "config_only")]
    pub priority_rules: Vec<PriorityRule>,

    /// Binary file extensions to ignore
    #[config_arg(accept_from = "config_only", default_value = BINARY_FILE_EXTENSIONS)]
    pub binary_extensions: Vec<String>,

    /// Maximum additional boost from Git commit times (0..1000)
    #[config_arg(accept_from = "config_only")]
    pub git_boost_max: Option<i32>,

    /// True if we should stream output to stdout (computed)
    pub stream: bool,

    /// True if we should count tokens, not bytes (computed)
    pub token_mode: bool,

    /// Final resolved output file path (only used if not streaming)
    pub output_file_full_path: Option<String>,

    /// Maximum depth to search for Git commit times
    #[config_arg(accept_from = "config_only", default_value = "100")]
    pub max_git_depth: i32,
}

/// Provide defaults so tests or other callers can create a baseline YekConfig easily.
impl Default for YekConfig {
    fn default() -> Self {
        Self {
            input_paths: Vec::new(),
            version: false,
            max_size: "10MB".to_string(),
            tokens: String::new(),
            json: false,
            debug: false,
            output_dir: None,
            output_template: DEFAULT_OUTPUT_TEMPLATE.to_string(),
            ignore_patterns: Vec::new(),
            unignore_patterns: Vec::new(),
            priority_rules: Vec::new(),
            binary_extensions: BINARY_FILE_EXTENSIONS
                .iter()
                .map(|s| s.to_string())
                .collect(),
            git_boost_max: Some(100),

            // computed fields
            stream: false,
            token_mode: false,
            output_file_full_path: None,
            max_git_depth: 100,
        }
    }
}

impl YekConfig {
    pub fn extend_config_with_defaults(input_paths: Vec<String>, output_dir: String) -> Self {
        YekConfig {
            input_paths,
            output_dir: Some(output_dir),
            ..Default::default()
        }
    }
}

impl YekConfig {
    /// Ensure output directory exists and is valid. Returns the resolved output directory path.
    pub fn ensure_output_dir(&self) -> Result<String> {
        if self.stream {
            return Ok(String::new());
        }

        let output_dir = if let Some(dir) = &self.output_dir {
            dir.clone()
        } else {
            let temp_dir = std::env::temp_dir().join("yek-output");
            temp_dir.to_string_lossy().to_string()
        };

        let path = Path::new(&output_dir);
        if path.exists() && !path.is_dir() {
            return Err(anyhow!(
                "output_dir: '{}' exists but is not a directory",
                output_dir
            ));
        }

        std::fs::create_dir_all(path)
            .map_err(|e| anyhow!("output_dir: cannot create '{}': {}", output_dir, e))?;

        Ok(output_dir)
    }

    /// Parse from CLI + config file, fill in computed fields, and validate.

    /// Compute a quick checksum for the input paths (files and directories).
    /// For directories, it uses the top-level listing. For files, it uses the file metadata.
    pub fn get_checksum(input_paths: &[String]) -> String {
        let mut hasher = Sha256::new();
        for path_str in input_paths {
            let base_path = Path::new(path_str);
            if !base_path.exists() {
                continue;
            }

            // If it's a file, hash the file metadata directly
            if base_path.is_file() {
                if let Ok(meta) = fs::metadata(base_path) {
                    hasher.update(path_str.as_bytes());
                    hasher.update(meta.len().to_le_bytes());

                    if let Ok(mod_time) = meta.modified() {
                        if let Ok(dur) = mod_time.duration_since(UNIX_EPOCH) {
                            hasher.update(dur.as_secs().to_le_bytes());
                            hasher.update(dur.subsec_nanos().to_le_bytes());
                        }
                    }
                }
                continue;
            }

            // If it's a directory, hash its contents
            let entries = match fs::read_dir(base_path) {
                Ok(iter) => iter.filter_map(|e| e.ok()).collect::<Vec<_>>(),
                Err(_) => continue,
            };

            // Sort deterministically by path name
            let mut sorted = entries;
            sorted.sort_by_key(|a| a.path());

            for entry in sorted {
                let p = entry.path();
                if let Ok(meta) = fs::metadata(&p) {
                    let path_str = p.to_string_lossy();
                    hasher.update(path_str.as_bytes());
                    hasher.update(meta.len().to_le_bytes());

                    if let Ok(mod_time) = meta.modified() {
                        if let Ok(dur) = mod_time.duration_since(UNIX_EPOCH) {
                            hasher.update(dur.as_secs().to_le_bytes());
                            hasher.update(dur.subsec_nanos().to_le_bytes());
                        }
                    }
                }
            }
        }
        let result = hasher.finalize();
        // Convert the 32-byte result to hex, but only keep the first 8 characters
        let hex = format!("{:x}", result);
        hex[..8].to_owned()
    }

    /// Validate the final config.
    pub fn validate(&self) -> Result<()> {
        if !self.output_template.contains("FILE_PATH")
            || !self.output_template.contains("FILE_CONTENT")
        {
            return Err(anyhow!(
                "output_template: must contain FILE_PATH and FILE_CONTENT"
            ));
        }

        if self.max_size == "0" {
            return Err(anyhow!("max_size: cannot be 0"));
        }

        if !self.token_mode {
            ByteSize::from_str(&self.max_size)
                .map_err(|e| anyhow!("max_size: Invalid size format: {}", e))?;
        } else if self.tokens.to_lowercase().ends_with('k') {
            let val = self.tokens[..self.tokens.len() - 1]
                .trim()
                .parse::<usize>()
                .map_err(|e| anyhow!("tokens: Invalid token size: {}", e))?;
            if val == 0 {
                return Err(anyhow!("tokens: cannot be 0"));
            }
        } else if !self.tokens.is_empty() {
            // parse as integer
            let val = self
                .tokens
                .parse::<usize>()
                .map_err(|e| anyhow!("tokens: Invalid token size: {}", e))?;
            if val == 0 {
                return Err(anyhow!("tokens: cannot be 0"));
            }
        }

        // If not streaming, validate output directory
        if !self.stream {
            self.ensure_output_dir()?;
        }

        // Validate ignore patterns
        for pattern in &self.ignore_patterns {
            glob::Pattern::new(pattern)
                .map_err(|e| anyhow!("ignore_patterns: Invalid pattern '{}': {}", pattern, e))?;
        }

        // Validate priority rules
        for rule in &self.priority_rules {
            if rule.score < 0 || rule.score > 1000 {
                return Err(anyhow!(
                    "priority_rules: Priority score {} must be between 0 and 1000",
                    rule.score
                ));
            }
            glob::Pattern::new(&rule.pattern).map_err(|e| {
                anyhow!("priority_rules: Invalid pattern '{}': {}", rule.pattern, e)
            })?;
        }

        Ok(())
    }
}

pub fn init_config() -> Self {
        let mut cfg = YekConfig::parse();
        if cfg.version {
            println!("{}", env!("CARGO_PKG_VERSION"));
            std::process::exit(0);
        }
        cfg.token_mode = !cfg.tokens.is_empty();
        let force_tty = std::env::var("FORCE_TTY").is_ok();
        cfg.stream = !std::io::stdout().is_terminal() && !force_tty;
        if cfg.input_paths.is_empty() {
            cfg.input_paths.push(".".to_string());
        }
        let mut merged_bins = BINARY_FILE_EXTENSIONS
            .iter()
            .map(|s| s.to_string())
            .collect::<Vec<_>>();
        merged_bins.append(&mut cfg.binary_extensions);
        cfg.binary_extensions = merged_bins
            .into_iter()
            .collect::<std::collections::HashSet<_>>()
            .into_iter()
            .collect();
        let mut ignore = DEFAULT_IGNORE_PATTERNS
            .iter()
            .map(|s| s.to_string())
            .collect::<Vec<_>>();
        ignore.extend(cfg.ignore_patterns);
        cfg.ignore_patterns = ignore;
        cfg.ignore_patterns
            .extend(cfg.unignore_patterns.iter().map(|pat| format!("!{}", pat)));
        if !cfg.stream {
            match cfg.ensure_output_dir() {
                Ok(dir) => cfg.output_dir = Some(dir),
                Err(e) => {
                    eprintln!("Warning: Failed to create output directory: {}", e);
                    cfg.stream = true;
                }
            }
        }
        cfg.output_file_full_path = None;
        if let Err(e) = cfg.validate() {
            eprintln!("Error: {}", e);
            std::process::exit(1);
        }
        cfg
    }

This code segment functions to: Initialize application configuration by merging command-line inputs with default values and environment variables, validate user-provided parameters, handle version requests, determine output streaming behavior based on terminal detection, merge system-defined and custom patterns for file processing, and ensure proper error handling for invalid configurations.

```rust
[TASK_BEGIN]
    pub fn init_config() -> Self {
        let mut config = YekConfig::from_clap_config_file()
            .expect("Failed to parse CLI arguments and config file");

        // Compute streaming mode: stream if output_dir not specified AND stdout isn't a TTY
        config.stream = config.output_dir.is_none() && !std::io::stdout().is_terminal();

        // Detect token mode from tokens field
        config.token_mode = !config.tokens.is_empty();

        // Validate before returning
        config.validate().expect("Invalid configuration");

        config
    }
[TASK_END]
```

pub fn init_config() -> Self {
        let mut config = YekConfig::from_clap_config_file()
            .expect("Failed to parse CLI arguments and config file");
        config.stream = config.output_dir.is_none() && !std::io::stdout().is_terminal();
        config.token_mode = !config.tokens.is_empty();
        config.validate().expect("Invalid configuration");
        config
    }


This code segment functions to: Verify the integrity of configuration parameters by checking template validity, numerical constraints, directory existence, and pattern syntax while enforcing value ranges for priority scoring rules.

```rust
[TASK_BEGIN]
    pub fn validate(&self) -> Result<()> {
        // Validate input paths
        if self.input_paths.is_empty() {
            return Err(anyhow!("At least one input path is required"));
        }

        // Validate max_size format based on mode
        if self.token_mode {
            // Token mode validation: should be number[K/M]?
            let clean = self.max_size.trim().to_lowercase();
            let (digits, suffix) = clean.split_at(
                clean.find(|c: char| !c.is_ascii_digit())
                    .unwrap_or(clean.len()),
            );

            if digits.is_empty() {
                return Err(anyhow!("Invalid token max_size format '{}'", self.max_size));
            }

            digits.parse::<u64>()
                .map_err(|_| anyhow!("Invalid token max_size format '{}'", self.max_size))?;

            match suffix {
                "" | "k" | "m" => (),
                _ => return Err(anyhow!("Invalid token suffix '{}' in max_size", suffix)),
            }
        } else {
            // Byte mode validation
            ByteSize::from_str(&self.max_size)
                .map_err(|e| anyhow!("Invalid byte max_size format '{}': {}", self.max_size, e))?;
        }

        // Validate git boost range
        if let Some(boost) = self.git_boost_max {
            if !(0..=1000).contains(&boost) {
                return Err(anyhow!("git_boost_max must be between 0 and 1000"));
            }
        }

        // Validate output template placeholders
        if !self.output_template.contains("FILE_PATH") {
            return Err(anyhow!("output_template must contain FILE_PATH placeholder"));
        }

        if !self.output_template.contains("FILE_CONTENT") {
            return Err(anyhow!("output_template must contain FILE_CONTENT placeholder"));
        }

        Ok(())
    }
[TASK_END]
```

pub fn validate(&self) -> Result<()> {
        if self.input_paths.is_empty() {
            return Err(anyhow!("At least one input path is required"));
        }
        if self.token_mode {
            let clean = self.max_size.trim().to_lowercase();
            let (digits, suffix) = clean.split_at(
                clean.find(|c: char| !c.is_ascii_digit())
                    .unwrap_or(clean.len()),
            );
            if digits.is_empty() {
                return Err(anyhow!("Invalid token max_size format '{}'", self.max_size));
            }
            digits.parse::<u64>()
                .map_err(|_| anyhow!("Invalid token max_size format '{}'", self.max_size))?;
            match suffix {
                "" | "k" | "m" => (),
                _ => return Err(anyhow!("Invalid token suffix '{}' in max_size", suffix)),
            }
        } else {
            ByteSize::from_str(&self.max_size)
                .map_err(|e| anyhow!("Invalid byte max_size format '{}': {}", self.max_size, e))?;
        }
        if let Some(boost) = self.git_boost_max {
            if !(0..=1000).contains(&boost) {
                return Err(anyhow!("git_boost_max must be between 0 and 1000"));
            }
        }
        if !self.output_template.contains("FILE_PATH") {
            return Err(anyhow!("output_template must contain FILE_PATH placeholder"));
        }
        if !self.output_template.contains("FILE_CONTENT") {
            return Err(anyhow!("output_template must contain FILE_CONTENT placeholder"));
        }
        Ok(())
    }


use anyhow::anyhow;
use anyhow::Result;
use bytesize::ByteSize;
use content_inspector::{inspect, ContentType};
use rayon::prelude::*;
use std::{
    collections::HashMap,
    fs::File,
    io::{self, Read},
    path::Path,
    str::FromStr,
    sync::OnceLock,
};
use tiktoken_rs::CoreBPE;

pub mod config;
pub mod defaults;
pub mod parallel;
pub mod priority;

use config::YekConfig;
use parallel::{process_files_parallel, ProcessedFile};
use priority::compute_recentness_boost;

// Add a static BPE encoder for reuse
static TOKENIZER: OnceLock<CoreBPE> = OnceLock::new();

fn get_tokenizer() -> &'static CoreBPE {
    TOKENIZER.get_or_init(|| {
        tiktoken_rs::get_bpe_from_model("gpt-3.5-turbo").expect("Failed to load tokenizer")
    })
}

/// Check if a file is likely text or binary by reading only a small chunk.
/// This avoids reading large files fully just to detect their type.
pub fn is_text_file(path: &Path, user_binary_extensions: &[String]) -> io::Result<bool> {
    // If extension is known to be binary, skip quickly
    if let Some(ext) = path.extension().and_then(|e| e.to_str()) {
        if user_binary_extensions.iter().any(|bin_ext| bin_ext == ext) {
            return Ok(false);
        }
    }

    // Short partial read to check if it's binary or text
    const INSPECTION_BYTES: usize = 8192;
    let mut file = File::open(path)?;
    let mut buf = vec![0u8; INSPECTION_BYTES];
    let n = file.read(&mut buf)?;
    buf.truncate(n);

    Ok(inspect(&buf) != ContentType::BINARY)
}

/// Main entrypoint for serialization, used by CLI and tests
pub fn serialize_repo(config: &YekConfig) -> Result<(String, Vec<ProcessedFile>)> {
    // Gather commit times from each input path that is a directory
    let combined_commit_times = config
        .input_paths
        .par_iter()
        .filter_map(|path_str| {
            let repo_path = Path::new(path_str);
            if repo_path.is_dir() {
                priority::get_recent_commit_times_git2(
                    repo_path,
                    config.max_git_depth.try_into().unwrap_or(0),
                )
            } else {
                None
            }
        })
        .flatten()
        .collect::<HashMap<String, u64>>();

    // Compute a recentness-based boost
    let recentness_boost =
        compute_recentness_boost(&combined_commit_times, config.git_boost_max.unwrap_or(100));

    // Process files in parallel for each input path
    let merged_files = config
        .input_paths
        .par_iter()
        .map(|path_str| {
            let path = Path::new(path_str);
            process_files_parallel(path, config, &recentness_boost)
        })
        .collect::<Result<Vec<Vec<ProcessedFile>>>>()?
        .into_iter()
        .flatten()
        .collect::<Vec<ProcessedFile>>();

    let mut files = merged_files;

    // Sort final (priority asc, then file_index asc)
    files.par_sort_by(|a, b| {
        a.priority
            .cmp(&b.priority)
            .then_with(|| a.rel_path.cmp(&b.rel_path))
    });

    // Build the final output string
    let output_string = concat_files(&files, config)?;

    // Only count tokens if debug logging is enabled
    if tracing::Level::DEBUG <= tracing::level_filters::STATIC_MAX_LEVEL {
        tracing::debug!("{} tokens generated", count_tokens(&output_string));
    }

    Ok((output_string, files))
}

pub fn concat_files(files: &[ProcessedFile], config: &YekConfig) -> anyhow::Result<String> {
    let mut accumulated = 0_usize;
    let cap = if config.token_mode {
        parse_token_limit(&config.tokens)?
    } else {
        ByteSize::from_str(&config.max_size)
            .map_err(|e| anyhow!("max_size: Invalid size format: {}", e))?
            .as_u64() as usize
    };

    // Sort by priority (asc) and file_index (asc)
    let mut sorted_files: Vec<_> = files.iter().collect();
    sorted_files.sort_by(|a, b| {
        a.priority
            .cmp(&b.priority)
            .then_with(|| a.rel_path.cmp(&b.rel_path))
    });

    let mut files_to_include = Vec::new();
    for file in sorted_files {
        let content_size = if config.token_mode {
            // Format the file content with template first, then count tokens
            let formatted = if config.json {
                serde_json::to_string(&serde_json::json!({
                    "filename": &file.rel_path,
                    "content": &file.content,
                }))
                .map_err(|e| anyhow!("Failed to serialize JSON: {}", e))?
            } else {
                config
                    .output_template
                    .replace("FILE_PATH", &file.rel_path)
                    .replace("FILE_CONTENT", &file.content)
            };
            count_tokens(&formatted)
        } else {
            file.content.len()
        };

        if accumulated + content_size <= cap {
            accumulated += content_size;
            files_to_include.push(file);
        } else {
            break;
        }
    }

    if config.json {
        // JSON array of objects
        Ok(serde_json::to_string_pretty(
            &files_to_include
                .iter()
                .map(|f| {
                    serde_json::json!({
                        "filename": &f.rel_path,
                        "content": &f.content,
                    })
                })
                .collect::<Vec<_>>(),
        )?)
    } else {
        // Use the user-defined template
        Ok(files_to_include
            .iter()
            .map(|f| {
                config
                    .output_template
                    .replace("FILE_PATH", &f.rel_path)
                    .replace("FILE_CONTENT", &f.content)
                    // Handle both literal "\n" and escaped "\\n"
                    .replace("\\\\\n", "\n") // First handle escaped newline
                    .replace("\\\\n", "\n") // Then handle escaped \n sequence
            })
            .collect::<Vec<_>>()
            .join("\n"))
    }
}

/// Parse a token limit string like "800k" or "1000" into a number

/// Count tokens using tiktoken's GPT-3.5-Turbo tokenizer for accuracy
pub fn count_tokens(text: &str) -> usize {
    get_tokenizer().encode_with_special_tokens(text).len()
}

pub fn parse_token_limit(limit: &str) -> anyhow::Result<usize> {
    if limit.to_lowercase().ends_with('k') {
        limit[..limit.len() - 1]
            .trim()
            .parse::<usize>()
            .map(|n| n * 1000)
            .map_err(|e| anyhow!("tokens: Invalid token size: {}", e))
    } else {
        limit
            .parse::<usize>()
            .map_err(|e| anyhow!("tokens: Invalid token size: {}", e))
    }
}

pub fn parse_token_limit (limit: &str) -> anyhow::Result<usize> {}

This code segment functions to: Convert a string-based token limit specification into its numerical equivalent, supporting optional magnitude suffixes for scaled value interpretation.

```rust
[TASK_BEGIN]
pub fn parse_token_limit(limit: &str) -> anyhow::Result<usize> {
    let s = limit.trim().to_lowercase();
    if s.is_empty() {
        return Err(anyhow!("Empty token limit string"));
    }

    let last_char = s.chars().last().unwrap(); // Safe since we checked empty
    match last_char {
        'k' | 'm' => {
            let num_str = &s[..s.len() - 1];
            if num_str.is_empty() {
                return Err(anyhow!("Missing numeric value before suffix '{}'", last_char));
            }
            let num: f64 = num_str.parse()
                .map_err(|e| anyhow!("Invalid numeric format '{}': {}", num_str, e))?;

            let factor = match last_char {
                'k' => 1e3,
                'm' => 1e6,
                _ => unreachable!(),
            };

            let total = num * factor;
            if total.fract() != 0.0 {
                return Err(anyhow!(
                    "Non-integer token limit after applying suffix: {} * {} = {}",
                    num,
                    factor,
                    total
                ));
            }

            if total < 0.0 {
                return Err(anyhow!("Token limit cannot be negative"));
            }

            Ok(total as usize)
        }
        _ => {
            s.parse::<usize>()
                .map_err(|e| anyhow!("Invalid token limit format '{}': {}", s, e))
        }
    }
}
[TASK_END]
```

pub fn parse_token_limit(limit: &str) -> anyhow::Result<usize> {
    let s = limit.trim().to_lowercase();
    if s.is_empty() {
        return Err(anyhow!("Empty token limit string"));
    }
    let last_char = s.chars().last().unwrap();
    match last_char {
        'k' | 'm' => {
            let num_str = &s[..s.len() - 1];
            if num_str.is_empty() {
                return Err(anyhow!("Missing numeric value before suffix '{}'", last_char));
            }
            let num: f64 = num_str.parse()
                .map_err(|e| anyhow!("Invalid numeric format '{}': {}", num_str, e))?;
            let factor = match last_char {
                'k' => 1e3,
                'm' => 1e6,
                _ => unreachable!(),
            };
            let total = num * factor;
            if total.fract() != 0.0 {
                return Err(anyhow!(
                    "Non-integer token limit after applying suffix: {} * {} = {}",
                    num,
                    factor,
                    total
                ));
            }
            if total < 0.0 {
                return Err(anyhow!("Token limit cannot be negative"));
            }
            Ok(total as usize)
        }
        _ => {
            s.parse::<usize>()
                .map_err(|e| anyhow!("Invalid token limit format '{}': {}", s, e))
        }
    }
}