examples/ios/DolphinCoreMLTestApp/Sources/Knowledge/KnowledgeEngine.swift

import Foundation

enum KnowledgeKind: String, CaseIterable, Sendable, Codable, Equatable, Hashable {
  case fact
  case preference
  case decision
  case goal
  case code
  case context
  case insight
}

enum KnowledgeSource: String, CaseIterable, Sendable, Codable, Equatable, Hashable {
  case user
  case assistant
  case synthesis
  case `import`
}

enum KnowledgeInsightPriority: String, CaseIterable, Sendable, Codable, Equatable, Hashable {
  case high
  case medium
  case low
}

struct KnowledgeItem: Identifiable, Sendable, Codable, Equatable {
  let id: UUID
  var kind: KnowledgeKind
  var title: String
  var content: String
  var tags: [String]
  var relatedItemIDs: [UUID]
  var source: KnowledgeSource
  let createdAt: Date
  var updatedAt: Date
  var revision: Int

  init(
    id: UUID = UUID(),
    kind: KnowledgeKind,
    title: String,
    content: String,
    tags: [String] = [],
    relatedItemIDs: [UUID] = [],
    source: KnowledgeSource,
    createdAt: Date = Date(),
    updatedAt: Date? = nil,
    revision: Int = 1
  ) {
    let safeContent = KnowledgeTextSanitizer.text(
      content,
      maxCharacters: KnowledgeEngine.Limits.maxItemContentCharacters
    )
    let safeTitle = KnowledgeTextSanitizer.text(
      title,
      maxCharacters: KnowledgeEngine.Limits.maxItemTitleCharacters,
      multiline: false
    )

    self.id = id
    self.kind = kind
    self.title = safeTitle.isEmpty
      ? KnowledgeTextSanitizer.fallbackTitle(from: safeContent)
      : safeTitle
    self.content = safeContent.isEmpty ? self.title : safeContent
    self.tags = KnowledgeTextSanitizer.tags(tags)
    self.relatedItemIDs = KnowledgeTextSanitizer.uniqueIDs(
      relatedItemIDs,
      excluding: id,
      limit: KnowledgeEngine.Limits.maxRelatedItemCount
    )
    self.source = source
    self.createdAt = createdAt
    self.updatedAt = max(updatedAt ?? createdAt, createdAt)
    self.revision = min(max(revision, 1), KnowledgeEngine.Limits.maxRevision)
  }

  private enum CodingKeys: String, CodingKey {
    case id
    case kind
    case title
    case content
    case tags
    case relatedItemIDs
    case source
    case createdAt
    case updatedAt
    case revision
  }

  init(from decoder: Decoder) throws {
    let container = try decoder.container(keyedBy: CodingKeys.self)
    let id = try container.decode(UUID.self, forKey: .id)
    let createdAt = try container.decode(Date.self, forKey: .createdAt)
    self.init(
      id: id,
      kind: try container.decode(KnowledgeKind.self, forKey: .kind),
      title: try container.decodeIfPresent(String.self, forKey: .title) ?? "",
      content: try container.decodeIfPresent(String.self, forKey: .content) ?? "",
      tags: try container.decodeIfPresent([String].self, forKey: .tags) ?? [],
      relatedItemIDs: try container.decodeIfPresent([UUID].self, forKey: .relatedItemIDs) ?? [],
      source: try container.decodeIfPresent(KnowledgeSource.self, forKey: .source) ?? .import,
      createdAt: createdAt,
      updatedAt: try container.decodeIfPresent(Date.self, forKey: .updatedAt),
      revision: try container.decodeIfPresent(Int.self, forKey: .revision) ?? 1
    )
  }
}

struct SessionHandoff: Identifiable, Sendable, Codable, Equatable {
  let id: UUID
  var title: String
  var focus: String
  var summary: String
  var nextSteps: [String]
  var knowledgeItemIDs: [UUID]
  var taskIDs: [UUID]
  let createdAt: Date
  var restoredAt: Date?

  init(
    id: UUID = UUID(),
    title: String,
    focus: String,
    summary: String,
    nextSteps: [String] = [],
    knowledgeItemIDs: [UUID] = [],
    taskIDs: [UUID] = [],
    createdAt: Date = Date(),
    restoredAt: Date? = nil
  ) {
    let safeFocus = KnowledgeTextSanitizer.text(
      focus,
      maxCharacters: KnowledgeEngine.Limits.maxHandoffFocusCharacters
    )
    let safeTitle = KnowledgeTextSanitizer.text(
      title,
      maxCharacters: KnowledgeEngine.Limits.maxItemTitleCharacters,
      multiline: false
    )

    self.id = id
    self.title = safeTitle.isEmpty ? "Dolphin session handoff" : safeTitle
    self.focus = safeFocus
    self.summary = KnowledgeTextSanitizer.text(
      summary,
      maxCharacters: KnowledgeEngine.Limits.maxHandoffSummaryCharacters
    )
    self.nextSteps = KnowledgeTextSanitizer.uniqueTexts(
      nextSteps,
      itemLimit: KnowledgeEngine.Limits.maxNextStepCharacters,
      countLimit: KnowledgeEngine.Limits.maxNextStepCount
    )
    self.knowledgeItemIDs = KnowledgeTextSanitizer.uniqueIDs(
      knowledgeItemIDs,
      limit: KnowledgeEngine.Limits.maxHandoffKnowledgeCount
    )
    self.taskIDs = KnowledgeTextSanitizer.uniqueIDs(
      taskIDs,
      limit: KnowledgeEngine.Limits.maxHandoffTaskCount
    )
    self.createdAt = createdAt
    self.restoredAt = restoredAt.map { max($0, createdAt) }
  }

  /// Removes only the copied chat section from a checkpoint. All durable
  /// fields and historical references remain byte-for-byte unchanged.
  func removingRecentConversationContext() -> SessionHandoff {
    let lines = summary.split(
      separator: "\n",
      omittingEmptySubsequences: false
    ).map(String.init)
    guard let recentIndex = lines.firstIndex(where: Self.isRecentContextLine)
    else { return self }

    let durableIndex = lines.indices.reversed().first { index in
      index > recentIndex && Self.isDurableContextLine(lines[index])
    }
    let openTasksIndex = lines.indices.reversed().first { index in
      index > recentIndex && Self.isOpenTasksLine(lines[index])
    }
    // Generated handoffs always emit Durable context when knowledge IDs are
    // present and Open tasks when task IDs are present. Selecting the last
    // expected marker prevents an untrusted multiline chat message from
    // forging an earlier section heading that survives the scrub.
    let retainedSectionIndex =
      (!knowledgeItemIDs.isEmpty ? durableIndex : nil)
      ?? (!taskIDs.isEmpty ? openTasksIndex : nil)
    var retainedLines = Array(lines[..<recentIndex])
    if let retainedSectionIndex {
      retainedLines.append(contentsOf: lines[retainedSectionIndex...])
    }

    var copy = self
    copy.summary = KnowledgeTextSanitizer.text(
      retainedLines.joined(separator: "\n"),
      maxCharacters: KnowledgeEngine.Limits.maxHandoffSummaryCharacters
    )
    return copy
  }

  private static func isRecentContextLine(_ line: String) -> Bool {
    line.trimmingCharacters(in: .whitespaces)
      .lowercased()
      .hasPrefix("recent context:")
  }

  private static func isDurableContextLine(_ line: String) -> Bool {
    let normalized = line.trimmingCharacters(in: .whitespaces).lowercased()
    return normalized.hasPrefix("durable context:")
  }

  private static func isOpenTasksLine(_ line: String) -> Bool {
    let normalized = line.trimmingCharacters(in: .whitespaces).lowercased()
    return normalized.hasPrefix("open tasks:")
  }
}

struct KnowledgeDigest: Identifiable, Sendable, Codable, Equatable {
  let id: UUID
  var summary: String
  var priorities: [String]
  var openQuestions: [String]
  var evidenceItemIDs: [UUID]
  let createdAt: Date
  var trigger: String

  init(
    id: UUID = UUID(),
    summary: String,
    priorities: [String],
    openQuestions: [String],
    evidenceItemIDs: [UUID],
    createdAt: Date = Date(),
    trigger: String
  ) {
    self.id = id
    self.summary = KnowledgeTextSanitizer.text(
      summary,
      maxCharacters: KnowledgeEngine.Limits.maxDigestCharacters
    )
    self.priorities = KnowledgeTextSanitizer.uniqueTexts(
      priorities,
      itemLimit: KnowledgeEngine.Limits.maxPriorityCharacters,
      countLimit: KnowledgeEngine.Limits.maxPriorityCount
    )
    self.openQuestions = KnowledgeTextSanitizer.uniqueTexts(
      openQuestions,
      itemLimit: KnowledgeEngine.Limits.maxQuestionCharacters,
      countLimit: KnowledgeEngine.Limits.maxQuestionCount
    )
    self.evidenceItemIDs = KnowledgeTextSanitizer.uniqueIDs(
      evidenceItemIDs,
      limit: KnowledgeEngine.Limits.maxDigestEvidenceCount
    )
    self.createdAt = createdAt
    self.trigger = KnowledgeTextSanitizer.text(
      trigger,
      maxCharacters: KnowledgeEngine.Limits.maxTriggerCharacters,
      multiline: false
    )
  }
}

struct ProactiveKnowledgeInsight: Identifiable, Sendable, Codable, Equatable {
  let id: UUID
  var title: String
  var detail: String
  var evidenceItemIDs: [UUID]
  var priority: KnowledgeInsightPriority

  init(
    id: UUID = UUID(),
    title: String,
    detail: String,
    evidenceItemIDs: [UUID] = [],
    priority: KnowledgeInsightPriority
  ) {
    self.id = id
    self.title = KnowledgeTextSanitizer.text(
      title,
      maxCharacters: KnowledgeEngine.Limits.maxInsightTitleCharacters,
      multiline: false
    )
    self.detail = KnowledgeTextSanitizer.text(
      detail,
      maxCharacters: KnowledgeEngine.Limits.maxInsightDetailCharacters
    )
    self.evidenceItemIDs = KnowledgeTextSanitizer.uniqueIDs(
      evidenceItemIDs,
      limit: KnowledgeEngine.Limits.maxInsightEvidenceCount
    )
    self.priority = priority
  }
}

enum CodeFindingKind: String, CaseIterable, Sendable, Codable, Equatable, Hashable {
  case todo
  case forceUnwrap
  case forcedTry
  case unsafeCast
  case longLine
  case deepNesting
  case repeatedPattern
}

enum CodeFindingSeverity: String, CaseIterable, Sendable, Codable, Equatable, Hashable {
  case warning
  case note
}

struct CodeFinding: Sendable, Codable, Equatable {
  var kind: CodeFindingKind
  var severity: CodeFindingSeverity
  var line: Int?
  var message: String
  var evidence: String

  init(
    kind: CodeFindingKind,
    severity: CodeFindingSeverity,
    line: Int? = nil,
    message: String,
    evidence: String = ""
  ) {
    self.kind = kind
    self.severity = severity
    self.line = line.map { max(1, $0) }
    self.message = KnowledgeTextSanitizer.text(message, maxCharacters: 220, multiline: false)
    self.evidence = KnowledgeTextSanitizer.text(evidence, maxCharacters: 180, multiline: false)
  }
}

struct CodeAnalysis: Sendable, Codable, Equatable {
  var language: String
  var analyzedCharacterCount: Int
  var lineCount: Int
  var nonemptyLineCount: Int
  var maxNestingApproximation: Int
  var findings: [CodeFinding]
  var repeatedPatterns: [String]
  var wasTruncated: Bool

  init(
    language: String,
    analyzedCharacterCount: Int,
    lineCount: Int,
    nonemptyLineCount: Int,
    maxNestingApproximation: Int,
    findings: [CodeFinding],
    repeatedPatterns: [String],
    wasTruncated: Bool
  ) {
    self.language = KnowledgeTextSanitizer.tag(language, fallback: "unknown")
    self.analyzedCharacterCount = min(max(analyzedCharacterCount, 0), KnowledgeEngine.Limits.maxCodeCharacters)
    self.lineCount = max(lineCount, 0)
    self.nonemptyLineCount = min(max(nonemptyLineCount, 0), self.lineCount)
    self.maxNestingApproximation = min(max(maxNestingApproximation, 0), 100)
    self.findings = Array(findings.prefix(KnowledgeEngine.Limits.maxCodeFindings))
    self.repeatedPatterns = KnowledgeTextSanitizer.uniqueTexts(
      repeatedPatterns,
      itemLimit: 220,
      countLimit: KnowledgeEngine.Limits.maxRepeatedPatterns
    )
    self.wasTruncated = wasTruncated
  }
}

struct KnowledgeDeletionCascade: Sendable, Equatable {
  let handoffIDs: Set<UUID>
  let digestIDs: Set<UUID>

  let removesAllDigests: Bool
}

struct KnowledgeEngine: Sendable {
  enum Limits {
    static let maxCorpusItems = 500
    static let maxItemTitleCharacters = 120
    static let maxItemContentCharacters = 4_000
    static let maxTagCount = 12
    static let maxTagCharacters = 32
    static let maxRelatedItemCount = 24
    static let maxRevision = 1_000_000
    static let maxQueryCharacters = 256
    static let maxSearchResults = 20
    static let maxDigestCharacters = 4_000
    static let maxPriorityCount = 10
    static let maxPriorityCharacters = 180
    static let maxQuestionCount = 8
    static let maxQuestionCharacters = 240
    static let maxDigestEvidenceCount = 40
    static let maxTriggerCharacters = 120
    static let maxHandoffFocusCharacters = 500
    static let maxHandoffSummaryCharacters = 2_400
    static let maxNextStepCount = 12
    static let maxNextStepCharacters = 180
    static let maxHandoffKnowledgeCount = 32
    static let maxHandoffTaskCount = 24
    static let maxInsightCount = 12
    static let maxInsightTitleCharacters = 120
    static let maxInsightDetailCharacters = 600
    static let maxInsightEvidenceCount = 16
    static let maxPromptMemories = 8
    static let maxPromptMemoryCharacters = 700
    static let maxLegacyMemoryCharacters = 1_500
    static let maxTaskTitleCharacters = 300
    static let maxCodeCharacters = 5_000
    static let maxCodeFindings = 24
    static let maxRepeatedPatterns = 8
  }

  /// Applies the same deterministic safety normalization used by
  /// `KnowledgeItem` before a local-write approval is displayed. This keeps
  /// the approved arguments identical to the values that will be persisted.
  static func canonicalCaptureFields(
    title: String,
    content: String,
    tags: [String],
    relatedItemIDs: [UUID]
  ) -> (
    title: String,
    content: String,
    tags: [String],
    relatedItemIDs: [UUID]
  ) {
    let safeContent = KnowledgeTextSanitizer.text(
      content,
      maxCharacters: Limits.maxItemContentCharacters
    )
    let safeTitle = KnowledgeTextSanitizer.text(
      title,
      maxCharacters: Limits.maxItemTitleCharacters,
      multiline: false
    )
    let canonicalTitle = safeTitle.isEmpty
      ? KnowledgeTextSanitizer.fallbackTitle(from: safeContent)
      : safeTitle

    return (
      title: canonicalTitle,
      content: safeContent.isEmpty ? canonicalTitle : safeContent,
      tags: KnowledgeTextSanitizer.tags(tags),
      relatedItemIDs: KnowledgeTextSanitizer.uniqueIDs(
        relatedItemIDs,
        limit: Limits.maxRelatedItemCount
      )
    )
  }

  static func canonicalMemoryContent(_ content: String) -> String {
    KnowledgeTextSanitizer.text(
      content,
      maxCharacters: Limits.maxLegacyMemoryCharacters,
      multiline: false
    )
  }

  static func canonicalTaskTitle(_ title: String) -> String {
    KnowledgeTextSanitizer.text(
      title,
      maxCharacters: Limits.maxTaskTitleCharacters,
      multiline: false
    )
  }

  /// Canonicalizes the user-authored portion of a handoff before approval.
  /// The derived summary and evidence snapshot are intentionally produced at
  /// execution time from the then-current typed workspace.
  static func canonicalHandoffFields(
    title: String,
    focus: String
  ) -> (title: String, focus: String) {
    let safeTitle = KnowledgeTextSanitizer.text(
      title,
      maxCharacters: Limits.maxItemTitleCharacters,
      multiline: false
    )
    return (
      title: safeTitle.isEmpty ? "Dolphin session handoff" : safeTitle,
      focus: KnowledgeTextSanitizer.text(
        focus,
        maxCharacters: Limits.maxHandoffFocusCharacters
      )
    )
  }

  /// Plans the complete derived-state cascade before mutation so the UI and
  /// persistence layer disclose and execute the same scope. Digests do not
  /// yet record handoff provenance, so removing any linked handoff requires a
  /// conservative removal of every digest that could contain its focus.
  func deletionCascade(
    deletingKnowledgeIDs knowledgeIDs: Set<UUID>,
    handoffs: [SessionHandoff],
    digests: [KnowledgeDigest]
  ) -> KnowledgeDeletionCascade {
    let handoffIDs = Set(handoffs.compactMap { handoff in
      knowledgeIDs.isDisjoint(with: handoff.knowledgeItemIDs)
        ? nil
        : handoff.id
    })
    let removesAllDigests = !handoffIDs.isEmpty
    let digestIDs = removesAllDigests
      ? Set(digests.map(\.id))
      : Set(digests.compactMap { digest in
          knowledgeIDs.isDisjoint(with: digest.evidenceItemIDs)
            ? nil
            : digest.id
        })
    return KnowledgeDeletionCascade(
      handoffIDs: handoffIDs,
      digestIDs: digestIDs,
      removesAllDigests: removesAllDigests
    )
  }

  static func searchableKnowledge(
    typedItems: [KnowledgeItem],
    legacyMemories: [MemoryItem]
  ) -> [KnowledgeItem] {
    let capturedContents = Set(typedItems.map { Self.normalized($0.content) })
    let legacyItems = legacyMemories.compactMap { memory -> KnowledgeItem? in
      guard !capturedContents.contains(Self.normalized(memory.content)) else {
        return nil
      }
      return KnowledgeItem(
        id: memory.id,
        kind: .fact,
        title: "Saved memory",
        content: memory.content,
        tags: ["memory"],
        source: .import,
        createdAt: memory.createdAt
      )
    }
    return typedItems + legacyItems
  }

  func search(
    query rawQuery: String,
    in allItems: [KnowledgeItem],
    kinds: Set<KnowledgeKind>? = nil,
    limit requestedLimit: Int = 8
  ) -> [KnowledgeItem] {
    let items = Array(allItems.prefix(Limits.maxCorpusItems)).filter { item in
      kinds?.contains(item.kind) ?? true
    }
    let limit = min(max(requestedLimit, 0), Limits.maxSearchResults)
    guard limit > 0, !items.isEmpty else { return [] }

    let query = KnowledgeTextSanitizer.text(
      rawQuery,
      maxCharacters: Limits.maxQueryCharacters,
      multiline: false
    )
    let normalizedQuery = Self.normalized(query)
    let queryTokens = Self.meaningfulTokens(in: normalizedQuery)

    guard !normalizedQuery.isEmpty else {
      return Array(items.sorted(by: Self.knowledgeTieBreak).prefix(limit))
    }

    let documents = items.map(Self.indexedDocument)
    let averageLength = max(
      Double(documents.reduce(0) { $0 + $1.tokens.count }) / Double(documents.count),
      1
    )
    let corpusCount = Double(documents.count)
    var documentFrequency: [String: Int] = [:]
    for token in queryTokens {
      documentFrequency[token] = documents.reduce(into: 0) { count, document in
        if document.tokenSet.contains(token) { count += 1 }
      }
    }

    let scored: [(item: KnowledgeItem, score: Double)] = documents.compactMap { document in
      var bm25 = 0.0
      var exactMatches = 0
      for token in queryTokens {
        let frequency = document.weightedFrequency[token] ?? 0
        guard frequency > 0 else { continue }
        exactMatches += 1
        let df = Double(documentFrequency[token] ?? 0)
        let idf = log(1 + ((corpusCount - df + 0.5) / (df + 0.5)))
        let lengthNormalization = 1.2 * (0.25 + 0.75 * Double(document.tokens.count) / averageLength)
        bm25 += idf * ((frequency * 2.2) / (frequency + lengthNormalization))
      }

      let coverage = queryTokens.isEmpty
        ? 0
        : Double(exactMatches) / Double(queryTokens.count)
      let phraseScore: Double
      if document.normalizedTitle == normalizedQuery {
        phraseScore = 4
      } else if document.normalizedTitle.contains(normalizedQuery) {
        phraseScore = 3
      } else if document.normalizedText.contains(normalizedQuery) {
        phraseScore = 2
      } else {
        phraseScore = 0
      }

      let fuzzyScore = Self.fuzzyTokenCoverage(
        queryTokens: queryTokens,
        documentTokens: document.uniqueTokens
      )
      let score = bm25 + (coverage * 2.4) + phraseScore + (fuzzyScore * 1.6)
      return score >= 0.28 ? (document.item, score) : nil
    }

    return scored.sorted { lhs, rhs in
      if abs(lhs.score - rhs.score) > 0.000_001 {
        return lhs.score > rhs.score
      }
      return Self.knowledgeTieBreak(lhs.item, rhs.item)
    }
    .prefix(limit)
    .map(\.item)
  }

  /// Reuses the local lexical ranker for task titles while preserving task
  /// identity and deterministic created-at/UUID tie breaking.
  func searchTasks(
    query: String,
    in allTasks: [AssistantTaskItem],
    includeCompleted: Bool = false,
    limit requestedLimit: Int = 10
  ) -> [AssistantTaskItem] {
    let tasks = allTasks.filter { includeCompleted || !$0.isCompleted }
    let limit = min(max(requestedLimit, 0), 20)
    guard limit > 0, !tasks.isEmpty else { return [] }

    let proxyItems = tasks.map { task in
      KnowledgeItem(
        id: task.id,
        kind: .goal,
        title: task.title,
        content: task.title,
        tags: task.isCompleted ? ["completed-task"] : ["open-task"],
        source: .synthesis,
        createdAt: task.createdAt,
        updatedAt: task.completedAt ?? task.createdAt
      )
    }
    let tasksByID = Dictionary(
      uniqueKeysWithValues: tasks.map { ($0.id, $0) }
    )
    return search(
      query: query,
      in: proxyItems,
      kinds: [.goal],
      limit: limit
    ).compactMap { tasksByID[$0.id] }
  }

  func synthesizeContext(
    knowledge allItems: [KnowledgeItem],
    tasks allTasks: [AssistantTaskItem],
    handoff: SessionHandoff? = nil,
    trigger rawTrigger: String = "manual",
    createdAt: Date = Date(),
    maxCharacters requestedMaxCharacters: Int = 2_400
  ) -> KnowledgeDigest {
    let items = Array(allItems.prefix(Limits.maxCorpusItems)).sorted(by: Self.knowledgeTieBreak)
    let tasks = Array(allTasks.prefix(Limits.maxHandoffTaskCount)).sorted(by: Self.taskOrder)
    let maxCharacters = min(max(requestedMaxCharacters, 256), Limits.maxDigestCharacters)
    let trigger = KnowledgeTextSanitizer.text(
      rawTrigger,
      maxCharacters: Limits.maxTriggerCharacters,
      multiline: false
    )

    var sections: [String] = []
    if let handoff {
      let focus = handoff.focus.isEmpty ? handoff.summary : handoff.focus
      if !focus.isEmpty {
        sections.append("Current focus: \(Self.excerpt(focus, limit: 360))")
      }
    }

    let decisions = Array(items.filter { $0.kind == .decision }.prefix(4))
    if !decisions.isEmpty {
      sections.append("Decisions: " + decisions.map(Self.compactDescription).joined(separator: "; "))
    }

    let preferences = Array(items.filter { $0.kind == .preference }.prefix(4))
    if !preferences.isEmpty {
      sections.append("Preferences: " + preferences.map(Self.compactDescription).joined(separator: "; "))
    }

    let goals = Array(items.filter { $0.kind == .goal }.prefix(4))
    if !goals.isEmpty {
      sections.append("Goals: " + goals.map(Self.compactDescription).joined(separator: "; "))
    }

    let supporting = Array(
      items.filter { ![.decision, .preference, .goal].contains($0.kind) }
        .prefix(6)
    )
    if !supporting.isEmpty {
      sections.append("Known context: " + supporting.map(Self.compactDescription).joined(separator: "; "))
    }

    let openTasks = tasks.filter { !$0.isCompleted }.prefix(8)
    if !openTasks.isEmpty {
      sections.append("Open tasks: " + openTasks.map { Self.excerpt($0.title, limit: 140) }.joined(separator: "; "))
    }

    if sections.isEmpty {
      sections.append("No durable project knowledge or open tasks are available yet.")
    }

    let summary = KnowledgeTextSanitizer.text(
      sections.joined(separator: "\n"),
      maxCharacters: maxCharacters
    )

    var priorities = openTasks.map(\.title)
    priorities.append(contentsOf: goals.map { Self.compactDescription($0) })
    if let handoff {
      priorities.insert(contentsOf: handoff.nextSteps, at: 0)
    }
    priorities = KnowledgeTextSanitizer.uniqueTexts(
      priorities,
      itemLimit: Limits.maxPriorityCharacters,
      countLimit: Limits.maxPriorityCount
    )

    var openQuestions: [String] = []
    var questionEvidence: [UUID] = []
    var seenQuestions = Set<String>()
    for item in items where item.content.contains("?") {
      let question = KnowledgeTextSanitizer.text(
        Self.questionExcerpt(item.content),
        maxCharacters: Limits.maxQuestionCharacters
      )
      let key = Self.normalized(question)
      guard
        !question.isEmpty,
        !key.isEmpty,
        seenQuestions.insert(key).inserted
      else { continue }
      openQuestions.append(question)
      questionEvidence.append(item.id)
      if openQuestions.count == Limits.maxQuestionCount { break }
    }
    let evidenceIDs = KnowledgeTextSanitizer.uniqueIDs(
      (decisions + preferences + goals + supporting).map(\.id)
        + questionEvidence,
      limit: Limits.maxDigestEvidenceCount
    )
    let identifierSeed = ([summary, trigger] + priorities + openQuestions
      + evidenceIDs.map(\.uuidString)).joined(separator: "|")

    return KnowledgeDigest(
      id: Self.stableUUID(identifierSeed),
      summary: summary,
      priorities: priorities,
      openQuestions: openQuestions,
      evidenceItemIDs: evidenceIDs,
      createdAt: createdAt,
      trigger: trigger
    )
  }

  func makeHandoff(
    title: String = "Dolphin session handoff",
    focus rawFocus: String,
    knowledge allItems: [KnowledgeItem],
    tasks allTasks: [AssistantTaskItem],
    recentMessages: [String] = [],
    nextSteps explicitNextSteps: [String] = [],
    createdAt: Date = Date()
  ) -> SessionHandoff {
    let items = Array(allItems.prefix(Limits.maxCorpusItems)).sorted(by: Self.knowledgeTieBreak)
    let tasks = Array(allTasks.prefix(Limits.maxHandoffTaskCount)).sorted(by: Self.taskOrder)
    let focus = KnowledgeTextSanitizer.text(
      rawFocus,
      maxCharacters: Limits.maxHandoffFocusCharacters
    )
    var summaryParts: [String] = []
    if !focus.isEmpty {
      summaryParts.append("Focus: \(focus)")
    }

    let safeMessages = KnowledgeTextSanitizer.uniqueTexts(
      Array(recentMessages.suffix(5)).map {
        KnowledgeTextSanitizer.text(
          $0,
          maxCharacters: 280,
          multiline: false
        )
      },
      itemLimit: 280,
      countLimit: 5
    )
    if !safeMessages.isEmpty {
      summaryParts.append("Recent context: " + safeMessages.joined(separator: " | "))
    }

    let durableContext = items.prefix(8).map(Self.compactDescription)
    if !durableContext.isEmpty {
      summaryParts.append("Durable context: " + durableContext.joined(separator: "; "))
    }

    let openTasks = tasks.filter { !$0.isCompleted }
    if !openTasks.isEmpty {
      summaryParts.append("Open tasks: " + openTasks.prefix(8).map(\.title).joined(separator: "; "))
    }

    let summary = KnowledgeTextSanitizer.text(
      summaryParts.isEmpty ? "No active session context was supplied." : summaryParts.joined(separator: "\n"),
      maxCharacters: Limits.maxHandoffSummaryCharacters
    )
    let nextSteps = KnowledgeTextSanitizer.uniqueTexts(
      explicitNextSteps + openTasks.map(\.title),
      itemLimit: Limits.maxNextStepCharacters,
      countLimit: Limits.maxNextStepCount
    )
    let knowledgeIDs = KnowledgeTextSanitizer.uniqueIDs(
      items.map(\.id),
      limit: Limits.maxHandoffKnowledgeCount
    )
    let taskIDs = KnowledgeTextSanitizer.uniqueIDs(
      openTasks.map(\.id),
      limit: Limits.maxHandoffTaskCount
    )
    let identifierSeed = ([title, focus, summary] + nextSteps
      + knowledgeIDs.map(\.uuidString) + taskIDs.map(\.uuidString)).joined(separator: "|")

    return SessionHandoff(
      id: Self.stableUUID(identifierSeed),
      title: title,
      focus: focus,
      summary: summary,
      nextSteps: nextSteps,
      knowledgeItemIDs: knowledgeIDs,
      taskIDs: taskIDs,
      createdAt: createdAt
    )
  }

  func deriveProactiveInsights(
    knowledge allItems: [KnowledgeItem],
    tasks allTasks: [AssistantTaskItem],
    handoff: SessionHandoff? = nil,
    now: Date = Date()
  ) -> [ProactiveKnowledgeInsight] {
    let items = Array(allItems.prefix(Limits.maxCorpusItems))
    let tasks = Array(allTasks.prefix(Limits.maxHandoffTaskCount))
    var insights: [ProactiveKnowledgeInsight] = []

    let grouped = Dictionary(grouping: items) { item in
      "\(item.kind.rawValue)|\(Self.normalized(item.title))"
    }
    for group in grouped.values where group.count > 1 {
      let distinctContents = Set(group.map { Self.normalized($0.content) })
      guard distinctContents.count > 1 else { continue }
      let sortedGroup = group.sorted(by: Self.knowledgeTieBreak)
      let title = "Review conflicting \(sortedGroup[0].kind.rawValue) knowledge"
      let detail = "Multiple saved items named “\(Self.excerpt(sortedGroup[0].title, limit: 80))” contain different information. Reconcile or version them before relying on either one."
      insights.append(Self.insight(
        title: title,
        detail: detail,
        evidence: sortedGroup.map(\.id),
        priority: .high
      ))
    }

    let openTasks = tasks.filter { !$0.isCompleted }.sorted(by: Self.taskOrder)
    for task in openTasks.prefix(4) {
      let taskTokens = Set(Self.meaningfulTokens(in: Self.normalized(task.title)))
      let bestCoverage = items.map { item -> Double in
        guard !taskTokens.isEmpty else { return 0 }
        let itemTokens = Set(Self.meaningfulTokens(
          in: Self.normalized("\(item.title) \(item.content) \(item.tags.joined(separator: " "))")
        ))
        return Double(taskTokens.intersection(itemTokens).count) / Double(taskTokens.count)
      }.max() ?? 0
      if bestCoverage < 0.5 {
        insights.append(Self.insight(
          title: "Capture context for an open task",
          detail: "“\(Self.excerpt(task.title, limit: 120))” has little supporting knowledge. Capture constraints, decisions, or evidence before the next session.",
          evidence: [],
          priority: .medium
        ))
      }
    }

    if let handoff,
       !handoff.nextSteps.isEmpty,
       now.timeIntervalSince(handoff.createdAt) > 7 * 24 * 60 * 60 {
      insights.append(Self.insight(
        title: "Refresh the session handoff",
        detail: "The current handoff is more than seven days old and still contains next steps. Confirm what remains active before restoring it.",
        evidence: handoff.knowledgeItemIDs,
        priority: .medium
      ))
    }

    let knownIDs = Set(items.map(\.id))
    let orphaned = items.filter { item in
      item.relatedItemIDs.contains { !knownIDs.contains($0) }
    }
    if !orphaned.isEmpty {
      insights.append(Self.insight(
        title: "Repair missing knowledge relationships",
        detail: "Some knowledge relationships point to items that are no longer available. Remove or reconnect those references.",
        evidence: orphaned.map(\.id),
        priority: .low
      ))
    }

    let untagged = items.filter(\.tags.isEmpty)
    if untagged.count >= 3 {
      insights.append(Self.insight(
        title: "Tag uncategorized knowledge",
        detail: "\(untagged.count) items have no tags. A few stable project tags will improve exact retrieval and future synthesis.",
        evidence: untagged.map(\.id),
        priority: .low
      ))
    }

    return Array(insights.sorted { lhs, rhs in
      let lhsRank = Self.insightPriority(lhs.priority)
      let rhsRank = Self.insightPriority(rhs.priority)
      if lhsRank != rhsRank { return lhsRank > rhsRank }
      if lhs.title != rhs.title { return lhs.title < rhs.title }
      return lhs.id.uuidString < rhs.id.uuidString
    }.prefix(Limits.maxInsightCount))
  }

  func promptContext(
    knowledge items: [KnowledgeItem],
    query: String,
    limit requestedLimit: Int = 5
  ) -> [MemoryItem] {
    let limit = min(max(requestedLimit, 0), Limits.maxPromptMemories)
    guard limit > 0 else { return [] }
    return search(query: query, in: items, limit: limit).map { item in
      let text = KnowledgeTextSanitizer.text(
        "[\(item.kind.rawValue)] \(item.title): \(item.content)",
        maxCharacters: Limits.maxPromptMemoryCharacters
      )
      return MemoryItem(id: item.id, content: text, createdAt: item.updatedAt)
    }
  }

  func analyzeCode(_ rawSnippet: String, language rawLanguage: String = "unknown") -> CodeAnalysis {
    let sanitized = KnowledgeTextSanitizer.code(rawSnippet, maxCharacters: Limits.maxCodeCharacters)
    let lines = sanitized.text.isEmpty
      ? []
      : sanitized.text.split(separator: "\n", omittingEmptySubsequences: false).map(String.init)
    let nonemptyCount = lines.reduce(into: 0) { count, line in
      if !line.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { count += 1 }
    }
    let language = KnowledgeTextSanitizer.tag(rawLanguage, fallback: "unknown")
    var findings: [CodeFinding] = []
    var braceDepth = 0
    var maximumBraceDepth = 0
    var indentationStack = [0]
    var maximumIndentationDepth = 0
    var normalizedLineCounts: [String: (count: Int, firstLine: Int, sample: String)] = [:]

    for (offset, line) in lines.enumerated() {
      let lineNumber = offset + 1
      let trimmed = line.trimmingCharacters(in: .whitespacesAndNewlines)
      guard !trimmed.isEmpty else { continue }

      let indentation = line.prefix { $0 == " " || $0 == "\t" }.reduce(into: 0) { count, character in
        count += character == "\t" ? 2 : 1
      }
      while indentation < (indentationStack.last ?? 0), indentationStack.count > 1 {
        indentationStack.removeLast()
      }
      if indentation > (indentationStack.last ?? 0) {
        indentationStack.append(indentation)
      }
      maximumIndentationDepth = max(maximumIndentationDepth, indentationStack.count - 1)

      let normalizedLine = Self.normalized(trimmed)
      if normalizedLine.count >= 8,
         !["{", "}", "(", ")", "[", "]"].contains(normalizedLine) {
        let current = normalizedLineCounts[normalizedLine]
        normalizedLineCounts[normalizedLine] = (
          (current?.count ?? 0) + 1,
          current?.firstLine ?? lineNumber,
          current?.sample ?? Self.excerpt(trimmed, limit: 140)
        )
      }

      if trimmed.range(of: #"\b(TODO|FIXME)\b"#, options: [.regularExpression, .caseInsensitive]) != nil {
        findings.append(CodeFinding(
          kind: .todo,
          severity: .note,
          line: lineNumber,
          message: "Unresolved TODO or FIXME marker.",
          evidence: trimmed
        ))
      }
      if trimmed.range(of: #"\btry\s*!"#, options: .regularExpression) != nil {
        findings.append(CodeFinding(
          kind: .forcedTry,
          severity: .warning,
          line: lineNumber,
          message: "Forced try can terminate the process when the operation throws.",
          evidence: trimmed
        ))
      }
      if trimmed.range(of: #"\bas\s*!"#, options: .regularExpression) != nil {
        findings.append(CodeFinding(
          kind: .unsafeCast,
          severity: .warning,
          line: lineNumber,
          message: "Forced cast can terminate the process when the value has another type.",
          evidence: trimmed
        ))
      }
      let withoutForcedOperators = trimmed
        .replacingOccurrences(of: #"\btry\s*!"#, with: "try", options: .regularExpression)
        .replacingOccurrences(of: #"\bas\s*!"#, with: "as", options: .regularExpression)
      if withoutForcedOperators.range(
        of: #"(?:[A-Za-z_][A-Za-z0-9_]*|\]|\))!(?!=)"#,
        options: .regularExpression
      ) != nil {
        findings.append(CodeFinding(
          kind: .forceUnwrap,
          severity: .warning,
          line: lineNumber,
          message: "Force unwrap can terminate the process when the optional is nil.",
          evidence: trimmed
        ))
      }
      if line.count > 120 {
        findings.append(CodeFinding(
          kind: .longLine,
          severity: .note,
          line: lineNumber,
          message: "Line is \(line.count) characters; consider splitting it for readability.",
          evidence: trimmed
        ))
      }

      for character in Self.codeCharactersOutsideSimpleStrings(line) {
        switch character {
        case "{", "[", "(":
          braceDepth += 1
          maximumBraceDepth = max(maximumBraceDepth, braceDepth)
        case "}", "]", ")":
          braceDepth = max(0, braceDepth - 1)
        default:
          break
        }
      }
    }

    let maximumNesting = max(maximumBraceDepth, maximumIndentationDepth)
    if maximumNesting >= 5 {
      findings.append(CodeFinding(
        kind: .deepNesting,
        severity: .warning,
        message: "Approximate nesting depth reached \(maximumNesting); extract smaller units or use early exits."
      ))
    }

    let repeated = normalizedLineCounts.values
      .filter { $0.count >= 2 }
      .sorted { lhs, rhs in
        if lhs.count != rhs.count { return lhs.count > rhs.count }
        if lhs.firstLine != rhs.firstLine { return lhs.firstLine < rhs.firstLine }
        return lhs.sample < rhs.sample
      }
      .prefix(Limits.maxRepeatedPatterns)
      .map { "Repeated \($0.count)x: \($0.sample)" }
    for pattern in repeated {
      findings.append(CodeFinding(
        kind: .repeatedPattern,
        severity: .note,
        message: "A normalized line is repeated and may be extractable.",
        evidence: pattern
      ))
    }

    return CodeAnalysis(
      language: language,
      analyzedCharacterCount: sanitized.text.count,
      lineCount: lines.count,
      nonemptyLineCount: nonemptyCount,
      maxNestingApproximation: maximumNesting,
      findings: findings,
      repeatedPatterns: repeated,
      wasTruncated: sanitized.wasTruncated
    )
  }
}

private extension KnowledgeEngine {
  struct IndexedDocument {
    let item: KnowledgeItem
    let normalizedTitle: String
    let normalizedText: String
    let tokens: [String]
    let uniqueTokens: [String]
    let tokenSet: Set<String>
    let weightedFrequency: [String: Double]
  }

  static let stopWords: Set<String> = [
    "a", "an", "and", "are", "as", "at", "be", "by", "for", "from", "in", "is", "it",
    "of", "on", "or", "that", "the", "this", "to", "was", "were", "with",
  ]

  static func indexedDocument(_ item: KnowledgeItem) -> IndexedDocument {
    let normalizedTitle = normalized(item.title)
    let normalizedContent = normalized(item.content)
    let normalizedTags = item.tags.map(normalized).joined(separator: " ")
    let titleTokens = tokens(in: normalizedTitle)
    let contentTokens = tokens(in: normalizedContent)
    let tagTokens = tokens(in: normalizedTags)
    let allTokens = titleTokens + contentTokens + tagTokens
    var weightedFrequency: [String: Double] = [:]
    for token in titleTokens { weightedFrequency[token, default: 0] += 2.5 }
    for token in contentTokens { weightedFrequency[token, default: 0] += 1 }
    for token in tagTokens { weightedFrequency[token, default: 0] += 2 }
    return IndexedDocument(
      item: item,
      normalizedTitle: normalizedTitle,
      normalizedText: [normalizedTitle, normalizedContent, normalizedTags].joined(separator: " "),
      tokens: allTokens,
      uniqueTokens: Array(Set(allTokens)).sorted(),
      tokenSet: Set(allTokens),
      weightedFrequency: weightedFrequency
    )
  }

  static func normalized(_ value: String) -> String {
    let folded = value.folding(
      options: [.caseInsensitive, .diacriticInsensitive, .widthInsensitive],
      locale: Locale(identifier: "en_US_POSIX")
    )
    let scalars = folded.unicodeScalars.map { scalar -> Unicode.Scalar in
      CharacterSet.alphanumerics.contains(scalar) ? scalar : " "
    }
    return String(String.UnicodeScalarView(scalars))
      .split(whereSeparator: \Character.isWhitespace)
      .joined(separator: " ")
  }

  static func tokens(in normalizedText: String) -> [String] {
    normalizedText.split(separator: " ").map(String.init)
  }

  static func meaningfulTokens(in normalizedText: String) -> [String] {
    let allTokens = tokens(in: normalizedText)
    let filtered = allTokens.filter { !stopWords.contains($0) }
    let selected = filtered.isEmpty ? allTokens : filtered
    var seen = Set<String>()
    return selected.filter { seen.insert($0).inserted }
  }

  static func fuzzyTokenCoverage(queryTokens: [String], documentTokens: [String]) -> Double {
    guard !queryTokens.isEmpty, !documentTokens.isEmpty else { return 0 }
    let total = queryTokens.reduce(0.0) { sum, queryToken in
      let best = documentTokens.reduce(0.0) { current, documentToken in
        max(current, trigramSimilarity(queryToken, documentToken))
      }
      return sum + (best >= 0.38 ? best : 0)
    }
    return total / Double(queryTokens.count)
  }

  static func trigramSimilarity(_ lhs: String, _ rhs: String) -> Double {
    if lhs == rhs { return 1 }
    if min(lhs.count, rhs.count) < 3 {
      return lhs.hasPrefix(rhs) || rhs.hasPrefix(lhs) ? 0.65 : 0
    }
    let left = trigrams(lhs)
    let right = trigrams(rhs)
    guard !left.isEmpty, !right.isEmpty else { return 0 }
    return (2 * Double(left.intersection(right).count)) / Double(left.count + right.count)
  }

  static func trigrams(_ value: String) -> Set<String> {
    let characters = Array("  \(value)  ")
    guard characters.count >= 3 else { return [value] }
    return Set((0...(characters.count - 3)).map { index in
      String(characters[index...index + 2])
    })
  }

  static func kindRank(_ kind: KnowledgeKind) -> Int {
    switch kind {
    case .decision: 7
    case .preference: 6
    case .goal: 5
    case .fact: 4
    case .code: 3
    case .context: 2
    case .insight: 1
    }
  }

  static func knowledgeTieBreak(_ lhs: KnowledgeItem, _ rhs: KnowledgeItem) -> Bool {
    let lhsRank = kindRank(lhs.kind)
    let rhsRank = kindRank(rhs.kind)
    if lhsRank != rhsRank { return lhsRank > rhsRank }
    if lhs.updatedAt != rhs.updatedAt { return lhs.updatedAt > rhs.updatedAt }
    if lhs.title != rhs.title { return lhs.title < rhs.title }
    return lhs.id.uuidString < rhs.id.uuidString
  }

  static func taskOrder(_ lhs: AssistantTaskItem, _ rhs: AssistantTaskItem) -> Bool {
    if lhs.isCompleted != rhs.isCompleted { return !lhs.isCompleted }
    if lhs.createdAt != rhs.createdAt { return lhs.createdAt < rhs.createdAt }
    if lhs.title != rhs.title { return lhs.title < rhs.title }
    return lhs.id.uuidString < rhs.id.uuidString
  }

  static func compactDescription(_ item: KnowledgeItem) -> String {
    let title = excerpt(item.title, limit: 90)
    let content = excerpt(item.content, limit: 220)
    if normalized(title) == normalized(content) { return title }
    return "\(title) — \(content)"
  }

  static func excerpt(_ value: String, limit: Int) -> String {
    KnowledgeTextSanitizer.text(value, maxCharacters: limit, multiline: false)
  }

  static func questionExcerpt(_ value: String) -> String {
    let pieces = value.split(separator: "?", omittingEmptySubsequences: true)
    guard let first = pieces.first else { return "" }
    return excerpt(String(first) + "?", limit: Limits.maxQuestionCharacters)
  }

  static func insight(
    title: String,
    detail: String,
    evidence: [UUID],
    priority: KnowledgeInsightPriority
  ) -> ProactiveKnowledgeInsight {
    let safeEvidence = KnowledgeTextSanitizer.uniqueIDs(
      evidence,
      limit: Limits.maxInsightEvidenceCount
    )
    let seed = ([title, detail, priority.rawValue] + safeEvidence.map(\.uuidString)).joined(separator: "|")
    return ProactiveKnowledgeInsight(
      id: stableUUID(seed),
      title: title,
      detail: detail,
      evidenceItemIDs: safeEvidence,
      priority: priority
    )
  }

  static func insightPriority(_ priority: KnowledgeInsightPriority) -> Int {
    switch priority {
    case .high: 3
    case .medium: 2
    case .low: 1
    }
  }

  static func stableUUID(_ value: String) -> UUID {
    var first: UInt64 = 14_695_981_039_346_656_037
    var second: UInt64 = 7_804_984_196_043_216_021
    for byte in value.utf8 {
      first = (first ^ UInt64(byte)) &* 1_099_511_628_211
      second = (second ^ UInt64(byte &+ 31)) &* 1_099_511_628_211
    }
    var bytes = [UInt8](repeating: 0, count: 16)
    for index in 0..<8 {
      bytes[index] = UInt8(truncatingIfNeeded: first >> UInt64(index * 8))
      bytes[index + 8] = UInt8(truncatingIfNeeded: second >> UInt64(index * 8))
    }
    bytes[6] = (bytes[6] & 0x0F) | 0x50
    bytes[8] = (bytes[8] & 0x3F) | 0x80
    return UUID(uuid: (
      bytes[0], bytes[1], bytes[2], bytes[3],
      bytes[4], bytes[5], bytes[6], bytes[7],
      bytes[8], bytes[9], bytes[10], bytes[11],
      bytes[12], bytes[13], bytes[14], bytes[15]
    ))
  }

  static func codeCharactersOutsideSimpleStrings(_ line: String) -> [Character] {
    var result: [Character] = []
    var quote: Character?
    var escaped = false
    for character in line {
      if escaped {
        escaped = false
        continue
      }
      if character == "\\" {
        escaped = quote != nil
        continue
      }
      if character == "\"" || character == "'" {
        if quote == character {
          quote = nil
        } else if quote == nil {
          quote = character
        }
        continue
      }
      if quote == nil { result.append(character) }
    }
    return result
  }
}

private enum KnowledgeTextSanitizer {
  static let unsafeDelimiters = [
    "<tool_call>", "</tool_call>",
    "<tool_response>", "</tool_response>",
    "<assistant>", "</assistant>",
    "<system>", "</system>",
  ]

  static func text(_ raw: String, maxCharacters: Int, multiline: Bool = true) -> String {
    guard maxCharacters > 0 else { return "" }
    var value = neutralized(raw.precomposedStringWithCanonicalMapping)
    if !multiline {
      value = value
        .replacingOccurrences(of: "\r\n", with: " ")
        .replacingOccurrences(of: "\r", with: " ")
        .replacingOccurrences(of: "\n", with: " ")
        .replacingOccurrences(of: "\t", with: " ")
    }
    let scalars = value.unicodeScalars.filter { scalar in
      if scalar == "\n" || scalar == "\t" { return multiline }
      return !CharacterSet.controlCharacters.contains(scalar)
        && scalar.value != 0x200B
        && scalar.value != 0x200C
        && scalar.value != 0x200D
        && scalar.value != 0x2060
        && scalar.value != 0xFEFF
    }
    value = String(String.UnicodeScalarView(scalars))
      .replacingOccurrences(of: "\r\n", with: "\n")
      .replacingOccurrences(of: "\r", with: "\n")

    if multiline {
      value = value.split(separator: "\n", omittingEmptySubsequences: false)
        .map { line in
          line.split(whereSeparator: \Character.isWhitespace).joined(separator: " ")
        }
        .joined(separator: "\n")
      while value.contains("\n\n\n") {
        value = value.replacingOccurrences(of: "\n\n\n", with: "\n\n")
      }
    } else {
      value = value.split(whereSeparator: \Character.isWhitespace).joined(separator: " ")
    }

    value = value.trimmingCharacters(in: .whitespacesAndNewlines)
    return String(value.prefix(maxCharacters))
  }

  static func code(_ raw: String, maxCharacters: Int) -> (text: String, wasTruncated: Bool) {
    var value = neutralized(raw.precomposedStringWithCanonicalMapping)
      .replacingOccurrences(of: "\r\n", with: "\n")
      .replacingOccurrences(of: "\r", with: "\n")
    let scalars = value.unicodeScalars.filter { scalar in
      scalar == "\n" || scalar == "\t" || !CharacterSet.controlCharacters.contains(scalar)
    }
    value = String(String.UnicodeScalarView(scalars))
    let wasTruncated = value.count > maxCharacters
    return (String(value.prefix(maxCharacters)), wasTruncated)
  }

  static func neutralized(_ raw: String) -> String {
    var value = raw
    for delimiter in unsafeDelimiters {
      value = value.replacingOccurrences(
        of: delimiter,
        with: delimiter.replacingOccurrences(of: "<", with: "[").replacingOccurrences(of: ">", with: "]"),
        options: .caseInsensitive
      )
    }
    value = value
      .replacingOccurrences(of: "<|", with: "[special:")
      .replacingOccurrences(of: "|>", with: "]")
    return value
  }

  static func fallbackTitle(from content: String) -> String {
    let fallback = text(content, maxCharacters: KnowledgeEngine.Limits.maxItemTitleCharacters, multiline: false)
    return fallback.isEmpty ? "Untitled knowledge" : fallback
  }

  static func tags(_ rawTags: [String]) -> [String] {
    var seen = Set<String>()
    var result: [String] = []
    for rawTag in rawTags {
      let safe = tag(rawTag)
      guard !safe.isEmpty, seen.insert(safe).inserted else { continue }
      result.append(safe)
      if result.count == KnowledgeEngine.Limits.maxTagCount { break }
    }
    return result
  }

  static func tag(_ raw: String, fallback: String = "") -> String {
    let folded = neutralized(raw).folding(
      options: [.caseInsensitive, .diacriticInsensitive, .widthInsensitive],
      locale: Locale(identifier: "en_US_POSIX")
    )
    var output = ""
    var previousWasSeparator = false
    for scalar in folded.unicodeScalars {
      if CharacterSet.alphanumerics.contains(scalar) {
        output.unicodeScalars.append(scalar)
        previousWasSeparator = false
      } else if !previousWasSeparator, !output.isEmpty {
        output.append("-")
        previousWasSeparator = true
      }
      if output.count >= KnowledgeEngine.Limits.maxTagCharacters { break }
    }
    let safe = output.trimmingCharacters(in: CharacterSet(charactersIn: "-"))
    return safe.isEmpty ? fallback : safe
  }

  static func uniqueTexts(_ rawValues: [String], itemLimit: Int, countLimit: Int) -> [String] {
    var seen = Set<String>()
    var result: [String] = []
    for rawValue in rawValues {
      let safe = text(rawValue, maxCharacters: itemLimit)
      let key = KnowledgeEngine.normalized(safe)
      guard !safe.isEmpty, !key.isEmpty, seen.insert(key).inserted else { continue }
      result.append(safe)
      if result.count == countLimit { break }
    }
    return result
  }

  static func uniqueIDs(
    _ values: [UUID],
    excluding excludedID: UUID? = nil,
    limit: Int
  ) -> [UUID] {
    var seen = Set<UUID>()
    var result: [UUID] = []
    for value in values where value != excludedID && seen.insert(value).inserted {
      result.append(value)
      if result.count == limit { break }
    }
    return result
  }
}