// Copyright (c) Epic Games Tools
// Licensed under the MIT license (https://opensource.org/license/mit/)

internal LNK_Symbol *
lnk_make_symbol(Arena *arena, String8 name, LNK_Obj *obj, U32 symbol_idx)
{
  LNK_ObjSymbolRefNode *ref = push_array(arena, LNK_ObjSymbolRefNode, 1);
  ref->v.obj                = obj;
  ref->v.symbol_idx         = symbol_idx;

  LNK_Symbol *symbol = push_array(arena, LNK_Symbol, 1);
  symbol->name       = name;
  SLLQueuePush(symbol->first_ref, symbol->last_ref, ref);

  return symbol;
}

internal int
lnk_obj_symbol_ref_is_before(void *raw_a, void *raw_b)
{
  LNK_ObjSymbolRef *a_ref           = raw_a;
  LNK_ObjSymbolRef *b_ref           = raw_b;
  LNK_Lib          *a_lib           = lnk_obj_get_lib(a_ref->obj);
  LNK_Lib          *b_lib           = lnk_obj_get_lib(b_ref->obj);
  U32               a_lib_input_idx = a_lib ? a_lib->input_idx : 0;
  U32               b_lib_input_idx = b_lib ? b_lib->input_idx : 0;
  if (a_lib_input_idx == b_lib_input_idx) {
    if (a_ref->obj->input_idx == b_ref->obj->input_idx) {
      return a_ref->symbol_idx < b_ref->symbol_idx;
    }
    return a_ref->obj->input_idx < b_ref->obj->input_idx;
  }
  return a_lib_input_idx < b_lib_input_idx;
}

internal int
lnk_obj_symbol_ref_ptr_is_before(void *raw_a, void *raw_b)
{
  LNK_ObjSymbolRef **a = raw_a, **b = raw_b;
  return lnk_obj_symbol_ref_is_before(*a, *b);
}

internal int
lnk_symbol_is_before(void *raw_a, void *raw_b)
{
  LNK_Symbol *a = raw_a, *b = raw_b;
  LNK_ObjSymbolRef a_ref = lnk_ref_from_symbol(a);
  LNK_ObjSymbolRef b_ref = lnk_ref_from_symbol(b);
  return lnk_obj_symbol_ref_is_before(&a_ref, &b_ref);
}

internal int
lnk_symbol_ptr_is_before(void *raw_a, void *raw_b)
{
  return lnk_symbol_is_before(*(LNK_Symbol **)raw_a, *(LNK_Symbol **)raw_b);
}

internal void
lnk_symbol_list_push_node(LNK_SymbolList *list, LNK_SymbolNode *node)
{
  SLLQueuePush(list->first, list->last, node);
  list->count += 1;
}

internal LNK_SymbolNode *
lnk_symbol_list_push(Arena *arena, LNK_SymbolList *list, LNK_Symbol *symbol)
{
  LNK_SymbolNode *node = push_array(arena, LNK_SymbolNode, 1);
  node->data           = symbol;
  lnk_symbol_list_push_node(list, node);
  return node;
}

internal LNK_SymbolHashTrie *
lnk_symbol_hash_trie_chunk_list_push(Arena *arena, LNK_SymbolHashTrieChunkList *list, U64 cap)
{
  if (list->last == 0 || list->last->count >= list->last->cap) {
    LNK_SymbolHashTrieChunk *chunk = push_array(arena, LNK_SymbolHashTrieChunk, 1);
    chunk->cap                     = cap;
    chunk->v                       = push_array_no_zero(arena, LNK_SymbolHashTrie, cap);
    SLLQueuePush(list->first, list->last, chunk);
    ++list->count;
  }

  LNK_SymbolHashTrie *result = &list->last->v[list->last->count++];
  return result;
}

internal void
lnk_symbol_hash_trie_chunk_list_concat_in_place(LNK_SymbolHashTrieChunkList *list, LNK_SymbolHashTrieChunkList *to_concat)
{
  SLLConcatInPlace(list, to_concat);
}

internal void
lnk_error_multiply_defined_symbol(LNK_Symbol *dst, LNK_Symbol *src)
{
  LNK_ObjSymbolRef dst_ref = lnk_ref_from_symbol(dst);
  LNK_ObjSymbolRef src_ref = lnk_ref_from_symbol(src);
  lnk_error_obj(LNK_Error_MultiplyDefinedSymbol, dst_ref.obj, "symbol \"%S\" (No. %#x) is multiply defined in %S (No. %#x)", dst->name, dst_ref.symbol_idx, src_ref.obj->path, src_ref.symbol_idx);
}

internal B32
lnk_can_replace_symbol(LNK_Symbol *dst, LNK_Symbol *src)
{
  B32 can_replace = 0;

  COFF_ParsedSymbol           dst_parsed = lnk_parsed_from_symbol(dst);
  COFF_ParsedSymbol           src_parsed = lnk_parsed_from_symbol(src);
  COFF_SymbolValueInterpType  dst_interp = lnk_interp_from_symbol(dst);
  COFF_SymbolValueInterpType  src_interp = lnk_interp_from_symbol(src);
  LNK_ObjSymbolRef            dst_ref    = lnk_ref_from_symbol(dst);
  LNK_ObjSymbolRef            src_ref    = lnk_ref_from_symbol(src);
  LNK_Obj                    *dst_obj    = dst_ref.obj;
  LNK_Obj                    *src_obj    = src_ref.obj;

  // undefined vs regular
  if (dst_interp == COFF_SymbolValueInterp_Undefined && src_interp == COFF_SymbolValueInterp_Regular) {
    can_replace = 1;
  }
  // (weak vs undefined) or (undefined vs weak)
  else if ((dst_interp == COFF_SymbolValueInterp_Weak && src_interp == COFF_SymbolValueInterp_Undefined) || (dst_interp == COFF_SymbolValueInterp_Undefined && src_interp == COFF_SymbolValueInterp_Weak)) {
    LNK_Symbol *weak, *undef;
    COFF_ParsedSymbol weak_parsed;
    if (dst_interp == COFF_SymbolValueInterp_Weak) {
      weak = dst, undef = src;
      weak_parsed = dst_parsed;
    } else {
      weak = src, undef = dst;
      weak_parsed = src_parsed;
    }

    LNK_ObjSymbolRef    weak_symbol_ref = lnk_ref_from_symbol(weak);
    COFF_SymbolWeakExt *weak_ext        = coff_parse_weak_tag(weak_parsed, weak_symbol_ref.obj->header.is_big_obj);
    if (weak_ext->characteristics == COFF_WeakExt_SearchLibrary) {
      // NOTE: MSVC does not let a weak symbol to replace an undefined one,
      // but LLD links without errors or warnings, meaning undefined symbols
      // are resolved to the weak, which can potentially change behaviour of
      // the linked image
      can_replace = dst_interp == COFF_SymbolValueInterp_Weak;
    } else if (weak_ext->characteristics == COFF_WeakExt_NoLibrary) {
      can_replace = dst_interp == COFF_SymbolValueInterp_Weak;
    } else if (weak_ext->characteristics == COFF_WeakExt_SearchAlias) {
      can_replace = dst_interp == COFF_SymbolValueInterp_Undefined;
    } else {
      can_replace = lnk_symbol_is_before(src, dst);
    }
  }
  // undefined vs undefined
  else if (dst_interp == COFF_SymbolValueInterp_Undefined && src_interp == COFF_SymbolValueInterp_Undefined) {
    can_replace = lnk_symbol_is_before(src, dst);
  }
  // undefined vs common
  else if (dst_interp == COFF_SymbolValueInterp_Undefined && src_interp == COFF_SymbolValueInterp_Common) {
    can_replace = 1;
  }
  // undefined vs abs
  else if (dst_interp == COFF_SymbolValueInterp_Undefined && src_interp == COFF_SymbolValueInterp_Abs) {
    can_replace = 1;
  }
  // undefined vs debug
  else if (dst_interp == COFF_SymbolValueInterp_Undefined && src_interp == COFF_SymbolValueInterp_Debug) {
    can_replace = 1;
  }
  // regular/common/abs/debug vs undefined
  else if (dst_interp != COFF_SymbolValueInterp_Undefined && src_interp == COFF_SymbolValueInterp_Undefined) {
    can_replace = 0;
  }
  // regular vs abs
  else if (dst_interp == COFF_SymbolValueInterp_Regular && src_interp == COFF_SymbolValueInterp_Abs) {
    lnk_error_multiply_defined_symbol(dst, src);
  }
  // abs vs regular
  else if (dst_interp == COFF_SymbolValueInterp_Abs && src_interp == COFF_SymbolValueInterp_Regular) {
    lnk_error_multiply_defined_symbol(dst, src);
  }
  // abs vs common
  else if (dst_interp == COFF_SymbolValueInterp_Abs && src_interp == COFF_SymbolValueInterp_Common) {
    if (lnk_symbol_is_before(dst, src)) {
      can_replace = 1;
    } else {
      lnk_error_multiply_defined_symbol(dst, src);
    }
  }
  // common vs abs
  else if (dst_interp == COFF_SymbolValueInterp_Common && src_interp == COFF_SymbolValueInterp_Abs) {
    if (lnk_symbol_is_before(dst, src)) {
      lnk_error_multiply_defined_symbol(dst, src);
    }
  }
  // abs vs abs
  else if (dst_interp == COFF_SymbolValueInterp_Abs && src_interp == COFF_SymbolValueInterp_Abs) {
    lnk_error_multiply_defined_symbol(dst, src);
  }
  // weak vs weak
  else if (dst_interp == COFF_SymbolValueInterp_Weak && src_interp == COFF_SymbolValueInterp_Weak) {
    COFF_SymbolWeakExt *dst_ext = coff_parse_weak_tag(dst_parsed, dst_ref.obj->header.is_big_obj);
    COFF_SymbolWeakExt *src_ext = coff_parse_weak_tag(src_parsed, src_ref.obj->header.is_big_obj);
    if ((dst_ext->characteristics == COFF_WeakExt_SearchAlias && src_ext->characteristics != COFF_WeakExt_SearchAlias)) {
      if (lnk_symbol_is_before(dst, src) || src_ext->characteristics == COFF_WeakExt_AntiDependency) {
        can_replace = 0;
      } else {
        lnk_error_multiply_defined_symbol(dst, src);
      }
    } else if (dst_ext->characteristics != COFF_WeakExt_SearchAlias && src_ext->characteristics == COFF_WeakExt_SearchAlias) {
      if (lnk_symbol_is_before(src, dst) || dst_ext->characteristics == COFF_WeakExt_AntiDependency) {
        can_replace = 1;
      } else {
        lnk_error_multiply_defined_symbol(dst, src);
      }
    } else if (dst_ext->characteristics == COFF_WeakExt_SearchAlias && src_ext->characteristics == COFF_WeakExt_SearchAlias) {
      lnk_error_multiply_defined_symbol(dst, src);
    } else {
      can_replace = lnk_symbol_is_before(src, dst);
    }
  }
  // weak vs regular/abs/common
  else if (dst_interp == COFF_SymbolValueInterp_Weak && (src_interp == COFF_SymbolValueInterp_Regular || src_interp == COFF_SymbolValueInterp_Abs || src_interp == COFF_SymbolValueInterp_Common)) {
    can_replace = 1;
  }
  // regular/abs/common vs weak
  else if ((dst_interp == COFF_SymbolValueInterp_Regular || dst_interp == COFF_SymbolValueInterp_Abs || dst_interp == COFF_SymbolValueInterp_Common) && src_interp == COFF_SymbolValueInterp_Weak) {
    can_replace = 0;
  }
  // regular/common vs regular/common
  else if ((dst_interp == COFF_SymbolValueInterp_Regular || dst_interp == COFF_SymbolValueInterp_Common) && (src_interp == COFF_SymbolValueInterp_Regular || src_interp == COFF_SymbolValueInterp_Common)) {
    // parse dst symbol properties
    B32                   dst_is_comdat = 0;
    COFF_ComdatSelectType dst_select;
    U32                   dst_section_length;
    U32                   dst_check_sum;
    if (dst_interp == COFF_SymbolValueInterp_Regular) {
      dst_is_comdat = lnk_try_comdat_props_from_section_number(dst_ref.obj, dst_parsed.section_number, &dst_select, 0, &dst_section_length, &dst_check_sum);
    } else if (dst_interp == COFF_SymbolValueInterp_Common) {
      dst_select         = COFF_ComdatSelect_Largest;
      dst_section_length = dst_parsed.value;
      dst_check_sum      = 0;
      dst_is_comdat      = 1;
    }

    // parse src symbol properties
    B32                   src_is_comdat = 0;
    COFF_ComdatSelectType src_select;
    U32                   src_section_length, src_checks;
    U32                   src_check_sum;
    if (src_interp == COFF_SymbolValueInterp_Regular) {
      src_is_comdat = lnk_try_comdat_props_from_section_number(src_ref.obj, src_parsed.section_number, &src_select, 0, &src_section_length, &src_check_sum);
    } else if (src_interp == COFF_SymbolValueInterp_Common) {
      src_select         = COFF_ComdatSelect_Largest;
      src_section_length = src_parsed.value;
      src_check_sum      = 0;
      src_is_comdat      = 1;
    }

    // regular non-comdat vs communal
    if (dst_interp == COFF_SymbolValueInterp_Regular && !dst_is_comdat && src_interp == COFF_SymbolValueInterp_Common) {
      can_replace = 0;
    }
    // communal vs regular non-comdat
    else if (dst_interp == COFF_SymbolValueInterp_Common && src_interp == COFF_SymbolValueInterp_Regular && !src_is_comdat) {
      can_replace = 1;
    }
    // handle COMDATs
    else if (dst_is_comdat && src_is_comdat) {
      if ((src_select == COFF_ComdatSelect_Any && dst_select == COFF_ComdatSelect_Largest)) {
        src_select = COFF_ComdatSelect_Largest;
      }
      if (src_select == COFF_ComdatSelect_Largest && dst_select == COFF_ComdatSelect_Any) {
        dst_select = COFF_ComdatSelect_Largest;
      }

      if (src_select == dst_select) {
        switch (src_select) {
        case COFF_ComdatSelect_Null:
        case COFF_ComdatSelect_Any: {
          can_replace = lnk_obj_is_before(src_obj, dst_obj);
        } break;
        case COFF_ComdatSelect_NoDuplicates: {
          lnk_error_multiply_defined_symbol(dst, src);
        } break;
        case COFF_ComdatSelect_SameSize: {
          if (dst_section_length == src_section_length) {
            can_replace = lnk_obj_is_before(src_obj, dst_obj);
          } else {
            lnk_error_multiply_defined_symbol(dst, src);
          }
        } break;
        case COFF_ComdatSelect_ExactMatch: {
          LNK_ObjSection dst_section = lnk_obj_section_from_section_number(dst_obj, dst_parsed.section_number);
          LNK_ObjSection src_section = lnk_obj_section_from_section_number(src_obj, src_parsed.section_number);
          String8        dst_data    = str8_substr(dst_obj->data, dst_section.frange);
          String8        src_data    = str8_substr(src_obj->data, src_section.frange);

          B32 is_exact_match = 0;
          if (dst_check_sum != 0 && src_check_sum != 0) {
            is_exact_match = dst_check_sum == src_check_sum && str8_match(dst_data, src_data, 0);
          } else {
            is_exact_match = str8_match(dst_data, src_data, 0);
          }

          if (is_exact_match) {
            can_replace = lnk_obj_is_before(src_obj, dst_obj);
          } else {
            lnk_error_multiply_defined_symbol(dst, src);
          }
        } break;
        case COFF_ComdatSelect_Largest: {
          if (dst_section_length == src_section_length) {
            can_replace = lnk_obj_is_before(src_obj, dst_obj);
          } else {
            can_replace = dst_section_length < src_section_length;
          }
        } break;
        case COFF_ComdatSelect_Associative: { /* ignore */ } break;
        default: { InvalidPath; } break;
        }
      } else {
        lnk_error_obj(LNK_Warning_UnresolvedComdat, src_obj,
            "%S: COMDAT selection conflict detected, current selection %S, leader selection %S from %S",
            src->name, coff_string_from_comdat_select_type(src_select), coff_string_from_comdat_select_type(dst_select), dst_obj);
      }
    } else {
      lnk_error_multiply_defined_symbol(dst, src);
    }
  } else {
    lnk_error(LNK_Error_InvalidPath, "unable to find a suitable replacement logic for symbol combination");
  }

  return can_replace;
}

internal void
lnk_on_symbol_replace(LNK_Symbol *dst, LNK_Symbol *src)
{
  COFF_ParsedSymbol          dst_parsed = lnk_parsed_from_symbol(dst);
  COFF_SymbolValueInterpType dst_interp = lnk_interp_from_symbol(dst);
  LNK_ObjSymbolRef           dst_ref    = lnk_ref_from_symbol(dst);

  if (dst_interp == COFF_SymbolValueInterp_Regular) {
    // remove replaced section from the output
    LNK_ObjSection dst_section = lnk_obj_section_from_section_number(dst_ref.obj, dst_parsed.section_number);
    *dst_section.flags |= COFF_SectionFlag_LnkRemove;

    // remove associated sections from the output
    for (U32Node *associated_section = dst_ref.obj->associated_sections[dst_parsed.section_number];
        associated_section != 0;
        associated_section = associated_section->next) {
      LNK_ObjSection section = lnk_obj_section_from_section_number(dst_ref.obj, associated_section->data);
      *section.flags |= COFF_SectionFlag_LnkRemove;
    }
  }

  // merge symbol refs
  src->last_ref->next = dst->first_ref;
  src->last_ref = dst->last_ref;

  // assert leader section is live
#if BUILD_DEBUG
  {
    COFF_ParsedSymbol          src_parsed = lnk_parsed_from_symbol(src);
    COFF_SymbolValueInterpType src_interp = lnk_interp_from_symbol(src);
    LNK_ObjSymbolRef           src_ref    = lnk_ref_from_symbol(src);

    if (src_interp == COFF_SymbolValueInterp_Regular) {
      LNK_ObjSection    src_section = lnk_obj_section_from_section_number(src_ref.obj, src_parsed.section_number);
      COFF_SectionFlags src_flags   = *src_section.flags;
      AssertAlways(~src_flags & COFF_SectionFlag_LnkRemove);
    }
  }
#endif
}

internal void
lnk_symbol_hash_trie_insert_or_replace(Arena                        *arena,
                                       LNK_SymbolHashTrieChunkList  *chunks,
                                       LNK_SymbolHashTrie          **trie,
                                       U64                           hash,
                                       LNK_Symbol                   *symbol)
{
  LNK_SymbolHashTrie **curr_trie_ptr = trie;
  for (U64 h = hash; ; h <<= 2) {
    // load current pointer
    LNK_SymbolHashTrie *curr_trie = ins_atomic_ptr_eval(curr_trie_ptr);

    if (curr_trie == 0) {
      // init node
      LNK_SymbolHashTrie *new_trie = lnk_symbol_hash_trie_chunk_list_push(arena, chunks, 0x1000);
      new_trie->name               = &symbol->name;
      new_trie->symbol             = symbol;
      MemoryZeroArray(new_trie->child);

      // try to insert new node
      LNK_SymbolHashTrie *cmp = ins_atomic_ptr_eval_cond_assign(curr_trie_ptr, new_trie, curr_trie);

      // was symbol inserted?
      if (cmp == curr_trie) {
        break;
      }

      // rollback chunk list push
      --chunks->last->count;

      // retry insert with trie node from another thread
      curr_trie = cmp;
    }

    // load current symbol
    String8 *curr_name = ins_atomic_ptr_eval(&curr_trie->name);

    if (curr_name && str8_match(*curr_name, symbol->name, 0)) {
      for (LNK_Symbol *src = symbol;;) {
        // try replacing current symbol with zero, otherwise loop back and retry
        LNK_Symbol *leader = ins_atomic_ptr_eval_assign(&curr_trie->symbol, 0);

        // apply replacement
        if (leader) {
          if (lnk_can_replace_symbol(leader, src)) {
            // discard leader
            lnk_on_symbol_replace(leader, src);
            leader = src;
          } else {
            // discard source
            lnk_on_symbol_replace(src, leader);
            src = leader;
          }
        } else {
          leader = src;
        }

        // try replacing symbol, if another thread has already taken the slot, rerun replacement loop again
        LNK_Symbol *was_replaced = ins_atomic_ptr_eval_cond_assign(&curr_trie->symbol, leader, 0);

        // symbol replaced, exit
        if (was_replaced == 0) {
          goto exit;
        }
      }
    }

    // pick child and descend
    curr_trie_ptr = curr_trie->child + (h >> 62);
  }
  exit:;
}

internal LNK_SymbolHashTrie *
lnk_symbol_hash_trie_search(LNK_SymbolHashTrie *trie, U64 hash, String8 name)
{
  LNK_SymbolHashTrie  *result   = 0;
  LNK_SymbolHashTrie **curr_ptr = ≜
  for (U64 h = hash; ; h <<= 2) {
    LNK_SymbolHashTrie *curr = ins_atomic_ptr_eval(curr_ptr);
    if (curr == 0) {
      break;
    }
    if (curr->name && str8_match(*curr->name, name, 0)) {
      result = curr;
      break;
    }
    curr_ptr = curr->child + (h >> 62);
  }
  return result;
}

internal void
lnk_symbol_hash_trie_remove(LNK_SymbolHashTrie *trie)
{
  ins_atomic_ptr_eval_assign(&trie->name,   0);
  ins_atomic_ptr_eval_assign(&trie->symbol, 0);
}

internal LNK_SymbolHashTrieChunk **
lnk_array_from_symbol_hash_trie_chunk_list(Arena *arena, LNK_SymbolHashTrieChunkList *lists, U64 lists_count, U64 *count_out)
{
  U64 chunks_count = 0;
  for EachIndex(i, lists_count) { chunks_count += lists[i].count; }

  LNK_SymbolHashTrieChunk **chunks        = push_array(arena, LNK_SymbolHashTrieChunk *, chunks_count);
  U64                       chunks_cursor = 0;
  for EachIndex(i, lists_count) {
    for (LNK_SymbolHashTrieChunk *chunk = lists[i].first; chunk != 0; chunk = chunk->next) {
      chunks[chunks_cursor++] = chunk;
    }
  }

  if (count_out) {
    *count_out = chunks_count;
  }

  return chunks;
}

internal LNK_ObjSymbolRef
lnk_ref_from_symbol(LNK_Symbol *symbol)
{
  return symbol->first_ref->v;
}

internal U64
lnk_ref_count_from_symbol(LNK_Symbol *symbol)
{
  U64 count = 0;
  for (LNK_ObjSymbolRefNode *node = symbol->first_ref; node != 0; node = node->next, count += 1);
  return count;
}

internal LNK_ObjSymbolRef **
lnk_ref_from_symbol_many(Arena *arena, LNK_Symbol *symbol, U64 *count_out)
{
  // TODO: would be simpler if we sorted refs on insert/update
  U64                refs_count = lnk_ref_count_from_symbol(symbol);
  LNK_ObjSymbolRef **refs       = push_array(arena, LNK_ObjSymbolRef *, refs_count);
  U64                i          = 0;
  for (LNK_ObjSymbolRefNode *node = symbol->first_ref; node != 0; node = node->next, i += 1) {
    refs[i] = &node->v;
  }
  radsort(refs, refs_count, lnk_obj_symbol_ref_ptr_is_before);
  if (count_out) {
    *count_out = refs_count;
  }
  return refs;
}

internal COFF_ParsedSymbol
lnk_parsed_from_symbol(LNK_Symbol *symbol)
{
  LNK_ObjSymbolRef ref = lnk_ref_from_symbol(symbol);
  return lnk_parsed_symbol_from_coff_symbol_idx(ref.obj, ref.symbol_idx);
}

internal COFF_SymbolValueInterpType
lnk_interp_from_symbol(LNK_Symbol *symbol)
{
  COFF_ParsedSymbol symbol_parsed = lnk_parsed_from_symbol(symbol);
  return coff_interp_from_parsed_symbol(symbol_parsed);
}

internal U64
lnk_symbol_table_hasher(String8 string)
{
  return u64_hash_from_str8(string);
}

internal LNK_SymbolTable *
lnk_symbol_table_init(TP_Arena *arena)
{
  LNK_SymbolTable *symtab   = push_array(arena->v[0], LNK_SymbolTable, 1);
  symtab->arena             = arena;
  symtab->chunks            = push_array(arena->v[0], LNK_SymbolHashTrieChunkList, arena->count);
  symtab->search_chunks     = push_array(arena->v[0], LNK_SymbolHashTrieChunkList, arena->count);
  return symtab;
}

internal void
lnk_symbol_table_push_(LNK_SymbolTable *symtab, Arena *arena, U64 worker_id, LNK_Symbol *symbol)
{
  U64                        hash   = lnk_symbol_table_hasher(symbol->name);
  COFF_SymbolValueInterpType interp = lnk_interp_from_symbol(symbol);
  LNK_SymbolHashTrieChunkList *chunks;
  if (interp == COFF_SymbolValueInterp_Weak || interp == COFF_SymbolValueInterp_Undefined) {
    chunks = &symtab->search_chunks[worker_id];
  } else {
    chunks = &symtab->chunks[worker_id];
  }
  lnk_symbol_hash_trie_insert_or_replace(arena, chunks, &symtab->root, hash, symbol);
}

internal void
lnk_symbol_table_push(LNK_SymbolTable *symtab, LNK_Symbol *symbol)
{
  lnk_symbol_table_push_(symtab, symtab->arena->v[0], 0, symbol);
}

internal LNK_SymbolHashTrie *
lnk_symbol_table_search_(LNK_SymbolTable *symtab, String8 name)
{
  U64 hash = lnk_symbol_table_hasher(name);
  return lnk_symbol_hash_trie_search(symtab->root, hash, name);
}

internal LNK_Symbol *
lnk_symbol_table_search(LNK_SymbolTable *symtab, String8 name)
{
  LNK_SymbolHashTrie *trie = lnk_symbol_table_search_(symtab, name);
  return trie ? trie->symbol : 0;
}

internal LNK_Symbol *
lnk_symbol_table_searchf(LNK_SymbolTable *symtab, char *fmt, ...)
{
  Temp scratch = scratch_begin(0, 0);

  va_list args; va_start(args, fmt);
  String8 name = push_str8fv(scratch.arena, fmt, args);
  va_end(args);

  LNK_Symbol *symbol = lnk_symbol_table_search(symtab, name);

  scratch_end(scratch);
  return symbol;
}

internal ISectOff
lnk_sc_from_symbol(LNK_Symbol *symbol)
{
  COFF_ParsedSymbol parsed_symbol = lnk_parsed_from_symbol(symbol);
  ISectOff sc = { .isect = parsed_symbol.section_number, .off = parsed_symbol.value };
  return sc;
}

internal U64
lnk_voff_from_symbol(COFF_SectionHeader **image_section_table, LNK_Symbol *symbol)
{
  ISectOff sc   = lnk_sc_from_symbol(symbol);
  U64      voff = image_section_table[sc.isect]->voff + sc.off;
  return voff;
}

internal U64
lnk_foff_from_symbol(COFF_SectionHeader **image_section_table, LNK_Symbol *symbol)
{
  ISectOff sc   = lnk_sc_from_symbol(symbol);
  U64      foff = image_section_table[sc.isect]->foff + sc.off;
  return foff;
}

internal B32
lnk_resolve_weak_symbol(LNK_SymbolTable *symtab, LNK_ObjSymbolRef symbol, LNK_ObjSymbolRef *resolved_symbol_out)
{
  Temp scratch = scratch_begin(0,0);

  B32 is_resolved = 0;

  struct S { struct S *next; LNK_ObjSymbolRef symbol; B32 is_anti_dep; };
  struct S *sf = 0, *sl = 0;

  LNK_ObjSymbolRef current_symbol = symbol;
  for (;;) {
    // guard against self-referencing weak symbols
    struct S *was_visited = 0;
    for (struct S *s = sf; s != 0; s = s->next) {
      if (MemoryCompare(&s->symbol, &current_symbol, sizeof(LNK_ObjSymbolRef)) == 0) { was_visited = s; break; }
    }
    if (was_visited) {
      String8List chain = {0};
      for (struct S *s = sf; s != 0; s = s->next) {
        COFF_ParsedSymbol s_parsed = lnk_parsed_symbol_from_coff_symbol_idx(s->symbol.obj, s->symbol.symbol_idx);
        str8_list_pushf(scratch.arena, &chain, "\t%S Symbol %S (No. %#x) =>", s->symbol.obj->path, s_parsed.name, s->symbol.symbol_idx);
      }
      COFF_ParsedSymbol symbol_parsed = lnk_parsed_symbol_from_coff_symbol_idx(symbol.obj, symbol.symbol_idx);
      str8_list_pushf(scratch.arena, &chain, "\t%S Symbol %S (No. %#x)", sf->symbol.obj->path, symbol_parsed.name, sf->symbol.symbol_idx);

      String8 chain_string = str8_list_join(scratch.arena, &chain, &(StringJoin){ .sep = str8_lit("\n") });
      lnk_error_obj(LNK_Error_WeakCycle, symbol.obj, "unable to resolve cyclic symbol %S; ref chain:\n%S", symbol_parsed.name, chain_string);

      goto exit;
    }

    COFF_ParsedSymbol          current_parsed = lnk_parsed_symbol_from_coff_symbol_idx(current_symbol.obj, current_symbol.symbol_idx);
    COFF_SymbolValueInterpType current_interp = coff_interp_symbol(current_parsed.section_number, current_parsed.value, current_parsed.storage_class);
    if (current_interp == COFF_SymbolValueInterp_Weak) {
      // record visited symbol
      struct S *s = push_array(scratch.arena, struct S, 1);
      s->symbol   = current_symbol;
      SLLQueuePush(sf, sl, s);

      // does weak symbol have a definition?
      LNK_Symbol                 *defn_symbol = lnk_symbol_table_search(symtab, current_parsed.name);
      COFF_ParsedSymbol           defn_parsed = lnk_parsed_from_symbol(defn_symbol);
      COFF_SymbolValueInterpType  defn_interp = coff_interp_symbol(defn_parsed.section_number, defn_parsed.value, defn_parsed.storage_class);
      if (defn_interp != COFF_SymbolValueInterp_Weak) {
        current_symbol = lnk_ref_from_symbol(defn_symbol);
        break;
      }

      COFF_SymbolWeakExt *weak_ext = coff_parse_weak_tag(current_parsed, current_symbol.obj->header.is_big_obj);

      // no definition -- fallback to default symbol
      COFF_ParsedSymbol           tag_parsed = lnk_parsed_symbol_from_coff_symbol_idx(current_symbol.obj, weak_ext->tag_index);
      COFF_SymbolValueInterpType  tag_interp = coff_interp_symbol(tag_parsed.section_number, tag_parsed.value, tag_parsed.storage_class);
      current_symbol = (LNK_ObjSymbolRef){ .obj = current_symbol.obj, .symbol_idx = weak_ext->tag_index };

      if (weak_ext->characteristics == COFF_WeakExt_AntiDependency) {
        if (tag_interp == COFF_SymbolValueInterp_Undefined || tag_interp == COFF_SymbolValueInterp_Weak) {
          LNK_Symbol *dep_symbol = lnk_symbol_table_search(symtab, tag_parsed.name);
          tag_interp = lnk_interp_from_symbol(dep_symbol);
        }
        if (tag_interp == COFF_SymbolValueInterp_Weak) { break; }
      }
    } else if (current_interp == COFF_SymbolValueInterp_Undefined) {
      LNK_Symbol                 *defn_symbol = lnk_symbol_table_search(symtab, current_parsed.name);
      COFF_SymbolValueInterpType  defn_interp = lnk_interp_from_symbol(defn_symbol);

      // unresolved undefined symbol
      if (defn_interp == COFF_SymbolValueInterp_Undefined) { break; }

      // follow symbol definition
      current_symbol = lnk_ref_from_symbol(defn_symbol);
    } else { break; }
  }

  if (resolved_symbol_out) {
    *resolved_symbol_out = current_symbol;
  }
  is_resolved = 1;

exit:;
  scratch_end(scratch);
  return is_resolved;
}

internal B32
lnk_resolve_symbol(LNK_SymbolTable *symtab, LNK_ObjSymbolRef symbol, LNK_ObjSymbolRef *symbol_out)
{
  B32                        is_resolved   = 1;
  COFF_ParsedSymbol          symbol_parsed = lnk_parsed_symbol_from_coff_symbol_idx(symbol.obj, symbol.symbol_idx);
  COFF_SymbolValueInterpType symbol_interp = coff_interp_symbol(symbol_parsed.section_number, symbol_parsed.value, symbol_parsed.storage_class);
  switch (symbol_interp) {
  case COFF_SymbolValueInterp_Regular: {
    LNK_Symbol *symlink = lnk_obj_get_comdat_symlink(symbol.obj, symbol_parsed.section_number);
    *symbol_out = symlink ? lnk_ref_from_symbol(symlink) : symbol;
  } break;
  case COFF_SymbolValueInterp_Weak: {
    LNK_Symbol                 *defn        = lnk_symbol_table_search(symtab, symbol_parsed.name);
    COFF_ParsedSymbol           defn_parsed = lnk_parsed_from_symbol(defn);
    COFF_SymbolValueInterpType  defn_interp = lnk_interp_from_symbol(defn);
    if (defn_interp != COFF_SymbolValueInterp_Undefined) {
      *symbol_out = lnk_ref_from_symbol(defn);
    } else {
      is_resolved = 0;
    }
  } break;
  case COFF_SymbolValueInterp_Undefined: {
    LNK_Symbol *defn = lnk_symbol_table_search(symtab, symbol_parsed.name);
    if (defn) {
      *symbol_out = lnk_ref_from_symbol(defn);
    } else {
      is_resolved = 0;
    }
  } break;
  case COFF_SymbolValueInterp_Common: {
    LNK_Symbol *defn = lnk_symbol_table_search(symtab, symbol_parsed.name);
    *symbol_out = lnk_ref_from_symbol(defn);
  } break;
  case COFF_SymbolValueInterp_Abs: {
    if (symbol_parsed.storage_class == COFF_SymStorageClass_External) {
      LNK_Symbol *defn = lnk_symbol_table_search(symtab, symbol_parsed.name);
      *symbol_out = lnk_ref_from_symbol(defn);
    } else {
      *symbol_out = symbol;
    }
  } break;
  case COFF_SymbolValueInterp_Debug: { *symbol_out = symbol; } break;
  }
  return is_resolved;
}

internal
THREAD_POOL_TASK_FUNC(lnk_replace_weak_with_default_symbol_task)
{
  LNK_SymbolTable *symtab = raw_task;
  for EachNode(c, LNK_SymbolHashTrieChunk, symtab->search_chunks[task_id].first) {
    for EachIndex(i, c->count) {
      LNK_Symbol                 *symbol        = c->v[i].symbol;
      LNK_ObjSymbolRef            symbol_ref    = lnk_ref_from_symbol(symbol);
      COFF_ParsedSymbol           symbol_parsed = lnk_parsed_from_symbol(symbol);
      COFF_SymbolValueInterpType  symbol_interp = coff_interp_from_parsed_symbol(symbol_parsed);
      if (symbol_interp == COFF_SymbolValueInterp_Weak) {
        LNK_ObjSymbolRef resolve = {0};
        if (lnk_resolve_weak_symbol(symtab, symbol_ref, &resolve)) {
          COFF_ParsedSymbol          resolve_parsed = lnk_parsed_symbol_from_coff_symbol_idx(resolve.obj, resolve.symbol_idx);
          COFF_SymbolValueInterpType resolve_interp = coff_interp_from_parsed_symbol(resolve_parsed);
          if (resolve_interp == COFF_SymbolValueInterp_Weak) {
            COFF_SymbolWeakExt *weak_ext = coff_parse_weak_tag(resolve_parsed, symbol_ref.obj->header.is_big_obj);
            if (symbol_ref.obj->header.is_big_obj) {
              COFF_Symbol32 *symbol32  = symbol_parsed.raw_symbol;
              symbol32->section_number = COFF_Symbol_UndefinedSection;
              symbol32->value          = 0;
              symbol32->storage_class  = COFF_SymStorageClass_External;
            } else {
              COFF_Symbol16 *symbol16  = symbol_parsed.raw_symbol;
              symbol16->section_number = COFF_Symbol_UndefinedSection;
              symbol16->value          = 0;
              symbol16->storage_class  = COFF_SymStorageClass_External;
            }
          } else {
            symbol->first_ref->v = resolve;
          }
        }
      }
    }
  }
}

internal void
lnk_replace_weak_with_default_symbols(TP_Context *tp, LNK_SymbolTable *symtab)
{
  ProfBeginFunction();

  tp_for_parallel_prof(tp, 0, tp->worker_count, lnk_replace_weak_with_default_symbol_task, symtab, "Replace Weak With Default Symbols");

  for EachIndex(i, tp->worker_count) {
    lnk_symbol_hash_trie_chunk_list_concat_in_place(&symtab->chunks[i], &symtab->search_chunks[i]);
  }

  ProfEnd();
}


import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
import {
  loadChangeContext,
  generateInstructions,
  formatChangeStatus,
} from '../../src/core/artifact-graph/instruction-loader.js';
import {
  printInstructionsText,
  generateApplyInstructions,
} from '../../src/commands/workflow/instructions.js';
import { printStatusText } from '../../src/commands/workflow/status.js';

describe('printInstructionsText for skip_specs changes', () => {
  let tempDir: string;

  beforeEach(() => {
    tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'openspec-test-'));
    const changeDir = path.join(tempDir, 'openspec', 'changes', 'my-change');
    fs.mkdirSync(changeDir, { recursive: true });
    fs.writeFileSync(path.join(changeDir, 'proposal.md'), '# Proposal');
    fs.writeFileSync(
      path.join(changeDir, '.openspec.yaml'),
      'schema: spec-driven\nskip_specs: true\n'
    );
  });

  afterEach(() => {
    fs.rmSync(tempDir, { recursive: true, force: true });
    vi.restoreAllMocks();
  });

  function capture(artifactId: string): string {
    const lines: string[] = [];
    vi.spyOn(console, 'log').mockImplementation((...args: unknown[]) => {
      lines.push(args.join(' '));
    });
    const context = loadChangeContext(tempDir, 'my-change');
    const instructions = generateInstructions(context, artifactId);
    const isBlocked = instructions.dependencies.some((d) => !d.done);
    printInstructionsText(instructions, isBlocked);
    vi.restoreAllMocks();
    return lines.join('\n');
  }

  it('emits only the warning for a skipped artifact, no creation directive', () => {
    const output = capture('specs');

    expect(output).toContain('skip_specs: true');
    expect(output).toContain('Do not create spec files');
    expect(output).toContain('</artifact>');
    expect(output).not.toContain('<task>');
    expect(output).not.toContain('<template>');
    expect(output).not.toContain('Write to:');
  });

  it('keeps the normal creation directive for non-skipped artifacts', () => {
    const output = capture('design');

    expect(output).toContain('<task>');
    expect(output).toContain('Create the design artifact for change "my-change".');
    expect(output).not.toContain('this artifact is skipped');
  });

  it('carries skipped and warning in the JSON-facing payload', () => {
    const context = loadChangeContext(tempDir, 'my-change');
    const instructions = generateInstructions(context, 'specs');

    expect(instructions.skipped).toBe(true);
    expect(instructions.warning).toContain('Do not create spec files');
  });

  it('marks the specs dependency as skipped instead of done with files to read', () => {
    const context = loadChangeContext(tempDir, 'my-change');
    const tasksInstructions = generateInstructions(context, 'tasks');
    const specsDep = tasksInstructions.dependencies.find((d) => d.id === 'specs');
    expect(specsDep?.skipped).toBe(true);

    const output = capture('tasks');
    expect(output).toContain('<dependency id="specs" status="skipped">');
    expect(output).toContain('no files to read');
    // The skipped dependency must not point the agent at spec file paths.
    expect(output).not.toContain('specs/**/*.md</path>');
  });

  it('renders the specs stage as skipped in status text with a reduced denominator', () => {
    const lines: string[] = [];
    vi.spyOn(console, 'log').mockImplementation((...args: unknown[]) => {
      lines.push(args.join(' '));
    });
    const context = loadChangeContext(tempDir, 'my-change');
    printStatusText(formatChangeStatus(context));
    vi.restoreAllMocks();
    const output = lines.join('\n');

    expect(output).toContain('Progress: 1/3 artifacts complete (1 skipped)');
    expect(output).toContain('[~] specs (skipped: change declares skip_specs)');
    expect(output).toContain('[x] proposal');
  });
});

describe('generateApplyInstructions for skip_specs changes', () => {
  let tempDir: string;

  beforeEach(() => {
    tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'openspec-test-'));
  });

  afterEach(() => {
    fs.rmSync(tempDir, { recursive: true, force: true });
  });

  it('does not block apply on a skipped artifact when the schema requires all artifacts', async () => {
    // A schema with no apply block falls back to requiring every artifact,
    // including the specs-producing one - the skip must count as present.
    const schemaDir = path.join(tempDir, 'openspec', 'schemas', 'mini');
    fs.mkdirSync(schemaDir, { recursive: true });
    fs.writeFileSync(
      path.join(schemaDir, 'schema.yaml'),
      [
        'name: mini',
        'version: 1',
        'artifacts:',
        '  - id: proposal',
        '    generates: proposal.md',
        '    description: p',
        '    template: proposal.md',
        '  - id: specs',
        '    generates: "specs/**/*.md"',
        '    description: s',
        '    template: spec.md',
        '    requires: [proposal]',
        '  - id: tasks',
        '    generates: tasks.md',
        '    description: t',
        '    template: tasks.md',
        '    requires: [specs]',
        '',
      ].join('\n')
    );
    const changeDir = path.join(tempDir, 'openspec', 'changes', 'my-change');
    fs.mkdirSync(changeDir, { recursive: true });
    fs.writeFileSync(path.join(changeDir, 'proposal.md'), '# Proposal');
    fs.writeFileSync(path.join(changeDir, 'tasks.md'), '## 1. W\n\n- [ ] 1.1 Do\n');
    fs.writeFileSync(
      path.join(changeDir, '.openspec.yaml'),
      'schema: mini\nskip_specs: true\n'
    );

    const instructions = await generateApplyInstructions(tempDir, 'my-change');

    expect(instructions.missingArtifacts ?? []).not.toContain('specs');
    expect(instructions.state).not.toBe('blocked');
  });
});


from typing import Any, BinaryIO, Optional
from ._stream_info import StreamInfo

class DocumentConverterResult:
    """The result of converting a document to Markdown."""

    def __init__(
        self,
        markdown: str,
        *,
        title: Optional[str] = None,
    ):
        """
        Initialize the DocumentConverterResult.

        The only required parameter is the converted Markdown text.
        The title, and any other metadata that may be added in the future, are optional.

        Parameters:
        - markdown: The converted Markdown text.
        - title: Optional title of the document.
        """
        self.markdown = markdown
        self.title = title

    @property
    def text_content(self) -> str:
        """Soft-deprecated alias for `markdown`. New code should migrate to using `markdown` or __str__."""
        return self.markdown

    @text_content.setter
    def text_content(self, markdown: str):
        """Soft-deprecated alias for `markdown`. New code should migrate to using `markdown` or __str__."""
        self.markdown = markdown

    def __str__(self) -> str:
        """Return the converted Markdown text."""
        return self.markdown

class DocumentConverter:
    """Abstract superclass of all DocumentConverters."""

    def accepts(
        self,
        file_stream: BinaryIO,
        stream_info: StreamInfo,
        **kwargs: Any,  # Options to pass to the converter
    ) -> bool:
        """
        Return a quick determination on if the converter should attempt converting the document.
        This is primarily based on `stream_info` (typically, `stream_info.mimetype`, `stream_info.extension`).
        In cases where the data is retrieved via HTTP, the `stream_info.url` might also be referenced to
        make a determination (e.g., special converters for Wikipedia, YouTube etc).
        Finally, it is conceivable that the `stream_info.filename` might be used in cases
        where the filename is well-known (e.g., `Dockerfile`, `Makefile`, etc)

        NOTE: The method signature is designed to match that of the convert() method. This provides some
        assurance that, if accepts() returns True, the convert() method will also be able to handle the document.

        IMPORTANT: In rare cases, (e.g., OutlookMsgConverter) we need to read more from the stream to make a final
        determination. Read operations inevitably advance the position in file_stream. In these cases, the position
        MUST be reset before returning. This is because the convert() method may be called immediately
        after accepts(), and will expect the file_stream to be at the original position.

        E.g.,
        cur_pos = file_stream.tell() # Save the current position
        data = file_stream.read(100) # ... peek at the first 100 bytes, etc.
        file_stream.seek(cur_pos)    # Reset the position to the original position

        Parameters:
        - file_stream: The file-like object to convert. Must support seek(), tell(), and read() methods.
        - stream_info: The StreamInfo object containing metadata about the file (mimetype, extension, charset, etc.)
        - kwargs: Additional keyword arguments for the converter.

        Returns:
        - bool: True if the converter can handle the document, False otherwise.
        """
        raise NotImplementedError(
            f"The subclass, {type(self).__name__}, must implement the accepts() method to determine if they can handle the document."
        )

    def convert(
        self,
        file_stream: BinaryIO,
        stream_info: StreamInfo,
        **kwargs: Any,  # Options to pass to the converter
    ) -> DocumentConverterResult:
        """
        Convert a document to Markdown text.

        Parameters:
        - file_stream: The file-like object to convert. Must support seek(), tell(), and read() methods.
        - stream_info: The StreamInfo object containing metadata about the file (mimetype, extension, charset, etc.)
        - kwargs: Additional keyword arguments for the converter.

        Returns:
        - DocumentConverterResult: The result of the conversion, which includes the title and markdown content.

        Raises:
        - FileConversionException: If the mimetype is recognized, but the conversion fails for some other reason.
        - MissingDependencyException: If the converter requires a dependency that is not installed.
        """
        raise NotImplementedError("Subclasses must implement this method")


//go:build testing

package agent

import (
	"context"
	"crypto/ed25519"
	"encoding/json"
	"fmt"
	"net"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/henrygd/beszel/internal/entities/container"
	"github.com/henrygd/beszel/internal/entities/system"

	"github.com/blang/semver"
	"github.com/fxamacker/cbor/v2"
	"github.com/gliderlabs/ssh"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	gossh "golang.org/x/crypto/ssh"
)

func TestStartServer(t *testing.T) {
	// Generate a test key pair
	pubKey, privKey, err := ed25519.GenerateKey(nil)
	require.NoError(t, err)
	signer, err := gossh.NewSignerFromKey(privKey)
	require.NoError(t, err)
	sshPubKey, err := gossh.NewPublicKey(pubKey)
	require.NoError(t, err)

	// Generate a different key pair for bad key test
	badPubKey, badPrivKey, err := ed25519.GenerateKey(nil)
	require.NoError(t, err)
	badSigner, err := gossh.NewSignerFromKey(badPrivKey)
	require.NoError(t, err)
	sshBadPubKey, err := gossh.NewPublicKey(badPubKey)
	require.NoError(t, err)

	socketFile := filepath.Join(t.TempDir(), "beszel-test.sock")

	tests := []struct {
		name        string
		config      ServerOptions
		wantErr     bool
		errContains string
		setup       func() error
		cleanup     func() error
	}{
		{
			name: "tcp port only",
			config: ServerOptions{
				Network: "tcp",
				Addr:    ":45987",
				Keys:    []gossh.PublicKey{sshPubKey},
			},
		},
		{
			name: "tcp with ipv4",
			config: ServerOptions{
				Network: "tcp4",
				Addr:    "127.0.0.1:45988",
				Keys:    []gossh.PublicKey{sshPubKey},
			},
		},
		{
			name: "tcp with ipv6",
			config: ServerOptions{
				Network: "tcp6",
				Addr:    "[::1]:45989",
				Keys:    []gossh.PublicKey{sshPubKey},
			},
		},
		{
			name: "unix socket",
			config: ServerOptions{
				Network: "unix",
				Addr:    socketFile,
				Keys:    []gossh.PublicKey{sshPubKey},
			},
			setup: func() error {
				// Create a socket file that should be removed
				f, err := os.Create(socketFile)
				if err != nil {
					return err
				}
				return f.Close()
			},
			cleanup: func() error {
				return os.Remove(socketFile)
			},
		},
		{
			name: "bad key should fail",
			config: ServerOptions{
				Network: "tcp",
				Addr:    ":45987",
				Keys:    []gossh.PublicKey{sshBadPubKey},
			},
			wantErr:     true,
			errContains: "ssh: handshake failed",
		},
		{
			name: "good key still good",
			config: ServerOptions{
				Network: "tcp",
				Addr:    ":45987",
				Keys:    []gossh.PublicKey{sshPubKey},
			},
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if tt.setup != nil {
				err := tt.setup()
				require.NoError(t, err)
			}

			if tt.cleanup != nil {
				defer tt.cleanup()
			}

			agent, err := NewAgent("")
			require.NoError(t, err)

			// Start server in a goroutine since it blocks
			errChan := make(chan error, 1)
			go func() {
				errChan <- agent.StartServer(tt.config)
			}()

			// Add a short delay to allow the server to start
			time.Sleep(100 * time.Millisecond)

			// Try to connect to verify server is running
			var client *gossh.Client

			// Choose the appropriate signer based on the test case
			testSigner := signer
			if tt.name == "bad key should fail" {
				testSigner = badSigner
			}

			sshClientConfig := &gossh.ClientConfig{
				User: "a",
				Auth: []gossh.AuthMethod{
					gossh.PublicKeys(testSigner),
				},
				HostKeyCallback: gossh.InsecureIgnoreHostKey(),
				Timeout:         4 * time.Second,
			}

			switch tt.config.Network {
			case "unix":
				client, err = gossh.Dial("unix", tt.config.Addr, sshClientConfig)
			default:
				if !strings.Contains(tt.config.Addr, ":") {
					tt.config.Addr = ":" + tt.config.Addr
				}
				client, err = gossh.Dial("tcp", tt.config.Addr, sshClientConfig)
			}

			if tt.wantErr {
				assert.Error(t, err)
				if tt.errContains != "" {
					assert.Contains(t, err.Error(), tt.errContains)
				}
				return
			}

			require.NoError(t, err)
			require.NotNil(t, client)
			client.Close()
		})
	}
}

func TestStartServerDisableSSH(t *testing.T) {
	t.Setenv("BESZEL_AGENT_DISABLE_SSH", "true")

	agent, err := NewAgent("")
	require.NoError(t, err)

	opts := ServerOptions{
		Network: "tcp",
		Addr:    ":45990",
	}

	err = agent.StartServer(opts)
	assert.Error(t, err)
	assert.Contains(t, err.Error(), "SSH disabled")
}

/////////////////////////////////////////////////////////////////
//////////////////// ParseKeys Tests ////////////////////////////
/////////////////////////////////////////////////////////////////

// Helper function to generate a temporary file with content
func createTempFile(content string) (string, error) {
	tmpFile, err := os.CreateTemp("", "ssh_keys_*.txt")
	if err != nil {
		return "", fmt.Errorf("failed to create temp file: %w", err)
	}
	defer tmpFile.Close()

	if _, err := tmpFile.WriteString(content); err != nil {
		return "", fmt.Errorf("failed to write to temp file: %w", err)
	}

	return tmpFile.Name(), nil
}

// Test case 1: String with a single SSH key
func TestParseSingleKeyFromString(t *testing.T) {
	input := "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKCBM91kukN7hbvFKtbpEeo2JXjCcNxXcdBH7V7ADMBo"
	keys, err := ParseKeys(input)
	if err != nil {
		t.Fatalf("Expected no error, got: %v", err)
	}
	if len(keys) != 1 {
		t.Fatalf("Expected 1 key, got %d keys", len(keys))
	}
	if keys[0].Type() != "ssh-ed25519" {
		t.Fatalf("Expected key type 'ssh-ed25519', got '%s'", keys[0].Type())
	}
}

// Test case 2: String with multiple SSH keys
func TestParseMultipleKeysFromString(t *testing.T) {
	input := "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKCBM91kukN7hbvFKtbpEeo2JXjCcNxXcdBH7V7ADMBo\nssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJDMtAOQfxDlCxe+A5lVbUY/DHxK1LAF2Z3AV0FYv36D \n #comment\n ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJDMtAOQfxDlCxe+A5lVbUY/DHxK1LAF2Z3AV0FYv36D"
	keys, err := ParseKeys(input)
	if err != nil {
		t.Fatalf("Expected no error, got: %v", err)
	}
	if len(keys) != 3 {
		t.Fatalf("Expected 3 keys, got %d keys", len(keys))
	}
	if keys[0].Type() != "ssh-ed25519" || keys[1].Type() != "ssh-ed25519" || keys[2].Type() != "ssh-ed25519" {
		t.Fatalf("Unexpected key types: %s, %s, %s", keys[0].Type(), keys[1].Type(), keys[2].Type())
	}
}

// Test case 3: File with a single SSH key
func TestParseSingleKeyFromFile(t *testing.T) {
	content := "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKCBM91kukN7hbvFKtbpEeo2JXjCcNxXcdBH7V7ADMBo"
	filePath, err := createTempFile(content)
	if err != nil {
		t.Fatalf("Failed to create temp file: %v", err)
	}
	defer os.Remove(filePath) // Clean up the file after the test

	// Read the file content
	fileContent, err := os.ReadFile(filePath)
	if err != nil {
		t.Fatalf("Failed to read temp file: %v", err)
	}

	// Parse the keys
	keys, err := ParseKeys(string(fileContent))
	if err != nil {
		t.Fatalf("Expected no error, got: %v", err)
	}
	if len(keys) != 1 {
		t.Fatalf("Expected 1 key, got %d keys", len(keys))
	}
	if keys[0].Type() != "ssh-ed25519" {
		t.Fatalf("Expected key type 'ssh-ed25519', got '%s'", keys[0].Type())
	}
}

// Test case 4: File with multiple SSH keys
func TestParseMultipleKeysFromFile(t *testing.T) {
	content := "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKCBM91kukN7hbvFKtbpEeo2JXjCcNxXcdBH7V7ADMBo\nssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJDMtAOQfxDlCxe+A5lVbUY/DHxK1LAF2Z3AV0FYv36D \n #comment\n ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJDMtAOQfxDlCxe+A5lVbUY/DHxK1LAF2Z3AV0FYv36D"
	filePath, err := createTempFile(content)
	if err != nil {
		t.Fatalf("Failed to create temp file: %v", err)
	}
	// defer os.Remove(filePath) // Clean up the file after the test

	// Read the file content
	fileContent, err := os.ReadFile(filePath)
	if err != nil {
		t.Fatalf("Failed to read temp file: %v", err)
	}

	// Parse the keys
	keys, err := ParseKeys(string(fileContent))
	if err != nil {
		t.Fatalf("Expected no error, got: %v", err)
	}
	if len(keys) != 3 {
		t.Fatalf("Expected 3 keys, got %d keys", len(keys))
	}
	if keys[0].Type() != "ssh-ed25519" || keys[1].Type() != "ssh-ed25519" || keys[2].Type() != "ssh-ed25519" {
		t.Fatalf("Unexpected key types: %s, %s, %s", keys[0].Type(), keys[1].Type(), keys[2].Type())
	}
}

// Test case 5: Invalid SSH key input
func TestParseInvalidKey(t *testing.T) {
	input := "invalid-key-data"
	_, err := ParseKeys(input)
	if err == nil {
		t.Fatalf("Expected an error for invalid key, got nil")
	}
	expectedErrMsg := "failed to parse key"
	if !strings.Contains(err.Error(), expectedErrMsg) {
		t.Fatalf("Expected error message to contain '%s', got: %v", expectedErrMsg, err)
	}
}

/////////////////////////////////////////////////////////////////
//////////////////// Hub Version Tests //////////////////////////
/////////////////////////////////////////////////////////////////

func TestExtractHubVersion(t *testing.T) {
	tests := []struct {
		name            string
		clientVersion   string
		expectedVersion string
		expectError     bool
	}{
		{
			name:            "valid beszel client version with underscore",
			clientVersion:   "SSH-2.0-beszel_0.11.1",
			expectedVersion: "0.11.1",
			expectError:     false,
		},
		{
			name:            "valid beszel client version with beta",
			clientVersion:   "SSH-2.0-beszel_1.0.0-beta",
			expectedVersion: "1.0.0-beta",
			expectError:     false,
		},
		{
			name:            "valid beszel client version with rc",
			clientVersion:   "SSH-2.0-beszel_0.12.0-rc1",
			expectedVersion: "0.12.0-rc1",
			expectError:     false,
		},
		{
			name:            "different SSH client",
			clientVersion:   "SSH-2.0-OpenSSH_8.0",
			expectedVersion: "8.0",
			expectError:     true,
		},
		{
			name:          "malformed version string without underscore",
			clientVersion: "SSH-2.0-beszel",
			expectError:   true,
		},
		{
			name:          "empty version string",
			clientVersion: "",
			expectError:   true,
		},
		{
			name:            "version string with underscore but no version",
			clientVersion:   "beszel_",
			expectedVersion: "",
			expectError:     true,
		},
		{
			name:            "version with patch and build metadata",
			clientVersion:   "SSH-2.0-beszel_1.2.3+build.123",
			expectedVersion: "1.2.3+build.123",
			expectError:     false,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			result, err := extractHubVersion(tt.clientVersion)

			if tt.expectError {
				assert.Error(t, err)
				return
			}

			require.NoError(t, err)
			assert.Equal(t, tt.expectedVersion, result.String())
		})
	}
}

/////////////////////////////////////////////////////////////////
/////////////// Hub Version Detection Tests ////////////////////
/////////////////////////////////////////////////////////////////

func TestGetHubVersion(t *testing.T) {
	agent, err := NewAgent("")
	require.NoError(t, err)

	// Mock SSH context that implements the ssh.Context interface
	mockCtx := &mockSSHContext{
		sessionID:     "test-session-123",
		clientVersion: "SSH-2.0-beszel_0.12.0",
	}

	// Test first call - should extract and cache version
	version := agent.getHubVersion("test-session-123", mockCtx)
	assert.Equal(t, "0.12.0", version.String())

	// Test second call - should return cached version
	mockCtx.clientVersion = "SSH-2.0-beszel_0.11.0" // Change version but should still return cached
	version = agent.getHubVersion("test-session-123", mockCtx)
	assert.Equal(t, "0.12.0", version.String()) // Should still be cached version

	// Test different session - should extract new version
	version = agent.getHubVersion("different-session", mockCtx)
	assert.Equal(t, "0.11.0", version.String())

	// Test with invalid version string (non-beszel client)
	mockCtx.clientVersion = "SSH-2.0-OpenSSH_8.0"
	version = agent.getHubVersion("invalid-session", mockCtx)
	assert.Equal(t, "0.0.0", version.String()) // Should be empty version for non-beszel clients

	// Test with no client version
	mockCtx.clientVersion = ""
	version = agent.getHubVersion("no-version-session", mockCtx)
	assert.True(t, version.EQ(semver.Version{})) // Should be empty version
}

// mockSSHContext implements ssh.Context for testing
type mockSSHContext struct {
	context.Context
	sync.Mutex
	sessionID     string
	clientVersion string
}

func (m *mockSSHContext) SessionID() string {
	return m.sessionID
}

func (m *mockSSHContext) ClientVersion() string {
	return m.clientVersion
}

func (m *mockSSHContext) ServerVersion() string {
	return "SSH-2.0-beszel_test"
}

func (m *mockSSHContext) Value(key interface{}) interface{} {
	if key == ssh.ContextKeyClientVersion {
		return m.clientVersion
	}
	return nil
}

func (m *mockSSHContext) User() string                    { return "test-user" }
func (m *mockSSHContext) RemoteAddr() net.Addr            { return nil }
func (m *mockSSHContext) LocalAddr() net.Addr             { return nil }
func (m *mockSSHContext) Permissions() *ssh.Permissions   { return nil }
func (m *mockSSHContext) SetValue(key, value interface{}) {}

/////////////////////////////////////////////////////////////////
/////////////// CBOR vs JSON Encoding Tests ////////////////////
/////////////////////////////////////////////////////////////////

// TestWriteToSessionEncoding tests that writeToSession actually encodes data in the correct format
func TestWriteToSessionEncoding(t *testing.T) {
	tests := []struct {
		name             string
		hubVersion       string
		expectedUsesCbor bool
	}{
		{
			name:             "old hub version should use JSON",
			hubVersion:       "0.11.1",
			expectedUsesCbor: false,
		},
		{
			name:             "non-beta release should use CBOR",
			hubVersion:       "0.12.0",
			expectedUsesCbor: true,
		},
		{
			name:             "even newer hub version should use CBOR",
			hubVersion:       "0.16.4",
			expectedUsesCbor: true,
		},
		{
			name:             "beta version below release threshold should use JSON",
			hubVersion:       "0.12.0-beta0",
			expectedUsesCbor: false,
		},
		// {
		// 	name:             "matching beta version should use CBOR",
		// 	hubVersion:       "0.12.0-beta2",
		// 	expectedUsesCbor: true,
		// },
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			// Reset the global hubVersions map to ensure clean state for each test
			hubVersions = nil

			agent, err := NewAgent("")
			require.NoError(t, err)

			// Parse the test version
			version, err := semver.Parse(tt.hubVersion)
			require.NoError(t, err)

			// Create test data to encode
			testData := createTestCombinedData()

			var buf strings.Builder
			err = agent.writeToSession(&buf, testData, version)
			require.NoError(t, err)

			encodedData := buf.String()
			require.NotEmpty(t, encodedData)

			// Verify the encoding format by attempting to decode
			if tt.expectedUsesCbor {
				var decodedCbor system.CombinedData
				err = cbor.Unmarshal([]byte(encodedData), &decodedCbor)
				assert.NoError(t, err, "Should be valid CBOR data")

				var decodedJson system.CombinedData
				err = json.Unmarshal([]byte(encodedData), &decodedJson)
				assert.Error(t, err, "Should not be valid JSON data")

				assert.Equal(t, testData.Details.Hostname, decodedCbor.Details.Hostname)
				assert.Equal(t, testData.Stats.Cpu, decodedCbor.Stats.Cpu)
			} else {
				// Should be JSON - try to decode as JSON
				var decodedJson system.CombinedData
				err = json.Unmarshal([]byte(encodedData), &decodedJson)
				assert.NoError(t, err, "Should be valid JSON data")

				var decodedCbor system.CombinedData
				err = cbor.Unmarshal([]byte(encodedData), &decodedCbor)
				assert.Error(t, err, "Should not be valid CBOR data")

				// Verify the decoded JSON data matches our test data
				assert.Equal(t, testData.Details.Hostname, decodedJson.Details.Hostname)
				assert.Equal(t, testData.Stats.Cpu, decodedJson.Stats.Cpu)

				// Verify it looks like JSON (starts with '{' and contains readable field names)
				assert.True(t, strings.HasPrefix(encodedData, "{"), "JSON should start with '{'")
				assert.Contains(t, encodedData, `"info"`, "JSON should contain readable field names")
				assert.Contains(t, encodedData, `"stats"`, "JSON should contain readable field names")
			}
		})
	}
}

// Helper function to create test data for encoding tests
func createTestCombinedData() *system.CombinedData {
	return &system.CombinedData{
		Stats: system.Stats{
			Cpu:       25.5,
			Mem:       8589934592, // 8GB
			MemUsed:   4294967296, // 4GB
			MemPct:    50.0,
			DiskTotal: 1099511627776, // 1TB
			DiskUsed:  549755813888,  // 512GB
			DiskPct:   50.0,
		},
		Details: &system.Details{
			Hostname: "test-host",
		},
		Info: system.Info{
			Uptime:       3600,
			AgentVersion: "0.12.0",
		},
		Containers: []*container.Stats{
			{
				Name: "test-container",
				Cpu:  10.5,
				Mem:  1073741824, // 1GB
			},
		},
	}
}

func TestHubVersionCaching(t *testing.T) {
	// Reset the global hubVersions map to ensure clean state
	hubVersions = nil

	agent, err := NewAgent("")
	require.NoError(t, err)

	ctx1 := &mockSSHContext{
		sessionID:     "session1",
		clientVersion: "SSH-2.0-beszel_0.12.0",
	}
	ctx2 := &mockSSHContext{
		sessionID:     "session2",
		clientVersion: "SSH-2.0-beszel_0.11.0",
	}

	// First calls should cache the versions
	v1 := agent.getHubVersion("session1", ctx1)
	v2 := agent.getHubVersion("session2", ctx2)

	assert.Equal(t, "0.12.0", v1.String())
	assert.Equal(t, "0.11.0", v2.String())

	// Verify caching by changing context but keeping same session ID
	ctx1.clientVersion = "SSH-2.0-beszel_0.10.0"
	v1Cached := agent.getHubVersion("session1", ctx1)
	assert.Equal(t, "0.12.0", v1Cached.String()) // Should still be cached version

	// New session should get new version
	ctx3 := &mockSSHContext{
		sessionID:     "session3",
		clientVersion: "SSH-2.0-beszel_0.13.0",
	}
	v3 := agent.getHubVersion("session3", ctx3)
	assert.Equal(t, "0.13.0", v3.String())
}


/*
 * Copyright 2025 CloudWeGo Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package compose

import (
	"context"
	"io"
	"testing"

	"github.com/stretchr/testify/assert"

	"github.com/cloudwego/eino/schema"
)

func TestMultiBranch(t *testing.T) {
	g := NewGraph[string, map[string]any]()
	emptyLambda := InvokableLambda(func(ctx context.Context, input string) (output string, err error) { return input, nil })
	err := g.AddLambdaNode("1", emptyLambda, WithOutputKey("1"))
	assert.NoError(t, err)
	err = g.AddLambdaNode("2", emptyLambda, WithOutputKey("2"))
	assert.NoError(t, err)
	err = g.AddLambdaNode("3", emptyLambda, WithOutputKey("3"))
	assert.NoError(t, err)

	err = g.AddBranch(START, NewGraphMultiBranch(func(ctx context.Context, in string) (endNode map[string]bool, err error) {
		return map[string]bool{"1": true, "2": true}, nil
	}, map[string]bool{"1": true, "2": true, "3": true}))
	assert.NoError(t, err)

	err = g.AddEdge("1", END)
	assert.NoError(t, err)
	err = g.AddEdge("2", END)
	assert.NoError(t, err)
	err = g.AddEdge("3", END)
	assert.NoError(t, err)

	ctx := context.Background()
	r, err := g.Compile(ctx)
	assert.NoError(t, err)

	result, err := r.Invoke(ctx, "start")
	assert.NoError(t, err)
	assert.Equal(t, map[string]any{
		"1": "start",
		"2": "start",
	}, result)

	streamResult, err := r.Stream(ctx, "start")
	assert.NoError(t, err)
	result = map[string]any{}
	for {
		chunk, err := streamResult.Recv()
		if err == io.EOF {
			break
		}
		assert.NoError(t, err)
		for k, v := range chunk {
			result[k] = v
		}
	}
	assert.Equal(t, map[string]any{
		"1": "start",
		"2": "start",
	}, result)
}

func TestStreamMultiBranch(t *testing.T) {
	g := NewGraph[string, map[string]any]()
	emptyLambda := InvokableLambda(func(ctx context.Context, input string) (output string, err error) { return input, nil })
	err := g.AddLambdaNode("1", emptyLambda, WithOutputKey("1"))
	assert.NoError(t, err)
	err = g.AddLambdaNode("2", emptyLambda, WithOutputKey("2"))
	assert.NoError(t, err)
	err = g.AddLambdaNode("3", emptyLambda, WithOutputKey("3"))
	assert.NoError(t, err)

	err = g.AddBranch(START, NewStreamGraphMultiBranch(func(ctx context.Context, in *schema.StreamReader[string]) (endNode map[string]bool, err error) {
		in.Close()
		return map[string]bool{"1": true, "2": true}, nil
	}, map[string]bool{"1": true, "2": true, "3": true}))
	assert.NoError(t, err)

	err = g.AddEdge("1", END)
	assert.NoError(t, err)
	err = g.AddEdge("2", END)
	assert.NoError(t, err)
	err = g.AddEdge("3", END)
	assert.NoError(t, err)

	ctx := context.Background()
	r, err := g.Compile(ctx)
	assert.NoError(t, err)

	result, err := r.Invoke(ctx, "start")
	assert.NoError(t, err)
	assert.Equal(t, map[string]any{
		"1": "start",
		"2": "start",
	}, result)

	streamResult, err := r.Stream(ctx, "start")
	assert.NoError(t, err)
	result = map[string]any{}
	for {
		chunk, err := streamResult.Recv()
		if err == io.EOF {
			break
		}
		assert.NoError(t, err)
		for k, v := range chunk {
			result[k] = v
		}
	}
	assert.Equal(t, map[string]any{
		"1": "start",
		"2": "start",
	}, result)
}


// Behavior gate: does the ponytail ruleset actually PRODUCE its refined
// behaviors, not just carry the text? One check per probe (vars.probe), each
// targeting a rule that a field review (rcstack, phases 0-8) showed mattered:
//   hardware     - "hardware is never the spec ideal, leave the calibration knob"
//   explanation  - "explanation the user explicitly asked for is not debt"
//   onecheck     - "lazy code without its check is unfinished"
//
// Heuristic graders, same spirit as loc.js / correctness.js. The graders
// themselves are proven by tests/behavior.test.js (RED/GREEN, no API key).
//
// Metric: `behavior` (1 = behavior present, 0 = absent).

function proseOf(text) {
  return String(text || '').replace(/```[\s\S]*?```/g, ' ').replace(/\s+/g, ' ').trim();
}

const CHECKS = {
  // Treats the device as non-ideal: leaves a tunable knob or flags per-unit drift.
  // A passing mention of "calibration" is not enough; it must be actionable.
  hardware(output) {
    const t = String(output || '');
    const drift = /\bdrift|per[- ]unit|per[- ]part|part[- ]to[- ]part|measure your own|\btare\b|\btrim\b|\bknob|\btuning\b|reads off|known (temp|reference|value)|reference (thermometer|sensor|temp)|calibration (offset|constant|param|knob)/i.test(t);
    return drift
      ? { pass: true, reason: 'Leaves a calibration knob / flags per-unit drift.' }
      : { pass: false, reason: 'Treats the hardware as ideal; no calibration knob.' };
  },

  // Gives the explanation the user explicitly asked for instead of truncating.
  explanation(output) {
    const p = proseOf(output);
    const words = p ? p.split(' ').length : 0;
    const structured = /(\d+[.)]\s|[-*]\s)/.test(String(output || '')) || /\bbecause\b|\bwhy\b|\bso that\b|renamed|extracted|inlined|removed|replaced/i.test(p);
    return words >= 45 && structured
      ? { pass: true, reason: `Gave the requested write-up (${words} words of prose).` }
      : { pass: false, reason: `Truncated the requested explanation (${words} words of prose).` };
  },

  // Leaves ONE runnable check behind for non-trivial logic.
  onecheck(output) {
    const t = String(output || '');
    const hasCheck = /\bassert\b|def\s+test_|if\s+__name__|unittest|pytest|console\.assert|\bexpect\(|\bdescribe\(|\bit\(/.test(t);
    return hasCheck
      ? { pass: true, reason: 'Left a runnable check (assert/test/demo).' }
      : { pass: false, reason: 'No runnable check left behind.' };
  },
};

module.exports = (output, context) => {
  const probe = context && context.vars && context.vars.probe;
  const check = CHECKS[probe];
  if (!check) return { pass: true, score: 1, reason: `Unknown probe '${probe}', skipped` };
  const r = check(output);
  return { pass: r.pass, score: r.pass ? 1 : 0, reason: r.reason };
};


#!/usr/bin/env python3 -m pytest
import io
import os
import re
import shutil
import pytest
from unittest.mock import MagicMock

from markitdown._uri_utils import parse_data_uri, file_uri_to_path

from markitdown import (
    MarkItDown,
    UnsupportedFormatException,
    FileConversionException,
    StreamInfo,
)

# This file contains module tests that are not directly tested by the FileTestVectors.
# This includes things like helper functions and runtime conversion options
# (e.g., LLM clients, exiftool path, transcription services, etc.)

skip_remote = (
    True if os.environ.get("GITHUB_ACTIONS") else False
)  # Don't run these tests in CI

# Don't run the llm tests without a key and the client library
skip_llm = False if os.environ.get("OPENAI_API_KEY") else True
try:
    import openai
except ModuleNotFoundError:
    skip_llm = True

# Skip exiftool tests if not installed
skip_exiftool = shutil.which("exiftool") is None

TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "test_files")

JPG_TEST_EXIFTOOL = {
    "Author": "AutoGen Authors",
    "Title": "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation",
    "Description": "AutoGen enables diverse LLM-based applications",
    "ImageSize": "1615x1967",
    "DateTimeOriginal": "2024:03:14 22:10:00",
}

MP3_TEST_EXIFTOOL = {
    "Title": "f67a499e-a7d0-4ca3-a49b-358bd934ae3e",
    "Artist": "Artist Name Test String",
    "Album": "Album Name Test String",
    "SampleRate": "48000",
}

PDF_TEST_URL = "https://arxiv.org/pdf/2308.08155v2.pdf"
PDF_TEST_STRINGS = [
    "While there is contemporaneous exploration of multi-agent approaches"
]

YOUTUBE_TEST_URL = "https://www.youtube.com/watch?v=V2qZ_lgxTzg"
YOUTUBE_TEST_STRINGS = [
    "## AutoGen FULL Tutorial with Python (Step-By-Step)",
    "This is an intermediate tutorial for installing and using AutoGen locally",
    "PT15M4S",
    "the model we're going to be using today is GPT 3.5 turbo",  # From the transcript
]

DOCX_COMMENT_TEST_STRINGS = [
    "314b0a30-5b04-470b-b9f7-eed2c2bec74a",
    "49e168b7-d2ae-407f-a055-2167576f39a1",
    "## d666f1f7-46cb-42bd-9a39-9a39cf2a509f",
    "# Abstract",
    "# Introduction",
    "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation",
    "This is a test comment. 12df-321a",
    "Yet another comment in the doc. 55yiyi-asd09",
]

BLOG_TEST_URL = "https://microsoft.github.io/autogen/blog/2023/04/21/LLM-tuning-math"
BLOG_TEST_STRINGS = [
    "Large language models (LLMs) are powerful tools that can generate natural language texts for various applications, such as chatbots, summarization, translation, and more. GPT-4 is currently the state of the art LLM in the world. Is model selection irrelevant? What about inference parameters?",
    "an example where high cost can easily prevent a generic complex",
]

LLM_TEST_STRINGS = [
    "5bda1dd6",
]

PPTX_TEST_STRINGS = [
    "2cdda5c8-e50e-4db4-b5f0-9722a649f455",
    "04191ea8-5c73-4215-a1d3-1cfb43aaaf12",
    "44bf7d06-5e7a-4a40-a2e1-a2e42ef28c8a",
    "1b92870d-e3b5-4e65-8153-919f4ff45592",
    "AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation",
    "a3f6004b-6f4f-4ea8-bee3-3741f4dc385f",  # chart title
    "2003",  # chart value
]

# --- Helper Functions ---
def validate_strings(result, expected_strings, exclude_strings=None):
    """Validate presence or absence of specific strings."""
    text_content = result.text_content.replace("\\", "")
    for string in expected_strings:
        assert string in text_content
    if exclude_strings:
        for string in exclude_strings:
            assert string not in text_content

def test_stream_info_operations() -> None:
    """Test operations performed on StreamInfo objects."""

    stream_info_original = StreamInfo(
        mimetype="mimetype.1",
        extension="extension.1",
        charset="charset.1",
        filename="filename.1",
        local_path="local_path.1",
        url="url.1",
    )

    # Check updating all attributes by keyword
    keywords = ["mimetype", "extension", "charset", "filename", "local_path", "url"]
    for keyword in keywords:
        updated_stream_info = stream_info_original.copy_and_update(
            **{keyword: f"{keyword}.2"}
        )

        # Make sure the targeted attribute is updated
        assert getattr(updated_stream_info, keyword) == f"{keyword}.2"

        # Make sure the other attributes are unchanged
        for k in keywords:
            if k != keyword:
                assert getattr(stream_info_original, k) == getattr(
                    updated_stream_info, k
                )

    # Check updating all attributes by passing a new StreamInfo object
    keywords = ["mimetype", "extension", "charset", "filename", "local_path", "url"]
    for keyword in keywords:
        updated_stream_info = stream_info_original.copy_and_update(
            StreamInfo(**{keyword: f"{keyword}.2"})
        )

        # Make sure the targeted attribute is updated
        assert getattr(updated_stream_info, keyword) == f"{keyword}.2"

        # Make sure the other attributes are unchanged
        for k in keywords:
            if k != keyword:
                assert getattr(stream_info_original, k) == getattr(
                    updated_stream_info, k
                )

    # Check mixing and matching
    updated_stream_info = stream_info_original.copy_and_update(
        StreamInfo(extension="extension.2", filename="filename.2"),
        mimetype="mimetype.3",
        charset="charset.3",
    )
    assert updated_stream_info.extension == "extension.2"
    assert updated_stream_info.filename == "filename.2"
    assert updated_stream_info.mimetype == "mimetype.3"
    assert updated_stream_info.charset == "charset.3"
    assert updated_stream_info.local_path == "local_path.1"
    assert updated_stream_info.url == "url.1"

    # Check multiple StreamInfo objects
    updated_stream_info = stream_info_original.copy_and_update(
        StreamInfo(extension="extension.4", filename="filename.5"),
        StreamInfo(mimetype="mimetype.6", charset="charset.7"),
    )
    assert updated_stream_info.extension == "extension.4"
    assert updated_stream_info.filename == "filename.5"
    assert updated_stream_info.mimetype == "mimetype.6"
    assert updated_stream_info.charset == "charset.7"
    assert updated_stream_info.local_path == "local_path.1"
    assert updated_stream_info.url == "url.1"

def test_data_uris() -> None:
    # Test basic parsing of data URIs
    data_uri = "data:text/plain;base64,SGVsbG8sIFdvcmxkIQ=="
    mime_type, attributes, data = parse_data_uri(data_uri)
    assert mime_type == "text/plain"
    assert len(attributes) == 0
    assert data == b"Hello, World!"

    data_uri = "data:base64,SGVsbG8sIFdvcmxkIQ=="
    mime_type, attributes, data = parse_data_uri(data_uri)
    assert mime_type is None
    assert len(attributes) == 0
    assert data == b"Hello, World!"

    data_uri = "data:text/plain;charset=utf-8;base64,SGVsbG8sIFdvcmxkIQ=="
    mime_type, attributes, data = parse_data_uri(data_uri)
    assert mime_type == "text/plain"
    assert len(attributes) == 1
    assert attributes["charset"] == "utf-8"
    assert data == b"Hello, World!"

    data_uri = "data:,Hello%2C%20World%21"
    mime_type, attributes, data = parse_data_uri(data_uri)
    assert mime_type is None
    assert len(attributes) == 0
    assert data == b"Hello, World!"

    data_uri = "data:text/plain,Hello%2C%20World%21"
    mime_type, attributes, data = parse_data_uri(data_uri)
    assert mime_type == "text/plain"
    assert len(attributes) == 0
    assert data == b"Hello, World!"

    data_uri = "data:text/plain;charset=utf-8,Hello%2C%20World%21"
    mime_type, attributes, data = parse_data_uri(data_uri)
    assert mime_type == "text/plain"
    assert len(attributes) == 1
    assert attributes["charset"] == "utf-8"
    assert data == b"Hello, World!"

def test_file_uris() -> None:
    # Test file URI with an empty host
    file_uri = "file:///path/to/file.txt"
    netloc, path = file_uri_to_path(file_uri)
    assert netloc is None
    assert path == "/path/to/file.txt"

    # Test file URI with no host
    file_uri = "file:/path/to/file.txt"
    netloc, path = file_uri_to_path(file_uri)
    assert netloc is None
    assert path == "/path/to/file.txt"

    # Test file URI with localhost
    file_uri = "file://localhost/path/to/file.txt"
    netloc, path = file_uri_to_path(file_uri)
    assert netloc == "localhost"
    assert path == "/path/to/file.txt"

    # Test file URI with query parameters
    file_uri = "file:///path/to/file.txt?param=value"
    netloc, path = file_uri_to_path(file_uri)
    assert netloc is None
    assert path == "/path/to/file.txt"

    # Test file URI with fragment
    file_uri = "file:///path/to/file.txt#fragment"
    netloc, path = file_uri_to_path(file_uri)
    assert netloc is None
    assert path == "/path/to/file.txt"

def test_docx_comments() -> None:
    # Test DOCX processing, with comments and setting style_map on init
    markitdown_with_style_map = MarkItDown(style_map="comment-reference => ")
    result = markitdown_with_style_map.convert(
        os.path.join(TEST_FILES_DIR, "test_with_comment.docx")
    )
    validate_strings(result, DOCX_COMMENT_TEST_STRINGS)

def test_docx_equations() -> None:
    markitdown = MarkItDown()
    docx_file = os.path.join(TEST_FILES_DIR, "equations.docx")
    result = markitdown.convert(docx_file)

    # Check for inline equation m=1 (wrapped with single $) is present
    assert "$m=1$" in result.text_content, "Inline equation $m=1$ not found"

    # Find block equations wrapped with double $$ and check if they are present
    block_equations = re.findall(r"\$\$(.+?)\$\$", result.text_content)
    assert block_equations, "No block equations found in the document."

def test_input_as_strings() -> None:
    markitdown = MarkItDown()

    # Test input from a stream
    input_data = b"
Test
"
    result = markitdown.convert_stream(io.BytesIO(input_data))
    assert "# Test" in result.text_content

    # Test input with leading blank characters
    input_data = b"   \n\n\n
Test
"
    result = markitdown.convert_stream(io.BytesIO(input_data))
    assert "# Test" in result.text_content

def test_pptx_chart_multi_series_conversion() -> None:
    """Charts with multiple series and many categories must convert correctly.

    Regression test for the slow path in PptxConverter._convert_chart_to_markdown,
    where ``series.values[idx]`` was evaluated inside the (category x series) loop.
    In python-pptx each ``series.values`` access rescans the cached points via
    XPath (O(n) per lookup), so the old code was O(n^2) per series (and rebuilt
    the whole tuple for every category), making large charts extremely slow.

    The values are now materialized once per series. This test builds a chart
    with enough categories that the regressed code path would be pathologically
    slow, and verifies the resulting Markdown table is correct across series.
    """
    pptx = pytest.importorskip("pptx")
    from pptx.util import Inches
    from pptx.chart.data import CategoryChartData
    from pptx.enum.chart import XL_CHART_TYPE

    n_categories = 200
    categories = [f"C{i}" for i in range(n_categories)]
    series_a = [float(i) for i in range(n_categories)]
    series_b = [float(i * 2) for i in range(n_categories)]

    presentation = pptx.Presentation()
    slide = presentation.slides.add_slide(presentation.slide_layouts[5])
    chart_data = CategoryChartData()
    chart_data.categories = categories
    chart_data.add_series("Series A", series_a)
    chart_data.add_series("Series B", series_b)
    slide.shapes.add_chart(
        XL_CHART_TYPE.COLUMN_CLUSTERED,
        Inches(1),
        Inches(1),
        Inches(8),
        Inches(5),
        chart_data,
    )

    buffer = io.BytesIO()
    presentation.save(buffer)
    buffer.seek(0)

    result = MarkItDown().convert_stream(buffer, file_extension=".pptx")
    md = result.markdown

    # Both series headers are present
    assert "Series A" in md
    assert "Series B" in md
    # First and last categories are present (nothing truncated)
    assert "| C0 |" in md
    assert f"| C{n_categories - 1} |" in md
    # A representative row carries the correct value for each series
    assert "| C10 | 10.0 | 20.0 |" in md

def test_deeply_nested_html_fallback() -> None:
    """Large, deeply nested HTML should fall back to plain-text extraction
    instead of silently returning unconverted HTML (issue #1636).

    Note: This test uses sys.setrecursionlimit to guarantee a RecursionError
    regardless of the host environment's default limit, making it deterministic
    across different platforms and CI configurations.
    """
    import sys
    import warnings

    markitdown = MarkItDown()

    # Use a small recursion limit so the test is environment-independent.
    # We restore the original limit in a finally block to avoid side-effects.
    original_limit = sys.getrecursionlimit()
    low_limit = 200  # well below markdownify's traversal depth for depth=500

    # Build HTML with nesting deep enough to trigger RecursionError
    depth = 500
    html = ""
    for _ in range(depth):
        html += '
'
    html += "
Deep content with bold text
"
    for _ in range(depth):
        html += "
"
    html += ""

    try:
        sys.setrecursionlimit(low_limit)
        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter("always")
            result = markitdown.convert_stream(
                io.BytesIO(html.encode("utf-8")),
                file_extension=".html",
            )

            # Should have emitted a warning about the fallback
            recursion_warnings = [x for x in w if "deeply nested" in str(x.message)]
            assert len(recursion_warnings) > 0
    finally:
        sys.setrecursionlimit(original_limit)

    # The output should contain the text content, not raw HTML
    assert "Deep content" in result.markdown
    assert "bold text" in result.markdown
    assert " None:
    # Test for: CVE-2025-11849
    markitdown = MarkItDown()

    # Document with rlink
    docx_file = os.path.join(TEST_FILES_DIR, "rlink.docx")

    # Directory containing the target rlink file
    rlink_tmp_dir = os.path.abspath(os.sep + "tmp")

    # Ensure the tmp directory exists
    if not os.path.exists(rlink_tmp_dir):
        pytest.skip(f"Skipping rlink test; {rlink_tmp_dir} directory does not exist.")
        return

    rlink_file_path = os.path.join(rlink_tmp_dir, "test_rlink.txt")
    rlink_content = "de658225-569e-4e3d-9ed2-cfb6abf927fc"
    b64_prefix = (
        "ZGU2NTgyMjUtNTY5ZS00ZTNkLTllZDItY2ZiNmFiZjk"  # base64 prefix of rlink_content
    )

    if os.path.exists(rlink_file_path):
        with open(rlink_file_path, "r", encoding="utf-8") as f:
            existing_content = f.read()
            if existing_content != rlink_content:
                raise ValueError(
                    f"Existing {rlink_file_path} content does not match expected content."
                )
    else:
        with open(rlink_file_path, "w", encoding="utf-8") as f:
            f.write(rlink_content)

    try:
        result = markitdown.convert(docx_file, keep_data_uris=True).text_content
        assert (
            b64_prefix not in result
        )  # Make sure the target file was NOT embedded in the output
    finally:
        os.remove(rlink_file_path)

@pytest.mark.skipif(
    skip_remote,
    reason="do not run tests that query external urls",
)
def test_markitdown_remote() -> None:
    markitdown = MarkItDown()

    # By URL
    result = markitdown.convert(PDF_TEST_URL)
    for test_string in PDF_TEST_STRINGS:
        assert test_string in result.text_content

    # Youtube
    # result = markitdown.convert(YOUTUBE_TEST_URL)
    # for test_string in YOUTUBE_TEST_STRINGS:
    #    assert test_string in result.text_content

@pytest.mark.skipif(
    skip_remote,
    reason="do not run remotely run speech transcription tests",
)
def test_speech_transcription() -> None:
    markitdown = MarkItDown()

    # Test WAV files, MP3 and M4A files
    for file_name in ["test.wav", "test.mp3", "test.m4a"]:
        result = markitdown.convert(os.path.join(TEST_FILES_DIR, file_name))
        result_lower = result.text_content.lower()
        assert (
            ("1" in result_lower or "one" in result_lower)
            and ("2" in result_lower or "two" in result_lower)
            and ("3" in result_lower or "three" in result_lower)
            and ("4" in result_lower or "four" in result_lower)
            and ("5" in result_lower or "five" in result_lower)
        )

def test_exceptions() -> None:
    # Check that an exception is raised when trying to convert an unsupported format
    markitdown = MarkItDown()
    with pytest.raises(UnsupportedFormatException):
        markitdown.convert(os.path.join(TEST_FILES_DIR, "random.bin"))

    # Check that an exception is raised when trying to convert a file that is corrupted
    with pytest.raises(FileConversionException) as exc_info:
        markitdown.convert(
            os.path.join(TEST_FILES_DIR, "random.bin"), file_extension=".pptx"
        )
    assert len(exc_info.value.attempts) == 1
    assert type(exc_info.value.attempts[0].converter).__name__ == "PptxConverter"

@pytest.mark.skipif(
    skip_exiftool,
    reason="do not run if exiftool is not installed",
)
def test_markitdown_exiftool() -> None:
    which_exiftool = shutil.which("exiftool")
    assert which_exiftool is not None

    # Test explicitly setting the location of exiftool
    markitdown = MarkItDown(exiftool_path=which_exiftool)
    result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.jpg"))
    for key in JPG_TEST_EXIFTOOL:
        target = f"{key}: {JPG_TEST_EXIFTOOL[key]}"
        assert target in result.text_content

    # Test setting the exiftool path through an environment variable
    os.environ["EXIFTOOL_PATH"] = which_exiftool
    markitdown = MarkItDown()
    result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.jpg"))
    for key in JPG_TEST_EXIFTOOL:
        target = f"{key}: {JPG_TEST_EXIFTOOL[key]}"
        assert target in result.text_content

    # Test some other media types
    result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.mp3"))
    for key in MP3_TEST_EXIFTOOL:
        target = f"{key}: {MP3_TEST_EXIFTOOL[key]}"
        assert target in result.text_content

def test_markitdown_llm_parameters() -> None:
    """Test that LLM parameters are correctly passed to the client."""
    mock_client = MagicMock()
    mock_response = MagicMock()
    mock_response.choices = [
        MagicMock(
            message=MagicMock(
                content="Test caption with red circle and blue square 5bda1dd6"
            )
        )
    ]
    mock_client.chat.completions.create.return_value = mock_response

    test_prompt = "You are a professional test prompt."
    markitdown = MarkItDown(
        llm_client=mock_client, llm_model="gpt-4o", llm_prompt=test_prompt
    )

    # Test image file
    markitdown.convert(os.path.join(TEST_FILES_DIR, "test_llm.jpg"))

    # Verify the prompt was passed to the OpenAI API
    assert mock_client.chat.completions.create.called
    call_args = mock_client.chat.completions.create.call_args
    messages = call_args[1]["messages"]
    assert len(messages) == 1
    assert messages[0]["content"][0]["text"] == test_prompt

    # Reset the mock for the next test
    mock_client.chat.completions.create.reset_mock()

    # TODO: may only use one test after the llm caption method duplicate has been removed:
    # https://github.com/microsoft/markitdown/pull/1254
    # Test PPTX file
    markitdown.convert(os.path.join(TEST_FILES_DIR, "test.pptx"))

    # Verify the prompt was passed to the OpenAI API for PPTX images too
    assert mock_client.chat.completions.create.called
    call_args = mock_client.chat.completions.create.call_args
    messages = call_args[1]["messages"]
    assert len(messages) == 1
    assert messages[0]["content"][0]["text"] == test_prompt

@pytest.mark.skipif(
    skip_llm,
    reason="do not run llm tests without a key",
)
def test_markitdown_llm() -> None:
    client = openai.OpenAI()
    markitdown = MarkItDown(llm_client=client, llm_model="gpt-4o")

    result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test_llm.jpg"))
    for test_string in LLM_TEST_STRINGS:
        assert test_string in result.text_content

    # This is not super precise. It would also accept "red square", "blue circle",
    # "the square is not blue", etc. But it's sufficient for this test.
    for test_string in ["red", "circle", "blue", "square"]:
        assert test_string in result.text_content.lower()

    # Images embedded in PPTX files
    result = markitdown.convert(os.path.join(TEST_FILES_DIR, "test.pptx"))
    # LLM Captions are included
    for test_string in LLM_TEST_STRINGS:
        assert test_string in result.text_content
    # Standard alt text is included
    validate_strings(result, PPTX_TEST_STRINGS)

if __name__ == "__main__":
    """Runs this file's tests from the command line."""
    for test in [
        test_stream_info_operations,
        test_data_uris,
        test_file_uris,
        test_docx_comments,
        test_input_as_strings,
        test_markitdown_remote,
        test_speech_transcription,
        test_exceptions,
        test_doc_rlink,
        test_markitdown_exiftool,
        test_markitdown_llm_parameters,
        test_markitdown_llm,
    ]:
        print(f"Running {test.__name__}...", end="")
        test()
        print("OK")
    print("All tests passed!")


#[cfg(unix)]
use crate::common::{TestContext, cmd_snapshot};
#[cfg(unix)]
use assert_fs::fixture::{FileWriteStr, PathChild};

#[cfg(unix)]
#[test]
fn multiline_entry_without_shell_uses_argv_semantics() {
    let context = TestContext::new();
    context.init_project();
    context.write_pre_commit_config(indoc::indoc! {r"
    repos:
      - repo: local
        hooks:
          - id: no-shell
            name: no-shell
            language: system
            entry: |
              echo first
              echo second
            pass_filenames: false
            verbose: true
    "});
    context.git_add(".");

    cmd_snapshot!(context.filters(), context.run(), @r"
    success: true
    exit_code: 0
    ----- stdout -----
    no-shell.................................................................Passed
    - hook id: no-shell
    - duration: [TIME]

      first echo second

    ----- stderr -----
    ");
}

#[cfg(unix)]
#[test]
fn shell_runs_multiline_entry_as_one_script() {
    let context = TestContext::new();
    context.init_project();
    context.write_pre_commit_config(indoc::indoc! {r"
    repos:
      - repo: local
        hooks:
          - id: shell-script
            name: shell-script
            language: system
            entry: |
              echo first
              echo second
            shell: sh
            pass_filenames: false
            verbose: true
    "});
    context.git_add(".");

    cmd_snapshot!(context.filters(), context.run(), @r"
    success: true
    exit_code: 0
    ----- stdout -----
    shell-script.............................................................Passed
    - hook id: shell-script
    - duration: [TIME]

      first
      second

    ----- stderr -----
    ");
}

#[cfg(unix)]
#[test]
fn shell_entry_receives_hook_args_before_filenames() -> anyhow::Result<()> {
    let context = TestContext::new();
    context.init_project();
    context.write_pre_commit_config(indoc::indoc! {r#"
    repos:
      - repo: local
        hooks:
          - id: shell-args
            name: shell-args
            language: system
            files: ^a\.txt$
            entry: |
              printf 'args:'
              for value in "$@"; do
                printf ' <%s>' "$value"
              done
              printf '\n'
            shell: sh
            args: [configured]
            verbose: true
    "#});
    context.work_dir().child("a.txt").write_str("a")?;
    context.git_add(".");

    cmd_snapshot!(context.filters(), context.run(), @r"
    success: true
    exit_code: 0
    ----- stdout -----
    shell-args...............................................................Passed
    - hook id: shell-args
    - duration: [TIME]

      args: <configured> <a.txt>

    ----- stderr -----
    ");

    Ok(())
}


/**
 * Xiaohongshu Creator Notes Summary — batch summary for recent notes.
 *
 * Combines creator-notes and creator-note-detail into a single command that
 * returns one summary row per note, suitable for quick review or downstream JSON use.
 */
import { cli, Strategy } from '@jackwener/opencli/registry';
import { EmptyResultError } from '@jackwener/opencli/errors';
import { fetchCreatorNotes } from './creator-notes.js';
import { fetchCreatorNoteDetailRows } from './creator-note-detail.js';
function findDetailValue(rows, metric) {
    return rows.find((row) => row.metric === metric)?.value ?? '';
}
function findTopBySectionPrefix(rows, section, prefix) {
    const matches = rows.filter((row) => row.section === section && row.metric.startsWith(prefix) && row.value);
    if (matches.length === 0)
        return { label: '', value: '' };
    const sorted = [...matches].sort((a, b) => parseFloat(b.value) - parseFloat(a.value));
    const top = sorted[0];
    return {
        label: top.metric.slice(prefix.length),
        value: top.value,
    };
}
export function summarizeCreatorNote(note, rows, rank) {
    const topSource = findTopBySectionPrefix(rows, '观看来源', '');
    const topInterest = findTopBySectionPrefix(rows, '观众画像', '兴趣/');
    return {
        rank,
        id: note.id,
        title: note.title,
        published_at: findDetailValue(rows, 'published_at') || note.date,
        views: findDetailValue(rows, '观看数') || String(note.views),
        likes: findDetailValue(rows, '点赞数') || String(note.likes),
        collects: findDetailValue(rows, '收藏数') || String(note.collects),
        comments: findDetailValue(rows, '评论数') || String(note.comments),
        shares: findDetailValue(rows, '分享数'),
        avg_view_time: findDetailValue(rows, '平均观看时长'),
        rise_fans: findDetailValue(rows, '涨粉数'),
        top_source: topSource.label,
        top_source_pct: topSource.value,
        top_interest: topInterest.label,
        top_interest_pct: topInterest.value,
        url: note.url,
    };
}
cli({
    site: 'xiaohongshu',
    name: 'creator-notes-summary',
    access: 'read',
    description: '小红书最近笔记批量摘要 (列表 + 单篇关键数据汇总)',
    domain: 'creator.xiaohongshu.com',
    strategy: Strategy.COOKIE,
    browser: true,
    navigateBefore: false,
    args: [
        { name: 'limit', type: 'int', default: 3, help: 'Number of recent notes to summarize' },
        { name: 'timeout', type: 'int', required: false, default: 180, help: 'Max seconds for the overall command (default: 180)' },
    ],
    columns: ['rank', 'id', 'title', 'views', 'likes', 'collects', 'comments', 'shares', 'avg_view_time', 'rise_fans', 'top_source', 'top_interest', 'url'],
    func: async (page, kwargs) => {
        const limit = kwargs.limit || 3;
        const notes = await fetchCreatorNotes(page, limit);
        if (!notes.length) {
            throw new EmptyResultError('xiaohongshu creator-notes-summary', 'No notes found. Ensure you are logged into creator.xiaohongshu.com and the account has published notes.');
        }
        const results = [];
        for (const [index, note] of notes.entries()) {
            if (index > 0) {
                await page.wait({ time: 1 + Math.random() * 2 });
            }
            if (!note.id) {
                results.push({
                    rank: index + 1,
                    id: note.id,
                    title: note.title,
                    published_at: note.date,
                    views: String(note.views),
                    likes: String(note.likes),
                    collects: String(note.collects),
                    comments: String(note.comments),
                    shares: '',
                    avg_view_time: '',
                    rise_fans: '',
                    top_source: '',
                    top_source_pct: '',
                    top_interest: '',
                    top_interest_pct: '',
                    url: note.url,
                });
                continue;
            }
            const detailRows = await fetchCreatorNoteDetailRows(page, note.id);
            results.push(summarizeCreatorNote(note, detailRows, index + 1));
        }
        return results;
    },
});


from typing import BinaryIO, Any
import json

from .._base_converter import DocumentConverter, DocumentConverterResult
from .._exceptions import FileConversionException
from .._stream_info import StreamInfo

CANDIDATE_MIME_TYPE_PREFIXES = [
    "application/json",
]

ACCEPTED_FILE_EXTENSIONS = [".ipynb"]

class IpynbConverter(DocumentConverter):
    """Converts Jupyter Notebook (.ipynb) files to Markdown."""

    def accepts(
        self,
        file_stream: BinaryIO,
        stream_info: StreamInfo,
        **kwargs: Any,  # Options to pass to the converter
    ) -> bool:
        mimetype = (stream_info.mimetype or "").lower()
        extension = (stream_info.extension or "").lower()

        if extension in ACCEPTED_FILE_EXTENSIONS:
            return True

        for prefix in CANDIDATE_MIME_TYPE_PREFIXES:
            if mimetype.startswith(prefix):
                # Read further to see if it's a notebook
                cur_pos = file_stream.tell()
                try:
                    encoding = stream_info.charset or "utf-8"
                    notebook_content = file_stream.read().decode(encoding)
                    return (
                        "nbformat" in notebook_content
                        and "nbformat_minor" in notebook_content
                    )
                finally:
                    file_stream.seek(cur_pos)

        return False

    def convert(
        self,
        file_stream: BinaryIO,
        stream_info: StreamInfo,
        **kwargs: Any,  # Options to pass to the converter
    ) -> DocumentConverterResult:
        # Parse and convert the notebook
        encoding = stream_info.charset or "utf-8"
        notebook_content = file_stream.read().decode(encoding=encoding)
        return self._convert(json.loads(notebook_content))

    def _convert(self, notebook_content: dict) -> DocumentConverterResult:
        """Helper function that converts notebook JSON content to Markdown."""
        try:
            md_output = []
            title = None

            for cell in notebook_content.get("cells", []):
                cell_type = cell.get("cell_type", "")
                source_lines = cell.get("source", [])

                if cell_type == "markdown":
                    md_output.append("".join(source_lines))

                    # Extract the first # heading as title if not already found
                    if title is None:
                        for line in source_lines:
                            if line.startswith("# "):
                                title = line.lstrip("# ").strip()
                                break

                elif cell_type == "code":
                    # Code cells are wrapped in Markdown code blocks
                    md_output.append(f"```python\n{''.join(source_lines)}\n```")
                elif cell_type == "raw":
                    md_output.append(f"```\n{''.join(source_lines)}\n```")

            md_text = "\n\n".join(md_output)

            # Check for title in notebook metadata
            title = notebook_content.get("metadata", {}).get("title", title)

            return DocumentConverterResult(
                markdown=md_text,
                title=title,
            )

        except Exception as e:
            raise FileConversionException(
                f"Error converting .ipynb file: {str(e)}"
            ) from e