/**
 * sqlite-vec-rescore.c — Rescore index logic for sqlite-vec.
 *
 * This file is #included into sqlite-vec.c after the vec0_vtab definition.
 * All functions receive a vec0_vtab *p and access p->vector_columns[i].rescore.
 *
 * Shadow tables per rescore-enabled vector column:
 *   _rescore_chunks{NN}  — quantized vectors in chunk layout (for coarse scan)
 *   _rescore_vectors{NN} — float vectors keyed by rowid (for fast rescore lookup)
 */

// ============================================================================
// Shadow table lifecycle
// ============================================================================

static int rescore_create_tables(vec0_vtab *p, sqlite3 *db, char **pzErr) {
  for (int i = 0; i < p->numVectorColumns; i++) {
    if (p->vector_columns[i].index_type != VEC0_INDEX_TYPE_RESCORE)
      continue;

    // Quantized chunk table (same layout as _vector_chunks)
    char *zSql = sqlite3_mprintf(
        "CREATE TABLE \"%w\".\"%w_rescore_chunks%02d\""
        "(rowid PRIMARY KEY, vectors BLOB NOT NULL)",
        p->schemaName, p->tableName, i);
    if (!zSql)
      return SQLITE_NOMEM;
    sqlite3_stmt *stmt;
    int rc = sqlite3_prepare_v2(db, zSql, -1, &stmt, 0);
    sqlite3_free(zSql);
    if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) {
      *pzErr = sqlite3_mprintf(
          "Could not create '_rescore_chunks%02d' shadow table: %s", i,
          sqlite3_errmsg(db));
      sqlite3_finalize(stmt);
      return SQLITE_ERROR;
    }
    sqlite3_finalize(stmt);

    // Float vector table (rowid-keyed for fast random access)
    zSql = sqlite3_mprintf(
        "CREATE TABLE \"%w\".\"%w_rescore_vectors%02d\""
        "(rowid INTEGER PRIMARY KEY, vector BLOB NOT NULL)",
        p->schemaName, p->tableName, i);
    if (!zSql)
      return SQLITE_NOMEM;
    rc = sqlite3_prepare_v2(db, zSql, -1, &stmt, 0);
    sqlite3_free(zSql);
    if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) {
      *pzErr = sqlite3_mprintf(
          "Could not create '_rescore_vectors%02d' shadow table: %s", i,
          sqlite3_errmsg(db));
      sqlite3_finalize(stmt);
      return SQLITE_ERROR;
    }
    sqlite3_finalize(stmt);
  }
  return SQLITE_OK;
}

static int rescore_drop_tables(vec0_vtab *p) {
  for (int i = 0; i < p->numVectorColumns; i++) {
    sqlite3_stmt *stmt;
    int rc;
    char *zSql;

    if (p->shadowRescoreChunksNames[i]) {
      zSql = sqlite3_mprintf("DROP TABLE IF EXISTS \"%w\".\"%w\"",
                              p->schemaName, p->shadowRescoreChunksNames[i]);
      if (!zSql)
        return SQLITE_NOMEM;
      rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, 0);
      sqlite3_free(zSql);
      if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) {
        sqlite3_finalize(stmt);
        return SQLITE_ERROR;
      }
      sqlite3_finalize(stmt);
    }

    if (p->shadowRescoreVectorsNames[i]) {
      zSql = sqlite3_mprintf("DROP TABLE IF EXISTS \"%w\".\"%w\"",
                              p->schemaName, p->shadowRescoreVectorsNames[i]);
      if (!zSql)
        return SQLITE_NOMEM;
      rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, 0);
      sqlite3_free(zSql);
      if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) {
        sqlite3_finalize(stmt);
        return SQLITE_ERROR;
      }
      sqlite3_finalize(stmt);
    }
  }
  return SQLITE_OK;
}

static size_t rescore_quantized_byte_size(struct VectorColumnDefinition *col) {
  switch (col->rescore.quantizer_type) {
  case VEC0_RESCORE_QUANTIZER_BIT:
    return col->dimensions / CHAR_BIT;
  case VEC0_RESCORE_QUANTIZER_INT8:
    return col->dimensions;
  default:
    return 0;
  }
}

/**
 * Insert a new chunk row into each _rescore_chunks{NN} table with a zeroblob.
 */
static int rescore_new_chunk(vec0_vtab *p, i64 chunk_rowid) {
  for (int i = 0; i < p->numVectorColumns; i++) {
    if (p->vector_columns[i].index_type != VEC0_INDEX_TYPE_RESCORE)
      continue;
    size_t quantized_size =
        rescore_quantized_byte_size(&p->vector_columns[i]);
    i64 blob_size = (i64)p->chunk_size * (i64)quantized_size;

    char *zSql = sqlite3_mprintf(
        "INSERT INTO \"%w\".\"%w\"(_rowid_, rowid, vectors) VALUES (?, ?, ?)",
        p->schemaName, p->shadowRescoreChunksNames[i]);
    if (!zSql)
      return SQLITE_NOMEM;
    sqlite3_stmt *stmt;
    int rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL);
    sqlite3_free(zSql);
    if (rc != SQLITE_OK) {
      sqlite3_finalize(stmt);
      return rc;
    }
    sqlite3_bind_int64(stmt, 1, chunk_rowid);
    sqlite3_bind_int64(stmt, 2, chunk_rowid);
    sqlite3_bind_zeroblob64(stmt, 3, blob_size);
    rc = sqlite3_step(stmt);
    sqlite3_finalize(stmt);
    if (rc != SQLITE_DONE)
      return rc;
  }
  return SQLITE_OK;
}

// ============================================================================
// Quantization
// ============================================================================

static void rescore_quantize_float_to_bit(const float *src, uint8_t *dst,
                                          size_t dimensions) {
  memset(dst, 0, dimensions / CHAR_BIT);
  for (size_t i = 0; i < dimensions; i++) {
    if (src[i] >= 0.0f) {
      dst[i / CHAR_BIT] |= (1 << (i % CHAR_BIT));
    }
  }
}

static void rescore_quantize_float_to_int8(const float *src, int8_t *dst,
                                           size_t dimensions) {
  float step = 2.0f / 255.0f;
  for (size_t i = 0; i < dimensions; i++) {
    float v = (src[i] - (-1.0f)) / step - 128.0f;
    if (!(v <= 127.0f)) v = 127.0f;
    if (!(v >= -128.0f)) v = -128.0f;
    dst[i] = (int8_t)v;
  }
}

// ============================================================================
// Insert path
// ============================================================================

/**
 * Quantize float vector to _rescore_chunks and store in _rescore_vectors.
 */
static int rescore_on_insert(vec0_vtab *p, i64 chunk_rowid, i64 chunk_offset,
                             i64 rowid, void *vectorDatas[]) {
  for (int i = 0; i < p->numVectorColumns; i++) {
    if (p->vector_columns[i].index_type != VEC0_INDEX_TYPE_RESCORE)
      continue;

    struct VectorColumnDefinition *col = &p->vector_columns[i];
    size_t qsize = rescore_quantized_byte_size(col);
    size_t fsize = vector_column_byte_size(*col);
    int rc;

    // 1. Write quantized vector to _rescore_chunks blob
    {
      void *qbuf = sqlite3_malloc(qsize);
      if (!qbuf)
        return SQLITE_NOMEM;

      switch (col->rescore.quantizer_type) {
      case VEC0_RESCORE_QUANTIZER_BIT:
        rescore_quantize_float_to_bit((const float *)vectorDatas[i],
                                      (uint8_t *)qbuf, col->dimensions);
        break;
      case VEC0_RESCORE_QUANTIZER_INT8:
        rescore_quantize_float_to_int8((const float *)vectorDatas[i],
                                       (int8_t *)qbuf, col->dimensions);
        break;
      }

      sqlite3_blob *blob = NULL;
      rc = sqlite3_blob_open(p->db, p->schemaName,
                             p->shadowRescoreChunksNames[i], "vectors",
                             chunk_rowid, 1, &blob);
      if (rc != SQLITE_OK) {
        sqlite3_free(qbuf);
        return rc;
      }
      rc = sqlite3_blob_write(blob, qbuf, qsize, chunk_offset * qsize);
      sqlite3_free(qbuf);
      int brc = sqlite3_blob_close(blob);
      if (rc != SQLITE_OK)
        return rc;
      if (brc != SQLITE_OK)
        return brc;
    }

    // 2. Insert float vector into _rescore_vectors (rowid-keyed)
    {
      char *zSql = sqlite3_mprintf(
          "INSERT INTO \"%w\".\"%w\"(rowid, vector) VALUES (?, ?)",
          p->schemaName, p->shadowRescoreVectorsNames[i]);
      if (!zSql)
        return SQLITE_NOMEM;
      sqlite3_stmt *stmt;
      rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL);
      sqlite3_free(zSql);
      if (rc != SQLITE_OK) {
        sqlite3_finalize(stmt);
        return rc;
      }
      sqlite3_bind_int64(stmt, 1, rowid);
      sqlite3_bind_blob(stmt, 2, vectorDatas[i], fsize, SQLITE_TRANSIENT);
      rc = sqlite3_step(stmt);
      sqlite3_finalize(stmt);
      if (rc != SQLITE_DONE)
        return SQLITE_ERROR;
    }
  }
  return SQLITE_OK;
}

// ============================================================================
// Delete path
// ============================================================================

/**
 * Zero out quantized vector in _rescore_chunks and delete from _rescore_vectors.
 */
static int rescore_on_delete(vec0_vtab *p, i64 chunk_id, u64 chunk_offset,
                             i64 rowid) {
  for (int i = 0; i < p->numVectorColumns; i++) {
    if (p->vector_columns[i].index_type != VEC0_INDEX_TYPE_RESCORE)
      continue;
    int rc;

    // 1. Zero out quantized data in _rescore_chunks
    {
      size_t qsize = rescore_quantized_byte_size(&p->vector_columns[i]);
      void *zeroBuf = sqlite3_malloc(qsize);
      if (!zeroBuf)
        return SQLITE_NOMEM;
      memset(zeroBuf, 0, qsize);

      sqlite3_blob *blob = NULL;
      rc = sqlite3_blob_open(p->db, p->schemaName,
                             p->shadowRescoreChunksNames[i], "vectors",
                             chunk_id, 1, &blob);
      if (rc != SQLITE_OK) {
        sqlite3_free(zeroBuf);
        return rc;
      }
      rc = sqlite3_blob_write(blob, zeroBuf, qsize, chunk_offset * qsize);
      sqlite3_free(zeroBuf);
      int brc = sqlite3_blob_close(blob);
      if (rc != SQLITE_OK)
        return rc;
      if (brc != SQLITE_OK)
        return brc;
    }

    // 2. Delete from _rescore_vectors
    {
      char *zSql = sqlite3_mprintf(
          "DELETE FROM \"%w\".\"%w\" WHERE rowid = ?",
          p->schemaName, p->shadowRescoreVectorsNames[i]);
      if (!zSql)
        return SQLITE_NOMEM;
      sqlite3_stmt *stmt;
      rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL);
      sqlite3_free(zSql);
      if (rc != SQLITE_OK)
        return rc;
      sqlite3_bind_int64(stmt, 1, rowid);
      rc = sqlite3_step(stmt);
      sqlite3_finalize(stmt);
      if (rc != SQLITE_DONE)
        return SQLITE_ERROR;
    }
  }
  return SQLITE_OK;
}

/**
 * Delete a chunk row from _rescore_chunks{NN} tables.
 * (_rescore_vectors rows were already deleted per-row in rescore_on_delete)
 */
static int rescore_delete_chunk(vec0_vtab *p, i64 chunk_id) {
  for (int i = 0; i < p->numVectorColumns; i++) {
    if (!p->shadowRescoreChunksNames[i])
      continue;
    char *zSql = sqlite3_mprintf(
        "DELETE FROM \"%w\".\"%w\" WHERE rowid = ?",
        p->schemaName, p->shadowRescoreChunksNames[i]);
    if (!zSql)
      return SQLITE_NOMEM;
    sqlite3_stmt *stmt;
    int rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL);
    sqlite3_free(zSql);
    if (rc != SQLITE_OK)
      return rc;
    sqlite3_bind_int64(stmt, 1, chunk_id);
    rc = sqlite3_step(stmt);
    sqlite3_finalize(stmt);
    if (rc != SQLITE_DONE)
      return SQLITE_ERROR;
  }
  return SQLITE_OK;
}

// ============================================================================
// KNN rescore query
// ============================================================================

/**
 * Phase 1: Coarse scan of quantized chunks → top k*oversample candidates (rowids).
 * Phase 2: For each candidate, blob_open _rescore_vectors by rowid, read float
 *          vector, compute float distance. Sort, return top k.
 *
 * Phase 2 is fast because _rescore_vectors has INTEGER PRIMARY KEY, so
 * sqlite3_blob_open/reopen addresses rows directly by rowid — no index lookup.
 */
static int rescore_knn(vec0_vtab *p, vec0_cursor *pCur,
                       struct VectorColumnDefinition *vector_column,
                       int vectorColumnIdx, struct Array *arrayRowidsIn,
                       struct Array *aMetadataIn, const char *idxStr, int argc,
                       sqlite3_value **argv, void *queryVector, i64 k,
                       struct vec0_query_knn_data *knn_data) {
  (void)pCur;
  (void)aMetadataIn;
  int rc = SQLITE_OK;
  int oversample = vector_column->rescore.oversample_search > 0
      ? vector_column->rescore.oversample_search
      : vector_column->rescore.oversample;
  i64 k_oversample = k * oversample;
  if (k_oversample > 4096)
    k_oversample = 4096;

  size_t qdim = vector_column->dimensions;
  size_t qsize = rescore_quantized_byte_size(vector_column);
  size_t fsize = vector_column_byte_size(*vector_column);

  // Quantize the query vector
  void *quantizedQuery = sqlite3_malloc(qsize);
  if (!quantizedQuery)
    return SQLITE_NOMEM;

  switch (vector_column->rescore.quantizer_type) {
  case VEC0_RESCORE_QUANTIZER_BIT:
    rescore_quantize_float_to_bit((const float *)queryVector,
                                  (uint8_t *)quantizedQuery, qdim);
    break;
  case VEC0_RESCORE_QUANTIZER_INT8:
    rescore_quantize_float_to_int8((const float *)queryVector,
                                   (int8_t *)quantizedQuery, qdim);
    break;
  }

  // Phase 1: Scan quantized chunks for k*oversample candidates
  sqlite3_stmt *stmtChunks = NULL;
  rc = vec0_chunks_iter(p, idxStr, argc, argv, &stmtChunks);
  if (rc != SQLITE_OK) {
    sqlite3_free(quantizedQuery);
    return rc;
  }

  i64 *cand_rowids = sqlite3_malloc(k_oversample * sizeof(i64));
  f32 *cand_distances = sqlite3_malloc(k_oversample * sizeof(f32));
  i64 *tmp_rowids = sqlite3_malloc(k_oversample * sizeof(i64));
  f32 *tmp_distances = sqlite3_malloc(k_oversample * sizeof(f32));
  f32 *chunk_distances = sqlite3_malloc(p->chunk_size * sizeof(f32));
  i32 *chunk_topk_idxs = sqlite3_malloc(k_oversample * sizeof(i32));
  u8 *b = sqlite3_malloc(p->chunk_size / CHAR_BIT);
  u8 *bTaken = sqlite3_malloc(p->chunk_size / CHAR_BIT);
  u8 *bmRowids = NULL;
  void *baseVectors = sqlite3_malloc((i64)p->chunk_size * (i64)qsize);

  if (!cand_rowids || !cand_distances || !tmp_rowids || !tmp_distances ||
      !chunk_distances || !chunk_topk_idxs || !b || !bTaken || !baseVectors) {
    rc = SQLITE_NOMEM;
    goto cleanup;
  }
  memset(cand_rowids, 0, k_oversample * sizeof(i64));
  memset(cand_distances, 0, k_oversample * sizeof(f32));

  if (arrayRowidsIn) {
    bmRowids = sqlite3_malloc(p->chunk_size / CHAR_BIT);
    if (!bmRowids) {
      rc = SQLITE_NOMEM;
      goto cleanup;
    }
  }

  i64 cand_used = 0;

  while (1) {
    rc = sqlite3_step(stmtChunks);
    if (rc == SQLITE_DONE)
      break;
    if (rc != SQLITE_ROW) {
      rc = SQLITE_ERROR;
      goto cleanup;
    }

    i64 chunk_id = sqlite3_column_int64(stmtChunks, 0);
    unsigned char *chunkValidity =
        (unsigned char *)sqlite3_column_blob(stmtChunks, 1);
    i64 *chunkRowids = (i64 *)sqlite3_column_blob(stmtChunks, 2);
    int validityBytes = sqlite3_column_bytes(stmtChunks, 1);
    int rowidsBytes = sqlite3_column_bytes(stmtChunks, 2);
    if (!chunkValidity || !chunkRowids) {
      rc = SQLITE_ERROR;
      goto cleanup;
    }
    // Validate blob sizes match chunk_size expectations
    if (validityBytes < (p->chunk_size + 7) / 8 ||
        rowidsBytes < p->chunk_size * (int)sizeof(i64)) {
      rc = SQLITE_ERROR;
      goto cleanup;
    }

    memset(chunk_distances, 0, p->chunk_size * sizeof(f32));
    memset(chunk_topk_idxs, 0, k_oversample * sizeof(i32));
    bitmap_copy(b, chunkValidity, p->chunk_size);

    if (arrayRowidsIn) {
      bitmap_clear(bmRowids, p->chunk_size);
      for (int j = 0; j < p->chunk_size; j++) {
        if (!bitmap_get(chunkValidity, j))
          continue;
        i64 rid = chunkRowids[j];
        void *found = bsearch(&rid, arrayRowidsIn->z, arrayRowidsIn->length,
                               sizeof(i64), _cmp);
        bitmap_set(bmRowids, j, found ? 1 : 0);
      }
      bitmap_and_inplace(b, bmRowids, p->chunk_size);
    }

    // Read quantized vectors
    sqlite3_blob *blobQ = NULL;
    rc = sqlite3_blob_open(p->db, p->schemaName,
                           p->shadowRescoreChunksNames[vectorColumnIdx],
                           "vectors", chunk_id, 0, &blobQ);
    if (rc != SQLITE_OK)
      goto cleanup;
    rc = sqlite3_blob_read(blobQ, baseVectors,
                           (i64)p->chunk_size * (i64)qsize, 0);
    sqlite3_blob_close(blobQ);
    if (rc != SQLITE_OK)
      goto cleanup;

    // Compute quantized distances
    for (int j = 0; j < p->chunk_size; j++) {
      if (!bitmap_get(b, j))
        continue;
      f32 dist = FLT_MAX;
      switch (vector_column->rescore.quantizer_type) {
      case VEC0_RESCORE_QUANTIZER_BIT: {
        const u8 *base_j = ((u8 *)baseVectors) + (j * (qdim / CHAR_BIT));
        dist = distance_hamming(base_j, (u8 *)quantizedQuery, &qdim);
        break;
      }
      case VEC0_RESCORE_QUANTIZER_INT8: {
        const i8 *base_j = ((i8 *)baseVectors) + (j * qdim);
        switch (vector_column->distance_metric) {
        case VEC0_DISTANCE_METRIC_L2:
          dist = distance_l2_sqr_int8(base_j, (i8 *)quantizedQuery, &qdim);
          break;
        case VEC0_DISTANCE_METRIC_COSINE:
          dist = distance_cosine_int8(base_j, (i8 *)quantizedQuery, &qdim);
          break;
        case VEC0_DISTANCE_METRIC_L1:
          dist = (f32)distance_l1_int8(base_j, (i8 *)quantizedQuery, &qdim);
          break;
        }
        break;
      }
      }
      chunk_distances[j] = dist;
    }

    int used1;
    min_idx(chunk_distances, p->chunk_size, b, chunk_topk_idxs,
            min(k_oversample, p->chunk_size), bTaken, &used1);

    i64 merged_used;
    merge_sorted_lists(cand_distances, cand_rowids, cand_used, chunk_distances,
                       chunkRowids, chunk_topk_idxs,
                       min(min(k_oversample, p->chunk_size), used1),
                       tmp_distances, tmp_rowids, k_oversample, &merged_used);

    for (i64 j = 0; j < merged_used; j++) {
      cand_rowids[j] = tmp_rowids[j];
      cand_distances[j] = tmp_distances[j];
    }
    cand_used = merged_used;
  }
  rc = SQLITE_OK;

  // Phase 2: Rescore candidates using _rescore_vectors (rowid-keyed)
  if (cand_used == 0) {
    knn_data->current_idx = 0;
    knn_data->k = 0;
    knn_data->rowids = NULL;
    knn_data->distances = NULL;
    knn_data->k_used = 0;
    goto cleanup;
  }
  {
    f32 *float_distances = sqlite3_malloc(cand_used * sizeof(f32));
    void *fBuf = sqlite3_malloc(fsize);
    if (!float_distances || !fBuf) {
      sqlite3_free(float_distances);
      sqlite3_free(fBuf);
      rc = SQLITE_NOMEM;
      goto cleanup;
    }

    // Open blob on _rescore_vectors, then reopen for each candidate rowid.
    // blob_reopen is O(1) for INTEGER PRIMARY KEY tables.
    sqlite3_blob *blobFloat = NULL;
    rc = sqlite3_blob_open(p->db, p->schemaName,
                           p->shadowRescoreVectorsNames[vectorColumnIdx],
                           "vector", cand_rowids[0], 0, &blobFloat);
    if (rc != SQLITE_OK) {
      sqlite3_free(float_distances);
      sqlite3_free(fBuf);
      goto cleanup;
    }

    rc = sqlite3_blob_read(blobFloat, fBuf, fsize, 0);
    if (rc != SQLITE_OK) {
      sqlite3_blob_close(blobFloat);
      sqlite3_free(float_distances);
      sqlite3_free(fBuf);
      goto cleanup;
    }
    float_distances[0] =
        vec0_distance_full(fBuf, queryVector, vector_column->dimensions,
                           vector_column->element_type,
                           vector_column->distance_metric);

    for (i64 j = 1; j < cand_used; j++) {
      rc = sqlite3_blob_reopen(blobFloat, cand_rowids[j]);
      if (rc != SQLITE_OK) {
        sqlite3_blob_close(blobFloat);
        sqlite3_free(float_distances);
        sqlite3_free(fBuf);
        goto cleanup;
      }
      rc = sqlite3_blob_read(blobFloat, fBuf, fsize, 0);
      if (rc != SQLITE_OK) {
        sqlite3_blob_close(blobFloat);
        sqlite3_free(float_distances);
        sqlite3_free(fBuf);
        goto cleanup;
      }
      float_distances[j] =
          vec0_distance_full(fBuf, queryVector, vector_column->dimensions,
                             vector_column->element_type,
                             vector_column->distance_metric);
    }
    sqlite3_blob_close(blobFloat);
    sqlite3_free(fBuf);

    // Sort by float distance
    for (i64 a = 0; a + 1 < cand_used; a++) {
      i64 minIdx = a;
      for (i64 c = a + 1; c < cand_used; c++) {
        if (float_distances[c] < float_distances[minIdx])
          minIdx = c;
      }
      if (minIdx != a) {
        f32 td = float_distances[a];
        float_distances[a] = float_distances[minIdx];
        float_distances[minIdx] = td;
        i64 tr = cand_rowids[a];
        cand_rowids[a] = cand_rowids[minIdx];
        cand_rowids[minIdx] = tr;
      }
    }

    i64 result_k = min(k, cand_used);
    i64 *out_rowids = sqlite3_malloc(result_k * sizeof(i64));
    f32 *out_distances = sqlite3_malloc(result_k * sizeof(f32));
    if (!out_rowids || !out_distances) {
      sqlite3_free(out_rowids);
      sqlite3_free(out_distances);
      sqlite3_free(float_distances);
      rc = SQLITE_NOMEM;
      goto cleanup;
    }
    for (i64 j = 0; j < result_k; j++) {
      out_rowids[j] = cand_rowids[j];
      out_distances[j] = float_distances[j];
    }

    knn_data->current_idx = 0;
    knn_data->k = result_k;
    knn_data->rowids = out_rowids;
    knn_data->distances = out_distances;
    knn_data->k_used = result_k;

    sqlite3_free(float_distances);
  }

cleanup:
  sqlite3_finalize(stmtChunks);
  sqlite3_free(quantizedQuery);
  sqlite3_free(cand_rowids);
  sqlite3_free(cand_distances);
  sqlite3_free(tmp_rowids);
  sqlite3_free(tmp_distances);
  sqlite3_free(chunk_distances);
  sqlite3_free(chunk_topk_idxs);
  sqlite3_free(b);
  sqlite3_free(bTaken);
  sqlite3_free(bmRowids);
  sqlite3_free(baseVectors);
  return rc;
}

/**
 * Handle FTS5-style command dispatch for rescore parameters.
 * Returns SQLITE_OK if handled, SQLITE_EMPTY if not a rescore command.
 */
static int rescore_handle_command(vec0_vtab *p, const char *command) {
  if (strncmp(command, "oversample=", 11) == 0) {
    int val = atoi(command + 11);
    if (val < 1) {
      vtab_set_error(&p->base, "oversample must be >= 1");
      return SQLITE_ERROR;
    }
    for (int i = 0; i < p->numVectorColumns; i++) {
      if (p->vector_columns[i].index_type == VEC0_INDEX_TYPE_RESCORE) {
        p->vector_columns[i].rescore.oversample_search = val;
      }
    }
    return SQLITE_OK;
  }
  return SQLITE_EMPTY;
}

#ifdef SQLITE_VEC_TEST
void _test_rescore_quantize_float_to_bit(const float *src, uint8_t *dst, size_t dim) {
  rescore_quantize_float_to_bit(src, dst, dim);
}
void _test_rescore_quantize_float_to_int8(const float *src, int8_t *dst, size_t dim) {
  rescore_quantize_float_to_int8(src, dst, dim);
}
size_t _test_rescore_quantized_byte_size_bit(size_t dimensions) {
  struct VectorColumnDefinition col;
  memset(&col, 0, sizeof(col));
  col.dimensions = dimensions;
  col.rescore.quantizer_type = VEC0_RESCORE_QUANTIZER_BIT;
  return rescore_quantized_byte_size(&col);
}
size_t _test_rescore_quantized_byte_size_int8(size_t dimensions) {
  struct VectorColumnDefinition col;
  memset(&col, 0, sizeof(col));
  col.dimensions = dimensions;
  col.rescore.quantizer_type = VEC0_RESCORE_QUANTIZER_INT8;
  return rescore_quantized_byte_size(&col);
}
#endif


from asyncio import Lock

from scrapling.spiders.request import Request
from scrapling.engines.static import _ASyncSessionLogic
from scrapling.engines.toolbelt.convertor import Response
from scrapling.core._types import Set, cast, SUPPORTED_HTTP_METHODS
from scrapling.fetchers import AsyncDynamicSession, AsyncStealthySession, FetcherSession

Session = FetcherSession | AsyncDynamicSession | AsyncStealthySession

class SessionManager:
    """Manages pre-configured session instances."""

    def __init__(self) -> None:
        self._sessions: dict[str, Session] = {}
        self._default_session_id: str | None = None
        self._started: bool = False
        self._lazy_sessions: Set[str] = set()
        self._lazy_lock = Lock()

    def add(self, session_id: str, session: Session, *, default: bool = False, lazy: bool = False) -> "SessionManager":
        """Register a session instance.

        :param session_id: Name to reference this session in requests
        :param session: Your pre-configured session instance
        :param default: If True, this becomes the default session
        :param lazy: If True, the session will be started only when a request uses its ID.
        """
        if session_id in self._sessions:
            raise ValueError(f"Session '{session_id}' already registered")

        self._sessions[session_id] = session

        if default or self._default_session_id is None:
            self._default_session_id = session_id

        if lazy:
            self._lazy_sessions.add(session_id)

        return self

    def remove(self, session_id: str) -> None:
        """Removes a session.

        :param session_id: ID of session to remove
        """
        _ = self.pop(session_id)

    def pop(self, session_id: str) -> Session:
        """Remove and returns a session.

        :param session_id: ID of session to remove
        """
        if session_id not in self._sessions:
            raise KeyError(f"Session '{session_id}' not found")

        session = self._sessions.pop(session_id)
        if session_id in self._lazy_sessions:
            self._lazy_sessions.remove(session_id)

        if session and self._default_session_id == session_id:
            self._default_session_id = next(iter(self._sessions), None)

        return session

    @property
    def default_session_id(self) -> str:
        if self._default_session_id is None:
            raise RuntimeError("No sessions registered")
        return self._default_session_id

    @property
    def session_ids(self) -> list[str]:
        return list(self._sessions.keys())

    def get(self, session_id: str) -> Session:
        if session_id not in self._sessions:
            available = ", ".join(self._sessions.keys())
            raise KeyError(f"Session '{session_id}' not found. Available: {available}")
        return self._sessions[session_id]

    async def start(self) -> None:
        """Start all sessions that aren't already alive."""
        if self._started:
            return

        for sid, session in self._sessions.items():
            if sid not in self._lazy_sessions and not session._is_alive:
                await session.__aenter__()

        self._started = True

    async def close(self) -> None:
        """Close all registered sessions."""
        for sid, session in self._sessions.items():
            if sid in self._lazy_sessions and not session._is_alive:
                continue
            _ = await session.__aexit__(None, None, None)

        self._started = False

    async def fetch(self, request: Request) -> Response:
        sid = request.sid if request.sid else self.default_session_id
        session = self.get(sid)

        if session:
            if sid in self._lazy_sessions and not session._is_alive:
                async with self._lazy_lock:
                    if not session._is_alive:
                        await session.__aenter__()

            if isinstance(session, FetcherSession):
                client = session._client

                if isinstance(client, _ASyncSessionLogic):
                    kwargs = request._session_kwargs.copy()
                    method = cast(SUPPORTED_HTTP_METHODS, kwargs.pop("method", "GET"))
                    response = await client._make_request(
                        method=method,
                        url=request.url,
                        **kwargs,
                    )
                else:
                    # Sync session or other types - shouldn't happen in async context
                    raise TypeError(f"Session type {type(client)} not supported for async fetch")
            else:
                response = await session.fetch(url=request.url, **request._session_kwargs)

            response.request = request
            # Merge request meta into response meta (response meta takes priority)
            response.meta = {**request.meta, **response.meta}
            return response
        raise RuntimeError("No session found with the request session id")

    async def __aenter__(self) -> "SessionManager":
        await self.start()
        return self

    async def __aexit__(self, *exc) -> None:
        await self.close()

    def __contains__(self, session_id: str) -> bool:
        """Check if a session ID is registered."""
        return session_id in self._sessions

    def __len__(self) -> int:
        """Number of registered sessions."""
        return len(self._sessions)


import { cli, Strategy } from '@jackwener/opencli/registry';
import { CliError } from '@jackwener/opencli/errors';
import { PAPERREVIEW_DOMAIN, ensureApiSuccess, ensureSuccess, normalizeVenue, readPdfFile, requestJson, summarizeSubmission, uploadPresignedPdf, } from './utils.js';
cli({
    site: 'paperreview',
    name: 'submit',
    access: 'write',
    description: 'Submit a PDF to paperreview.ai for review',
    domain: PAPERREVIEW_DOMAIN,
    strategy: Strategy.PUBLIC,
    browser: false,
    args: [
        { name: 'pdf', positional: true, required: true, help: 'Path to the paper PDF' },
        { name: 'email', required: true, help: 'Email address for the submission' },
        { name: 'venue', help: 'Optional target venue such as ICLR or NeurIPS' },
        { name: 'dry-run', type: 'bool', default: false, help: 'Validate the input and stop before remote submission' },
        { name: 'prepare-only', type: 'bool', default: false, help: 'Request an upload slot but stop before uploading the PDF' },
        { name: 'timeout', type: 'int', required: false, default: 120, help: 'Max seconds for the overall command (default: 120)' },
    ],
    columns: ['status', 'file', 'email', 'venue', 'token', 'review_url', 'message'],
    footerExtra: (kwargs) => {
        if (kwargs['dry-run'] === true)
            return 'dry run only';
        if (kwargs['prepare-only'] === true)
            return 'prepared only';
        return undefined;
    },
    func: async (kwargs) => {
        const pdfFile = await readPdfFile(kwargs.pdf);
        const email = String(kwargs.email ?? '').trim();
        const venue = normalizeVenue(kwargs.venue);
        const dryRun = kwargs['dry-run'] === true;
        const prepareOnly = kwargs['prepare-only'] === true;
        if (!email) {
            throw new CliError('ARGUMENT', 'An email address is required.', 'Pass --email <address>');
        }
        if (dryRun) {
            return summarizeSubmission({
                pdfFile,
                email,
                venue,
                message: 'Input validation passed. No remote request was sent.',
                dryRun: true,
            });
        }
        const { response: uploadUrlResponse, payload: uploadUrlPayload } = await requestJson('/api/get-upload-url', {
            method: 'POST',
            headers: { 'Content-Type': 'application/json' },
            body: JSON.stringify({
                filename: pdfFile.fileName,
                venue,
            }),
        });
        ensureSuccess(uploadUrlResponse, uploadUrlPayload, 'Failed to request an upload URL.', 'Try again in a moment');
        ensureApiSuccess(uploadUrlPayload, 'paperreview.ai did not return a usable upload URL.', 'Try again in a moment');
        if (prepareOnly) {
            return summarizeSubmission({
                pdfFile,
                email,
                venue,
                message: 'Upload slot prepared. The PDF was not uploaded and no submission was confirmed.',
                s3Key: uploadUrlPayload.s3_key,
                status: 'prepared',
            });
        }
        await uploadPresignedPdf(uploadUrlPayload.presigned_url, pdfFile, uploadUrlPayload);
        const confirmForm = new FormData();
        confirmForm.append('s3_key', uploadUrlPayload.s3_key);
        confirmForm.append('venue', venue);
        confirmForm.append('email', email);
        const { response: confirmResponse, payload: confirmPayload } = await requestJson('/api/confirm-upload', {
            method: 'POST',
            body: confirmForm,
        });
        ensureSuccess(confirmResponse, confirmPayload, 'Failed to confirm the upload with paperreview.ai.', 'Try again in a moment');
        ensureApiSuccess(confirmPayload, 'paperreview.ai did not confirm the submission.', 'Try again in a moment');
        return summarizeSubmission({
            pdfFile,
            email,
            venue,
            token: confirmPayload.token,
            message: confirmPayload.message,
            s3Key: uploadUrlPayload.s3_key,
        });
    },
});


use std::fmt::Write as _;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::sync::Arc;

use anyhow::{Context, Result};
use bstr::ByteSlice;
use clap::ValueEnum;
use owo_colors::OwoColorize;
use prek_consts::CONFIG_FILENAMES;
use same_file::is_same_file;

use crate::cli::reporter::{HookInitReporter, HookInstallReporter};
use crate::cli::run;
use crate::cli::run::InstallCache;
use crate::cli::run::{SelectorSource, Selectors};
use crate::cli::{ExitStatus, HookType};
use crate::config::load_config;
use crate::fs::{CWD, Simplified};
use crate::git::{GIT_ROOT, git_cmd};
use crate::printer::Printer;
use crate::store::Store;
use crate::workspace::{Error as WorkspaceError, HookInitFilters, Project, Workspace};
use crate::{git, warn_user};

#[allow(clippy::fn_params_excessive_bools)]
pub(crate) async fn install(
    store: &Store,
    config: Option<PathBuf>,
    includes: Vec<String>,
    skips: Vec<String>,
    hook_types: Vec<HookType>,
    prepare_hooks: bool,
    overwrite: bool,
    allow_missing_config: bool,
    refresh: bool,
    printer: Printer,
    git_dir: Option<&Path>,
) -> Result<ExitStatus> {
    // Upstream `pre-commit` deliberately refuses to install whenever
    // `git config core.hooksPath` resolves to a non-empty value. It does not
    // distinguish whether that effective value comes from local, worktree,
    // global, or system config. Once Git sees any effective `core.hooksPath`,
    // it stops consulting `.git/hooks` and runs hooks only from that configured
    // directory. If the value is external to this repository, "installing
    // anyway" would mean either writing into a user-managed shared hooks
    // directory or mutating the user's Git config to point back at this repo.
    // Both cross the repository's ownership boundary.
    //
    // In prek, we keep that safety boundary but narrow the refusal to the
    // actual unsafe case. Repo-owned `core.hooksPath` values are safe to honor
    // implicitly when they come from local/worktree scope, including repo
    // config reached through `include.path` / `includeIf`, because those values
    // are part of the repository's own Git setup. Only hooksPath values that
    // resolve entirely from external config still require the explicit
    // `--git-dir` escape hatch.
    if git_dir.is_none()
        && git::has_hooks_path_set().await?
        && !git::has_repo_hooks_path_set().await?
    {
        anyhow::bail!(
            concat!(
                "Refusing to install hooks because `core.hooksPath` is configured outside this repository.\n",
                "\n{} Git will execute hooks from the configured global/system hooks directory, not from this repository's hooks directory.\n",
                "\n{} Remove the global/system setting, or move `core.hooksPath` into repo scope for this repository instead.\n",
                "  {}\n",
                "  {}\n",
                "  {}\n",
            ),
            "note:".yellow().bold(),
            "hint:".yellow().bold(),
            "git config --unset-all --global core.hooksPath".cyan(),
            "git config --unset-all --system core.hooksPath".cyan(),
            "git config --local core.hooksPath <path>".cyan(),
        );
    }

    let hook_mode = git::get_shared_repository_file_mode(0o755)
        .await
        .unwrap_or(0o755);

    let project = match Project::discover(config.as_deref(), &CWD) {
        Ok(project) => Some(project),
        Err(err) => {
            if let WorkspaceError::Config(err) = &err {
                err.warn_parse_error();
            }
            None
        }
    };
    let hook_types = get_hook_types(hook_types, project.as_ref(), config.as_deref());

    let hooks_path = if let Some(dir) = git_dir {
        dir.join("hooks")
    } else {
        git::get_git_hooks_dir().await?
    };
    fs_err::create_dir_all(&hooks_path)?;

    let selectors = if let Some(project) = &project {
        Some(Selectors::load(&includes, &skips, project.path())?)
    } else if !includes.is_empty() || !skips.is_empty() {
        anyhow::bail!("Cannot use `--include` or `--skip` outside of a git repository");
    } else {
        None
    };

    for hook_type in hook_types {
        install_hook_script(
            project.as_ref(),
            config.clone(),
            selectors.as_ref(),
            hook_type,
            &hooks_path,
            overwrite,
            allow_missing_config,
            hook_mode,
            printer,
        )?;
    }

    if prepare_hooks {
        self::prepare_hooks(store, config, includes, skips, refresh, printer).await?;
    }

    Ok(ExitStatus::Success)
}

pub(crate) async fn prepare_hooks(
    store: &Store,
    config: Option<PathBuf>,
    includes: Vec<String>,
    skips: Vec<String>,
    refresh: bool,
    printer: Printer,
) -> Result<ExitStatus> {
    let workspace_root = Workspace::find_root(config.as_deref(), &CWD)?;
    let selectors = Selectors::load(&includes, &skips, &workspace_root)?;
    let workspace = Workspace::discover(store, workspace_root, config, Some(&selectors), refresh)?;

    let reporter = HookInitReporter::new(printer);
    let _lock = store.lock_async().await?;

    let hooks = workspace
        .init_hooks(
            store,
            HookInitFilters::new(Some(&selectors), None),
            Some(&reporter),
        )
        .await
        .context("Failed to init hooks")?;
    let filtered_hooks: Vec<_> = hooks
        .into_iter()
        .filter(|h| selectors.matches_hook(h))
        .map(Arc::new)
        .collect();

    let reporter = HookInstallReporter::new(printer);
    let mut install_cache = InstallCache::new();
    run::install_hooks(filtered_hooks, store, &reporter, &mut install_cache).await?;
    reporter.on_complete();

    Ok(ExitStatus::Success)
}

fn get_hook_types(
    mut hook_types: Vec<HookType>,
    project: Option<&Project>,
    config: Option<&Path>,
) -> Vec<HookType> {
    if !hook_types.is_empty() {
        return hook_types;
    }

    hook_types = if let Some(project) = project {
        project
            .config()
            .default_install_hook_types
            .clone()
            .unwrap_or_default()
    } else {
        let fallbacks = CONFIG_FILENAMES
            .iter()
            .map(Path::new)
            .filter(|p| p.exists());
        if let Some(path) = config.into_iter().chain(fallbacks).next() {
            match load_config(path) {
                Ok(cfg) => cfg.default_install_hook_types.unwrap_or_default(),
                Err(err) => {
                    err.warn_parse_error();
                    vec![]
                }
            }
        } else {
            vec![]
        }
    };
    if hook_types.is_empty() {
        hook_types = vec![HookType::PreCommit];
    }

    hook_types
}

#[allow(clippy::fn_params_excessive_bools)]
fn install_hook_script(
    project: Option<&Project>,
    config: Option<PathBuf>,
    selectors: Option<&Selectors>,
    hook_type: HookType,
    hooks_path: &Path,
    overwrite: bool,
    skip_on_missing_config: bool,
    hook_mode: u32,
    printer: Printer,
) -> Result<()> {
    let hook_path = hooks_path.join(hook_type.as_ref());
    let legacy_path = hook_path.with_added_extension("legacy");

    if hook_path.try_exists()? {
        if overwrite {
            writeln!(
                printer.stdout(),
                "Overwriting existing hook at `{}`",
                hook_path.user_display().cyan()
            )?;
        } else {
            if !is_our_script(&hook_path)? {
                fs_err::rename(&hook_path, &legacy_path)?;
                writeln!(
                    printer.stdout(),
                    "Hook already exists at `{}`, moved it to `{}`",
                    hook_path.user_display().cyan(),
                    legacy_path.user_display().yellow()
                )?;
            }
        }
    }

    if legacy_path.try_exists()? {
        if overwrite {
            // Remove existing legacy script too if we're overwriting.
            fs_err::remove_file(&legacy_path)?;
        } else {
            writeln!(
                printer.stdout(),
                "Migration mode: prek will also run legacy hook `{}`. Use `--overwrite` to remove legacy hooks.",
                legacy_path.user_display().yellow()
            )?;
        }
    }

    let mut args = vec![];

    // Add include/skip selectors.
    if let Some(selectors) = selectors {
        for include in selectors.includes() {
            args.push(include.as_normalized_flag());
        }

        // Find any skip selectors from environment variables.
        if let Some(env_var) = selectors.skips().iter().find_map(|skip| {
            if let SelectorSource::EnvVar(var) = skip.source() {
                Some(var)
            } else {
                None
            }
        }) {
            warn_user!(
                "Skip selectors from environment variables `{}` are ignored during installing hooks.",
                env_var.cyan()
            );
        }

        for skip in selectors.skips() {
            if matches!(skip.source(), SelectorSource::CliFlag(_)) {
                args.push(skip.as_normalized_flag());
            }
        }
    }

    args.push(format!("--hook-type={hook_type}"));

    let mut hint = format!("prek installed at `{}`", hook_path.user_display().cyan());

    // Prefer explicit config path if given (non-workspace mode).
    // Otherwise, use the config path from the discovered project (workspace mode).
    // If neither is available, don't pass a config path (let prek find it). In this case,
    // we're different with `pre-commit` which always sets `--config=.pre-commit-config.yaml`.
    if let Some(config) = config {
        args.push(format!(r#"--config="{}""#, config.display()));

        write!(hint, " with specified config `{}`", config.display().cyan())?;
    } else if let Some(project) = project {
        let git_root = GIT_ROOT.as_ref()?;
        let project_path = project.path();
        let relative_path = project_path.strip_prefix(git_root).unwrap_or(project_path);
        if !relative_path.as_os_str().is_empty() {
            args.push(format!(r#"--cd="{}""#, relative_path.display()));
        }

        // Show workspace path if it's not the root project.
        if project_path != git_root {
            writeln!(hint, " for workspace `{}`", project_path.display().cyan())?;
            write!(
                hint,
                "\n{} this hook installed for `{}` only; run `prek install` from `{}` to install for the entire repo.",
                "hint:".bold().yellow(),
                project_path.display().cyan(),
                git_root.display().cyan()
            )?;
        }
    }

    if skip_on_missing_config {
        args.push("--skip-on-missing-config".to_string());
    }

    let prek = std::env::current_exe()?;
    let prek = prek.simplified_display().to_string();
    let hook_script = HOOK_TMPL
        .replace("[CUR_SCRIPT_VERSION]", &CUR_SCRIPT_VERSION.to_string())
        .replace("[PREK_PATH]", &format!(r#""{prek}""#))
        .replace("[PREK_ARGS]", &args.join(" "));

    fs_err::OpenOptions::new()
        .write(true)
        .create(true)
        .truncate(true)
        .open(&hook_path)?
        .write_all(hook_script.as_bytes())?;

    #[cfg(unix)]
    {
        use std::os::unix::fs::PermissionsExt;

        let mut perms = hook_path.metadata()?.permissions();
        perms.set_mode(hook_mode);
        fs_err::set_permissions(&hook_path, perms)?;
    }

    // Unused on non-Unix platforms
    #[cfg(not(unix))]
    let _ = hook_mode;

    writeln!(printer.stdout(), "{hint}")?;

    Ok(())
}

/// The version of the hook script. Increment this when the script changes in a way that
/// requires re-installation.
pub(crate) static CUR_SCRIPT_VERSION: usize = 4;

static HOOK_TMPL: &str = r#"#!/bin/sh
# File generated by prek: https://github.com/j178/prek
# ID: 182c10f181da4464a3eec51b83331688

HERE="$(cd "$(dirname "$0")" && pwd)"
PREK=[PREK_PATH]

# Check if the full path to prek is executable, otherwise fallback to PATH
if [ ! -x "$PREK" ]; then
    PREK="prek"
fi

exec "$PREK" hook-impl --hook-dir "$HERE" --script-version [CUR_SCRIPT_VERSION] [PREK_ARGS] -- "$@"

"#;

static PRIOR_HASHES: &[&str] = &[];

// Use a different hash for each change to the script.
// Use a different hash from `pre-commit` since our script is different.
static CURRENT_HASH: &str = "182c10f181da4464a3eec51b83331688";

/// Checks if the script contains any of the hashes that `prek` has used in the past.
fn is_our_script(hook_path: &Path) -> std::io::Result<bool> {
    let content = fs_err::read_to_string(hook_path)?;
    Ok(std::iter::once(CURRENT_HASH)
        .chain(PRIOR_HASHES.iter().copied())
        .any(|hash| content.contains(hash)))
}

pub(crate) async fn uninstall(
    config: Option<PathBuf>,
    hook_types: Vec<HookType>,
    all: bool,
    printer: Printer,
    git_dir: Option<&Path>,
) -> Result<ExitStatus> {
    if git_dir.is_none()
        && git::has_hooks_path_set().await?
        && !git::has_repo_hooks_path_set().await?
    {
        anyhow::bail!(
            concat!(
                "Refusing to uninstall hooks because `core.hooksPath` is configured outside this repository.\n",
                "\n{} Git will execute hooks from the configured global/system hooks directory, not from this repository's hooks directory.\n",
                "\n{} Remove the global/system setting, or move `core.hooksPath` into repo scope for this repository instead.\n",
                "  {}\n",
                "  {}\n",
                "  {}\n",
            ),
            "note:".yellow().bold(),
            "hint:".yellow().bold(),
            "git config --unset-all --global core.hooksPath".cyan(),
            "git config --unset-all --system core.hooksPath".cyan(),
            "git config --local core.hooksPath <path>".cyan(),
        );
    }

    let project = Project::discover(config.as_deref(), &CWD).ok();
    let hooks_path = if let Some(dir) = git_dir {
        dir.join("hooks")
    } else {
        git::get_git_hooks_dir().await?
    };

    let types: Vec<HookType> = if all {
        HookType::value_variants().to_vec()
    } else {
        get_hook_types(hook_types, project.as_ref(), config.as_deref())
    };

    for hook_type in types {
        let hook_path = hooks_path.join(hook_type.as_ref());
        let legacy_path = hook_path.with_added_extension("legacy");

        if is_our_script(&legacy_path).unwrap_or(false) {
            fs_err::remove_file(&legacy_path)?;
            writeln!(
                printer.stderr(),
                "Found legacy hook at `{}`, removing it.",
                legacy_path.user_display().cyan()
            )?;
        }

        match is_our_script(&hook_path) {
            Ok(true) => {}
            Ok(false) => {
                if !all {
                    writeln!(
                        printer.stderr(),
                        "`{}` is not managed by prek, skipping.",
                        hook_path.user_display().cyan()
                    )?;
                }
                continue;
            }
            Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
                if !all {
                    writeln!(
                        printer.stderr(),
                        "`{}` does not exist, skipping.",
                        hook_path.user_display().cyan()
                    )?;
                }
                continue;
            }
            Err(err) => return Err(err.into()),
        }

        fs_err::remove_file(&hook_path)?;
        writeln!(
            printer.stdout(),
            "Uninstalled `{}`",
            hook_type.as_ref().cyan()
        )?;

        if legacy_path.try_exists()? {
            fs_err::rename(&legacy_path, &hook_path)?;
            writeln!(
                printer.stdout(),
                "Restored `{}` to `{}`",
                legacy_path.user_display().cyan(),
                hook_path.user_display().cyan()
            )?;
        }
    }

    Ok(ExitStatus::Success)
}

pub(crate) async fn init_template_dir(
    store: &Store,
    directory: PathBuf,
    config: Option<PathBuf>,
    hook_types: Vec<HookType>,
    requires_config: bool,
    refresh: bool,
    printer: Printer,
) -> Result<ExitStatus> {
    install(
        store,
        config,
        vec![],
        vec![],
        hook_types,
        false,
        true,
        !requires_config,
        refresh,
        printer,
        Some(&directory),
    )
    .await?;

    let output = git_cmd()?
        .arg("config")
        .arg("init.templateDir")
        .check(false)
        .output()
        .await?;
    let template_dir = String::from_utf8_lossy(output.stdout.trim()).to_string();

    if template_dir.is_empty() || !is_same_file(&directory, &template_dir)? {
        warn_user!(
            "`init.templateDir` does not point to the target directory. Run `{}` to set it",
            format!(
                "git config --global init.templateDir '{}'",
                directory.display()
            )
            .cyan()
        );
    }

    Ok(ExitStatus::Success)
}


#!/usr/bin/env node
// caveman — Claude Code SessionStart activation hook
//
// Runs on every session start:
//   1. Writes flag file at $CLAUDE_CONFIG_DIR/.caveman-active (statusline reads this)
//   2. Emits caveman ruleset as hidden SessionStart context
//   3. Detects missing statusline config and emits setup nudge

const fs = require('fs');
const path = require('path');
const os = require('os');
const { getDefaultMode, safeWriteFlag, recordModeChange } = require('./caveman-config');

const claudeDir = process.env.CLAUDE_CONFIG_DIR || path.join(os.homedir(), '.claude');
const flagPath = path.join(claudeDir, '.caveman-active');
const settingsPath = path.join(claudeDir, 'settings.json');

// Apply per-agent model overrides from env vars before emitting rules.
// Best-effort: any error is swallowed so SessionStart is never blocked.
try {
  const { applyOverrides, resolvePluginRoot } = require('./cavecrew-model-overrides');
  applyOverrides(resolvePluginRoot(__dirname));
} catch (e) {}

const mode = getDefaultMode();

// "off" mode — skip activation entirely, don't write flag or emit rules
if (mode === 'off') {
  recordModeChange(claudeDir, null); // #601: timestamped transition log
  try { fs.unlinkSync(flagPath); } catch (e) {}
  process.stdout.write('OK');
  process.exit(0);
}

// 1. Write flag file (symlink-safe)
recordModeChange(claudeDir, mode); // #601
safeWriteFlag(flagPath, mode);

// 2. Emit full caveman ruleset, filtered to the active intensity level.
//    The old 2-sentence summary was too weak — models drifted back to verbose
//    mid-conversation, especially after context compression pruned it away.
//    Full rules with examples anchor behavior much more reliably.
//
//    Reads SKILL.md at runtime so edits to the source of truth propagate
//    automatically — no hardcoded duplication to go stale.

// Modes that have their own independent skill files — not caveman intensity levels.
// For these, emit a short activation line; the skill itself handles behavior.
const INDEPENDENT_MODES = new Set(['commit', 'review', 'compress']);

if (INDEPENDENT_MODES.has(mode)) {
  process.stdout.write('CAVEMAN MODE ACTIVE — level: ' + mode + '. Behavior defined by /caveman-' + mode + ' skill.');
  process.exit(0);
}

// Resolve the canonical label for wenyan alias
const modeLabel = mode === 'wenyan' ? 'wenyan-full' : mode;

// Read SKILL.md — the single source of truth for caveman behavior.
// Candidate locations, tried in order (#587/#589 — the old single '..' path
// resolved to <plugin_root>/src/skills/, which doesn't exist, so plugin
// installs silently used the stale fallback ruleset):
//   1. $CLAUDE_PLUGIN_ROOT/skills/caveman/SKILL.md — Claude Code sets
//      CLAUDE_PLUGIN_ROOT when invoking plugin hooks; authoritative when present.
//   2. ../../skills/caveman/SKILL.md — hook at <plugin_root>/src/hooks/
//      (plugin.json layout) or a repo checkout.
//   3. ../skills/caveman/SKILL.md — standalone install with hooks at
//      $CLAUDE_CONFIG_DIR/hooks/ and the skill at $CLAUDE_CONFIG_DIR/skills/caveman/.
// All misses fall through to the hardcoded fallback ruleset below.
const skillCandidates = [];
if (process.env.CLAUDE_PLUGIN_ROOT) {
  skillCandidates.push(path.join(process.env.CLAUDE_PLUGIN_ROOT, 'skills', 'caveman', 'SKILL.md'));
}
skillCandidates.push(
  path.join(__dirname, '..', '..', 'skills', 'caveman', 'SKILL.md'),
  path.join(__dirname, '..', 'skills', 'caveman', 'SKILL.md')
);

let skillContent = '';
for (const candidate of skillCandidates) {
  try {
    skillContent = fs.readFileSync(candidate, 'utf8');
    break;
  } catch (e) { /* try next candidate */ }
}

let output;

if (skillContent) {
  // Strip YAML frontmatter
  const body = skillContent.replace(/^---[\s\S]*?---\s*/, '');

  // Filter intensity table: keep header rows + only the active level's row
  const filtered = body.split('\n').reduce((acc, line) => {
    // Intensity table rows start with | **level** |
    const tableRowMatch = line.match(/^\|\s*\*\*(\S+?)\*\*\s*\|/);
    if (tableRowMatch) {
      // Keep only the active level's row (and always keep header/separator)
      if (tableRowMatch[1] === modeLabel) {
        acc.push(line);
      }
      return acc;
    }

    // Example lines start with "- level:" — keep only lines matching active level
    const exampleMatch = line.match(/^- (\S+?):\s/);
    if (exampleMatch) {
      if (exampleMatch[1] === modeLabel) {
        acc.push(line);
      }
      return acc;
    }

    acc.push(line);
    return acc;
  }, []);

  output = 'CAVEMAN MODE ACTIVE — level: ' + modeLabel + '\n\n' + filtered.join('\n');
} else {
  // Fallback when SKILL.md is not found (standalone hook install without skills dir).
  // This is the minimum viable ruleset — better than nothing.
  output =
    'CAVEMAN MODE ACTIVE — level: ' + modeLabel + '\n\n' +
    'Respond terse like smart caveman. All technical substance stay. Only fluff die.\n\n' +
    '## Persistence\n\n' +
    'ACTIVE EVERY RESPONSE. No revert after many turns. No filler drift. Still active if unsure. Off only: "stop caveman" / "normal mode".\n\n' +
    'Current level: **' + modeLabel + '**. Switch: `/caveman lite|full|ultra`.\n\n' +
    '## Rules\n\n' +
    'Drop: articles (a/an/the), filler (just/really/basically/actually/simply), pleasantries (sure/certainly/of course/happy to), hedging. ' +
    'Fragments OK. Short synonyms (big not extensive, fix not "implement a solution for"). Technical terms exact. Code blocks unchanged. Errors quoted exact.\n\n' +
    "Preserve user's dominant language. User write Portuguese → reply Portuguese caveman. Compress the style, not the language. Technical terms, code, API names, commands, error strings stay verbatim.\n\n" +
    'No self-reference. Never name or announce the style. No "caveman mode on" tags. Output caveman-only.\n\n' +
    'Pattern: `[thing] [action] [reason]. [next step].`\n\n' +
    'Not: "Sure! I\'d be happy to help you with that. The issue you\'re experiencing is likely caused by..."\n' +
    'Yes: "Bug in auth middleware. Token expiry check use `<` not `<=`. Fix:"\n\n' +
    '## Auto-Clarity\n\n' +
    'Drop caveman for: security warnings, irreversible action confirmations, multi-step sequences where fragment order risks misread, user asks to clarify or repeats question. Resume caveman after clear part done.\n\n' +
    '## Boundaries\n\n' +
    'Code/commits/PRs: write normal. "stop caveman" or "normal mode": revert. Level persist until changed or session end.';
}

// 3. Detect missing statusline config — nudge Claude to help set it up
try {
  let hasStatusline = false;
  if (fs.existsSync(settingsPath)) {
    const settings = JSON.parse(fs.readFileSync(settingsPath, 'utf8'));
    if (settings.statusLine) {
      hasStatusline = true;
    }
  }

  if (!hasStatusline) {
    const isWindows = process.platform === 'win32';
    const scriptName = isWindows ? 'caveman-statusline.ps1' : 'caveman-statusline.sh';
    const scriptPath = path.join(__dirname, scriptName);
    const command = isWindows
      ? `powershell -ExecutionPolicy Bypass -File "${scriptPath}"`
      : `bash "${scriptPath}"`;
    const statusLineSnippet =
      '"statusLine": { "type": "command", "command": ' + JSON.stringify(command) + ' }';
    output += "\n\n" +
      "STATUSLINE SETUP NEEDED: The caveman plugin includes a statusline badge showing active mode " +
      "(e.g. [CAVEMAN], [CAVEMAN:ULTRA]). It is not configured yet. " +
      "To enable, add this to " + path.join(claudeDir, 'settings.json') + ": " +
      statusLineSnippet + " " +
      "Proactively offer to set this up for the user on first interaction.";
  }
} catch (e) {
  // Silent fail — don't block session start over statusline detection
}

process.stdout.write(output);


import hashlib
from io import BytesIO
from functools import cached_property
from urllib.parse import urlparse, urlencode

import orjson
from w3lib.url import canonicalize_url

from scrapling.engines.toolbelt.custom import Response
from scrapling.core._types import Any, AsyncGenerator, Callable, Dict, Optional, Union, Tuple, TYPE_CHECKING

if TYPE_CHECKING:
    from scrapling.spiders.spider import Spider

def _convert_to_bytes(value: str | bytes) -> bytes:
    if isinstance(value, bytes):
        return value
    if not isinstance(value, str):
        raise TypeError(f"Can't convert {type(value).__name__} to bytes")

    return value.encode(encoding="utf-8", errors="ignore")

def _stable_value_repr(value: Any) -> str:
    try:
        return orjson.dumps(value, option=orjson.OPT_SORT_KEYS, default=repr).decode()
    except TypeError:
        return repr(value)

class Request:
    def __init__(
        self,
        url: str,
        sid: str = "",
        callback: Callable[[Response], AsyncGenerator[Union[Dict[str, Any], "Request", None], None]] | None = None,
        priority: int = 0,
        dont_filter: bool = False,
        meta: dict[str, Any] | None = None,
        _retry_count: int = 0,
        **kwargs: Any,
    ) -> None:
        self.url: str = url
        self.sid: str = sid
        self.callback = callback
        self.priority: int = priority
        self.dont_filter: bool = dont_filter
        self.meta: dict[str, Any] = meta if meta else {}
        self._retry_count: int = _retry_count
        self._session_kwargs = kwargs if kwargs else {}
        self._fp: Optional[bytes] = None

    def copy(self) -> "Request":
        """Create a copy of this request."""
        return Request(
            url=self.url,
            sid=self.sid,
            callback=self.callback,
            priority=self.priority,
            dont_filter=self.dont_filter,
            meta=self.meta.copy(),
            _retry_count=self._retry_count,
            **self._session_kwargs,
        )

    @cached_property
    def domain(self) -> str:
        return urlparse(self.url).netloc

    def update_fingerprint(
        self,
        include_kwargs: bool = False,
        include_headers: bool = False,
        keep_fragments: bool = False,
    ) -> bytes:
        """Generate a unique fingerprint for deduplication.

        Caches the result in self._fp after first computation.
        """
        if self._fp is not None:
            return self._fp

        post_data = self._session_kwargs.get("data", {})
        body = b""
        if post_data:
            if isinstance(post_data, dict | list | tuple):
                body = urlencode(post_data).encode()
            elif isinstance(post_data, str):
                body = post_data.encode()
            elif isinstance(post_data, BytesIO):
                body = post_data.getvalue()
            elif isinstance(post_data, bytes):
                body = post_data
        else:
            post_data = self._session_kwargs.get("json", {})
            body = orjson.dumps(post_data) if post_data else b""

        data: Dict[str, str | Tuple] = {
            "sid": self.sid,
            "body": body.hex(),
            "method": self._session_kwargs.get("method", "GET"),
            "url": canonicalize_url(self.url, keep_fragments=keep_fragments),
        }

        if include_kwargs:
            filtered_kwargs = {
                key.lower(): _stable_value_repr(value)
                for key, value in self._session_kwargs.items()
                if key.lower() not in ("data", "json")
            }
            data["kwargs"] = tuple(sorted(filtered_kwargs.items()))

        if include_headers:
            headers = self._session_kwargs.get("headers") or self._session_kwargs.get("extra_headers") or {}
            processed_headers = {}
            # Some header normalization
            for key, value in headers.items():
                processed_headers[_convert_to_bytes(key.lower()).hex()] = _convert_to_bytes(value).hex()
            data["headers"] = tuple(processed_headers.items())

        fp = hashlib.sha1(orjson.dumps(data, option=orjson.OPT_SORT_KEYS), usedforsecurity=False).digest()
        self._fp = fp
        return fp

    def __repr__(self) -> str:
        callback_name = getattr(self.callback, "__name__", None) or "None"
        return f"<Request({self.url}) priority={self.priority} callback={callback_name}>"

    def __str__(self) -> str:
        return self.url

    def __lt__(self, other: object) -> bool:
        """Compare requests by priority"""
        if not isinstance(other, Request):
            return NotImplemented
        return self.priority < other.priority

    def __gt__(self, other: object) -> bool:
        """Compare requests by priority"""
        if not isinstance(other, Request):
            return NotImplemented
        return self.priority > other.priority

    def __eq__(self, other: object) -> bool:
        """Requests are equal if they have the same fingerprint."""
        if not isinstance(other, Request):
            return NotImplemented
        if self._fp is None or other._fp is None:
            raise RuntimeError("Cannot compare requests before generating their fingerprints!")
        return self._fp == other._fp

    def __getstate__(self) -> dict[str, Any]:
        """Prepare state for pickling - store callback as name string for pickle compatibility."""
        state = self.__dict__.copy()
        state["_callback_name"] = getattr(self.callback, "__name__", None) if self.callback is not None else None
        state["callback"] = None  # Don't pickle the actual callable
        return state

    def __setstate__(self, state: dict[str, Any]) -> None:
        """Restore state from pickle - callback restored later via _restore_callback()."""
        self._callback_name: str | None = state.pop("_callback_name", None)
        self.__dict__.update(state)

    def _restore_callback(self, spider: "Spider") -> None:
        """Restore callback from spider after unpickling.

        :param spider: Spider instance to look up callback method on
        """
        if hasattr(self, "_callback_name") and self._callback_name:
            self.callback = getattr(spider, self._callback_name, None) or spider.parse
            del self._callback_name
        elif hasattr(self, "_callback_name"):
            del self._callback_name


import { cva, type VariantProps } from "class-variance-authority"

import { cn } from "#/lib/utils.ts"

function Empty({ className, ...props }: React.ComponentProps<"div">) {
  return (
    <div
      data-slot="empty"
      className={cn(
        "flex w-full min-w-0 flex-1 flex-col items-center justify-center gap-4 rounded-xl border-dashed p-6 text-center text-balance",
        className
      )}
      {...props}
    />
  )
}

function EmptyHeader({ className, ...props }: React.ComponentProps<"div">) {
  return (
    <div
      data-slot="empty-header"
      className={cn("flex max-w-sm flex-col items-center gap-1", className)}
      {...props}
    />
  )
}

const emptyMediaVariants = cva(
  "mb-2 flex shrink-0 items-center justify-center [&_svg]:pointer-events-none [&_svg]:shrink-0",
  {
    variants: {
      variant: {
        default: "bg-transparent",
        icon: "flex size-8 shrink-0 items-center justify-center rounded-md bg-muted text-foreground [&_svg:not([class*='size-'])]:size-4",
      },
    },
    defaultVariants: {
      variant: "default",
    },
  }
)

function EmptyMedia({
  className,
  variant = "default",
  ...props
}: React.ComponentProps<"div"> & VariantProps<typeof emptyMediaVariants>) {
  return (
    <div
      data-slot="empty-icon"
      data-variant={variant}
      className={cn(emptyMediaVariants({ variant, className }))}
      {...props}
    />
  )
}

function EmptyTitle({ className, ...props }: React.ComponentProps<"div">) {
  return (
    <div
      data-slot="empty-title"
      className={cn(
        "font-heading text-sm font-medium tracking-tight",
        className
      )}
      {...props}
    />
  )
}

function EmptyDescription({ className, ...props }: React.ComponentProps<"p">) {
  return (
    <div
      data-slot="empty-description"
      className={cn(
        "text-xs/relaxed text-muted-foreground [&>a]:underline [&>a]:underline-offset-4 [&>a:hover]:text-primary",
        className
      )}
      {...props}
    />
  )
}

function EmptyContent({ className, ...props }: React.ComponentProps<"div">) {
  return (
    <div
      data-slot="empty-content"
      className={cn(
        "flex w-full max-w-sm min-w-0 flex-col items-center gap-2 text-xs/relaxed text-balance",
        className
      )}
      {...props}
    />
  )
}

export {
  Empty,
  EmptyHeader,
  EmptyTitle,
  EmptyDescription,
  EmptyContent,
  EmptyMedia,
}


# -*- coding: utf-8 -*-
from sys import stderr
from copy import deepcopy
from functools import wraps
from re import sub as re_sub, compile as re_compile
from collections import namedtuple
from shlex import split as shlex_split
from inspect import signature, Parameter
from tempfile import mkstemp as make_temp_file
from argparse import ArgumentParser, SUPPRESS
from webbrowser import open as open_in_browser
from urllib.parse import urlparse, urlunparse, parse_qsl
from logging import (
    DEBUG,
    INFO,
    WARNING,
    ERROR,
    CRITICAL,
    FATAL,
    getLogger,
    getLevelName,
)

from lxml.etree import XPath
from orjson import loads as json_loads, JSONDecodeError

from ._shell_signatures import Signatures_map
from scrapling import __version__
from scrapling.core.utils import log
from scrapling.parser import Selector, Selectors
from scrapling.core.custom_types import TextHandler
from scrapling.engines.toolbelt.custom import Response
from scrapling.core.utils._shell import _ParseHeaders, _CookieParser
from scrapling.core._types import (
    Callable,
    Dict,
    Any,
    cast,
    Optional,
    Generator,
    extraction_types,
)

_known_logging_levels = {
    "debug": DEBUG,
    "info": INFO,
    "warning": WARNING,
    "error": ERROR,
    "critical": CRITICAL,
    "fatal": FATAL,
}

# Define the structure for parsed context - Simplified for Fetcher args
Request = namedtuple(
    "Request",
    [
        "method",
        "url",
        "params",
        "data",  # Can be str, bytes, or dict (for urlencoded)
        "json_data",  # Python object (dict/list) for JSON payload
        "headers",
        "cookies",
        "proxy",
        "follow_redirects",  # Added for -L flag
    ],
)

# Precompiled for the prompt injection sanitizer
_HIDDEN_XPATH = XPath(
    './/*[contains(@style,"display:none") or contains(@style,"display: none")'
    ' or contains(@style,"visibility:hidden") or contains(@style,"visibility: hidden")'
    ' or contains(@style,"opacity:0") or contains(@style,"opacity: 0")'
    ' or contains(@style,"font-size:0") or contains(@style,"font-size: 0")'
    ' or contains(@style,"height:0") or contains(@style,"height: 0")'
    ' or contains(@style,"width:0") or contains(@style,"width: 0")]'
    " | .//*[@aria-hidden='true']"
    " | .//template"
)
_ZWC_PATTERN = re_compile(r"[\u200b\u200c\u200d\ufeff\u2060\u180e]")
_CONTROL_CHARS_PATTERN = re_compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f]")

# Suppress exit on error to handle parsing errors gracefully
class NoExitArgumentParser(ArgumentParser):  # pragma: no cover
    def error(self, message):
        log.error(f"Curl arguments parsing error: {message}")
        raise ValueError(f"Curl arguments parsing error: {message}")

    def exit(self, status=0, message=None):
        if message:
            log.error(f"Scrapling shell exited with status {status}: {message}")
            self._print_message(message, stderr)
        raise ValueError(f"Scrapling shell exited with status {status}: {message or 'Unknown reason'}")

class CurlParser:
    """Builds the argument parser for relevant curl flags from DevTools."""

    def __init__(self) -> None:
        from scrapling.fetchers import Fetcher as __Fetcher

        self.__fetcher = __Fetcher
        # We will use argparse parser to parse the curl command directly instead of regex
        # We will focus more on flags that will show up on curl commands copied from DevTools's network tab
        _parser = NoExitArgumentParser(add_help=False)  # Disable default help
        # Basic curl arguments
        _parser.add_argument("curl_command_placeholder", nargs="?", help=SUPPRESS)
        _parser.add_argument("url")
        _parser.add_argument("-X", "--request", dest="method", default=None)
        _parser.add_argument("-H", "--header", action="append", default=[])
        _parser.add_argument(
            "-A", "--user-agent", help="Will be parsed from -H if present"
        )  # Note: DevTools usually includes this in -H

        # Data arguments (prioritizing types common from DevTools)
        _parser.add_argument("-d", "--data", default=None)
        _parser.add_argument("--data-raw", default=None)  # Often used by browsers for JSON body
        _parser.add_argument("--data-binary", default=None)
        # Keep urlencode for completeness, though less common from browser copy/paste
        _parser.add_argument("--data-urlencode", action="append", default=[])
        _parser.add_argument("-G", "--get", action="store_true")  # Use GET and put data in URL

        _parser.add_argument(
            "-b",
            "--cookie",
            default=None,
            help="Send cookies from string/file (string format used by DevTools)",
        )

        # Proxy
        _parser.add_argument("-x", "--proxy", default=None)
        _parser.add_argument("-U", "--proxy-user", default=None)  # Basic proxy auth

        # Connection/Security
        _parser.add_argument("-k", "--insecure", action="store_true")
        _parser.add_argument("--compressed", action="store_true")  # Very common from browsers

        # Other flags often included but may not map directly to request args
        _parser.add_argument("-i", "--include", action="store_true")
        _parser.add_argument("-s", "--silent", action="store_true")
        _parser.add_argument("-v", "--verbose", action="store_true")

        self.parser: NoExitArgumentParser = _parser
        self._supported_methods = ("get", "post", "put", "delete")

    # --- Main Parsing Logic ---
    def parse(self, curl_command: str) -> Optional[Request]:
        """Parses the curl command string into a structured context for Fetcher."""

        clean_command = curl_command.strip().lstrip("curl").strip().replace("\\\n", " ")

        try:
            tokens = shlex_split(clean_command)  # Split the string using shell-like syntax
        except ValueError as e:  # pragma: no cover
            log.error(f"Could not split command line: {e}")
            return None

        try:
            parsed_args, unknown = self.parser.parse_known_args(tokens)
            if unknown:
                raise AttributeError(f"Unknown/Unsupported curl arguments: {unknown}")

        except ValueError:  # pragma: no cover
            return None

        except AttributeError:
            raise

        except Exception as e:  # pragma: no cover
            log.error(f"An unexpected error occurred during curl arguments parsing: {e}")
            return None

        # --- Determine Method ---
        method = "get"  # Default
        if parsed_args.get:  # `-G` forces GET
            method = "get"

        elif parsed_args.method:
            method = parsed_args.method.strip().lower()

        # Infer POST if data is present (unless overridden by -X or -G)
        elif any(
            [
                parsed_args.data,
                parsed_args.data_raw,
                parsed_args.data_binary,
                parsed_args.data_urlencode,
            ]
        ):
            method = "post"

        headers, cookies = _ParseHeaders(parsed_args.header)

        if parsed_args.cookie:
            # We are focusing on the string format from DevTools.
            try:
                for key, value in _CookieParser(parsed_args.cookie):
                    # Update the cookie dict, potentially overwriting cookies with the same name from -H 'cookie:'
                    cookies[key] = value
                log.debug(f"Parsed cookies from -b argument: {list(cookies.keys())}")
            except Exception as e:  # pragma: no cover
                log.error(f"Could not parse cookie string from -b '{parsed_args.cookie}': {e}")

        # --- Process Data Payload ---
        params = dict()
        data_payload: Optional[str | bytes | Dict] = None
        json_payload: Optional[Any] = None

        # DevTools often uses --data-raw for JSON bodies
        # Precedence: --data-binary > --data-raw / -d > --data-urlencode
        if parsed_args.data_binary is not None:  # pragma: no cover
            try:
                data_payload = parsed_args.data_binary.encode("utf-8")
                log.debug("Using data from --data-binary as bytes.")
            except Exception as e:
                log.warning(
                    f"Could not encode binary data '{parsed_args.data_binary}' as bytes: {e}. Using raw string."
                )
                data_payload = parsed_args.data_binary  # Fallback to string

        elif parsed_args.data_raw is not None:
            data_payload = parsed_args.data_raw.lstrip("$")

        elif parsed_args.data is not None:
            data_payload = parsed_args.data

        elif parsed_args.data_urlencode:  # pragma: no cover
            # Combine and parse urlencoded data
            combined_data = "&".join(parsed_args.data_urlencode)
            try:
                data_payload = dict(parse_qsl(combined_data, keep_blank_values=True))
            except Exception as e:
                log.warning(f"Could not parse urlencoded data '{combined_data}': {e}. Treating as raw string.")
                data_payload = combined_data

        # Check if raw data looks like JSON, prefer 'json' param if so
        if isinstance(data_payload, str):
            try:
                maybe_json = json_loads(data_payload)
                if isinstance(maybe_json, (dict, list)):
                    json_payload = maybe_json
                    data_payload = None
            except JSONDecodeError:
                pass  # Not JSON, keep it in data_payload

        # Handle `-G`: Move data to params if the method is GET
        if method == "get" and data_payload:  # pragma: no cover
            if isinstance(data_payload, dict):  # From --data-urlencode likely
                params.update(data_payload)
            elif isinstance(data_payload, str):
                try:
                    params.update(dict(parse_qsl(data_payload, keep_blank_values=True)))
                except ValueError:
                    log.warning(f"Could not parse data '{data_payload}' into GET parameters for -G.")

            if params:
                data_payload = None  # Clear data as it's moved to params
                json_payload = None  # Should not have JSON body with -G

        # --- Process Proxy ---
        proxies: Optional[Dict[str, str]] = None
        if parsed_args.proxy:
            proxy_url = f"http://{parsed_args.proxy}" if "://" not in parsed_args.proxy else parsed_args.proxy

            if parsed_args.proxy_user:
                user_pass = parsed_args.proxy_user
                parts = urlparse(proxy_url)
                netloc_parts = parts.netloc.split("@")
                netloc = f"{user_pass}@{netloc_parts[-1]}" if len(netloc_parts) > 1 else f"{user_pass}@{parts.netloc}"
                proxy_url = urlunparse(
                    (
                        parts.scheme,
                        netloc,
                        parts.path,
                        parts.params,
                        parts.query,
                        parts.fragment,
                    )
                )

            # Standard proxy dict format
            proxies = {"http": proxy_url, "https": proxy_url}
            log.debug(f"Using proxy configuration: {proxies}")

        # --- Final Context ---
        return Request(
            method=method,
            url=parsed_args.url,
            params=params,
            data=data_payload,
            json_data=json_payload,
            headers=headers,
            cookies=cookies,
            proxy=proxies,
            follow_redirects="safe",  # Follows redirects but rejects those to internal/private IPs
        )

    def convert2fetcher(self, curl_command: Request | str) -> Optional[Response]:
        if isinstance(curl_command, (Request, str)):
            request = self.parse(curl_command) if isinstance(curl_command, str) else curl_command

            # Ensure request parsing was successful before proceeding
            if request is None:  # pragma: no cover
                log.error("Failed to parse curl command, cannot convert to fetcher.")
                return None

            request_args = request._asdict()
            method = request_args.pop("method").strip().lower()
            if method in self._supported_methods:
                request_args["json"] = request_args.pop("json_data")

                # Ensure data/json are removed for non-POST/PUT methods
                if method not in ("post", "put"):
                    _ = request_args.pop("data", None)
                    _ = request_args.pop("json", None)

                try:
                    return getattr(self.__fetcher, method)(**request_args)
                except Exception as e:  # pragma: no cover
                    log.error(f"Error calling Fetcher.{method}: {e}")
                    return None
            else:  # pragma: no cover
                log.error(f'Request method "{method}" isn\'t supported by Scrapling yet')
                return None

        else:  # pragma: no cover
            log.error("Input must be a valid curl command string or a Request object.")
            return None

def _unpack_signature(func, signature_name=None):
    """
    Unpack TypedDict from Unpack[TypedDict] annotations in **kwargs and reconstruct the signature.

    This allows the interactive shell to show individual parameters instead of just **kwargs, similar to how IDEs display them.
    """
    try:
        sig = signature(func)
        func_name = signature_name or getattr(func, "__name__", None)

        # Check if this function has known parameters
        if func_name not in Signatures_map:
            return sig

        new_params = []
        for param in sig.parameters.values():
            if param.kind == Parameter.VAR_KEYWORD:
                # Replace **kwargs with individual keyword-only parameters
                for field_name, field_type in Signatures_map[func_name].items():
                    new_params.append(
                        Parameter(field_name, Parameter.KEYWORD_ONLY, default=Parameter.empty, annotation=field_type)
                    )
            else:
                new_params.append(param)

        # Reconstruct signature with unpacked parameters
        if len(new_params) != len(sig.parameters):
            return sig.replace(parameters=new_params)
        return sig

    except Exception:  # pragma: no cover
        return signature(func)

def show_page_in_browser(page: Selector):  # pragma: no cover
    if not page or not isinstance(page, Selector):
        log.error("Input must be of type `Selector`")
        return

    try:
        fd, fname = make_temp_file(prefix="scrapling_view_", suffix=".html")
        with open(fd, "w", encoding=page.encoding) as f:
            f.write(page.html_content)

        open_in_browser(f"file://{fname}")
    except IOError as e:
        log.error(f"Failed to write temporary file for viewing: {e}")
    except Exception as e:
        log.error(f"An unexpected error occurred while viewing the page: {e}")

class CustomShell:
    """A custom IPython shell with minimal dependencies"""

    def __init__(self, code, log_level="debug"):
        from IPython.terminal.embed import InteractiveShellEmbed as __InteractiveShellEmbed
        from scrapling.fetchers import (
            Fetcher as __Fetcher,
            AsyncFetcher as __AsyncFetcher,
            FetcherSession as __FetcherSession,
            DynamicFetcher as __DynamicFetcher,
            DynamicSession as __DynamicSession,
            AsyncDynamicSession as __AsyncDynamicSession,
            StealthyFetcher as __StealthyFetcher,
            StealthySession as __StealthySession,
            AsyncStealthySession as __AsyncStealthySession,
        )

        self.__InteractiveShellEmbed = __InteractiveShellEmbed
        self.__Fetcher = __Fetcher
        self.__AsyncFetcher = __AsyncFetcher
        self.__FetcherSession = __FetcherSession
        self.__DynamicFetcher = __DynamicFetcher
        self.__DynamicSession = __DynamicSession
        self.__AsyncDynamicSession = __AsyncDynamicSession
        self.__StealthyFetcher = __StealthyFetcher
        self.__StealthySession = __StealthySession
        self.__AsyncStealthySession = __AsyncStealthySession
        self.code = code
        self.page = None
        self.pages = Selectors([])
        self._curl_parser = CurlParser()
        log_level = log_level.strip().lower()

        if _known_logging_levels.get(log_level):
            self.log_level = _known_logging_levels[log_level]
        else:  # pragma: no cover
            log.warning(f'Unknown log level "{log_level}", defaulting to "DEBUG"')
            self.log_level = DEBUG

        self.shell = None

        # Initialize your application components
        self.init_components()

    def init_components(self):
        """Initialize application components"""
        # This is where you'd set up your application-specific objects
        if self.log_level:
            getLogger("scrapling").setLevel(self.log_level)

        settings = self.__Fetcher.display_config()
        settings.pop("storage", None)
        settings.pop("storage_args", None)
        log.info(f"Scrapling {__version__} shell started")
        log.info(f"Logging level is set to '{getLevelName(self.log_level)}'")
        log.info(f"Fetchers' parsing settings: {settings}")

    @staticmethod
    def banner():
        """Create a custom banner for the shell"""
        return f"""
-> Available Scrapling objects:
   - Fetcher/AsyncFetcher/FetcherSession
   - DynamicFetcher/DynamicSession/AsyncDynamicSession
   - StealthyFetcher/StealthySession/AsyncStealthySession
   - Selector

-> Useful shortcuts:
   - {"get":<30} Shortcut for `Fetcher.get`
   - {"post":<30} Shortcut for `Fetcher.post`
   - {"put":<30} Shortcut for `Fetcher.put`
   - {"delete":<30} Shortcut for `Fetcher.delete`
   - {"fetch":<30} Shortcut for `DynamicFetcher.fetch`
   - {"stealthy_fetch":<30} Shortcut for `StealthyFetcher.fetch`

-> Useful commands
   - {"page / response":<30} The response object of the last page you fetched
   - {"pages":<30} Selectors object of the last 5 response objects you fetched
   - {"uncurl('curl_command')":<30} Convert curl command to a Request object. (Optimized to handle curl commands copied from DevTools network tab.)
   - {"curl2fetcher('curl_command')":<30} Convert curl command and make the request with Fetcher. (Optimized to handle curl commands copied from DevTools network tab.)
   - {"view(page)":<30} View page in a browser
   - {"help()":<30} Show this help message (Shell help)

Type 'exit' or press Ctrl+D to exit.
        """

    def update_page(self, result):  # pragma: no cover
        """Update the current page and add to pages history"""
        self.page = result
        if isinstance(result, (Response, Selector)):
            self.pages.append(result)
            if len(self.pages) > 5:
                self.pages.pop(0)  # Remove the oldest item

            # Update in IPython namespace too
            if self.shell:
                self.shell.user_ns["page"] = self.page
                self.shell.user_ns["response"] = self.page
                self.shell.user_ns["pages"] = self.pages

        return result

    def create_wrapper(
        self, func: Callable, get_signature: bool = True, signature_name: Optional[str] = None
    ) -> Callable:
        """Create a wrapper that preserves function signature but updates page"""

        @wraps(func)
        def wrapper(*args: Any, **kwargs: Any) -> Any:
            result = func(*args, **kwargs)
            return self.update_page(result)

        if get_signature:
            # Explicitly preserve and unpack signature for IPython introspection and autocompletion
            setattr(wrapper, "__signature__", _unpack_signature(func, signature_name))
        else:
            setattr(wrapper, "__signature__", signature(func))

        return wrapper

    def get_namespace(self):
        """Create a namespace with application-specific objects"""

        # Create wrapped versions of fetch functions
        get = self.create_wrapper(self.__Fetcher.get)
        post = self.create_wrapper(self.__Fetcher.post)
        put = self.create_wrapper(self.__Fetcher.put)
        delete = self.create_wrapper(self.__Fetcher.delete)
        dynamic_fetch = self.create_wrapper(self.__DynamicFetcher.fetch)
        stealthy_fetch = self.create_wrapper(self.__StealthyFetcher.fetch, signature_name="stealthy_fetch")
        curl2fetcher = self.create_wrapper(self._curl_parser.convert2fetcher, get_signature=False)

        # Create the namespace dictionary
        return {
            "get": get,
            "post": post,
            "put": put,
            "delete": delete,
            "Fetcher": self.__Fetcher,
            "AsyncFetcher": self.__AsyncFetcher,
            "FetcherSession": self.__FetcherSession,
            "DynamicSession": self.__DynamicSession,
            "AsyncDynamicSession": self.__AsyncDynamicSession,
            "StealthySession": self.__StealthySession,
            "AsyncStealthySession": self.__AsyncStealthySession,
            "fetch": dynamic_fetch,
            "DynamicFetcher": self.__DynamicFetcher,
            "stealthy_fetch": stealthy_fetch,
            "StealthyFetcher": self.__StealthyFetcher,
            "Selector": Selector,
            "page": self.page,
            "response": self.page,
            "pages": self.pages,
            "view": show_page_in_browser,
            "uncurl": self._curl_parser.parse,
            "curl2fetcher": curl2fetcher,
            "help": self.show_help,
        }

    def show_help(self):  # pragma: no cover
        """Show help information"""
        print(self.banner())

    def start(self):  # pragma: no cover
        """Start the interactive shell"""

        # Get our namespace with application objects
        namespace = self.get_namespace()
        ipython_shell = self.__InteractiveShellEmbed(
            banner1=self.banner(),
            banner2="",
            enable_tip=False,
            exit_msg="Bye Bye",
            user_ns=namespace,
        )
        self.shell = ipython_shell

        # If a command was provided, execute it and exit
        if self.code:
            log.info(f"Executing provided code: {self.code}")
            try:
                ipython_shell.run_cell(self.code, store_history=False)
            except Exception as e:
                log.error(f"Error executing initial code: {e}")
            return

        ipython_shell()

class Convertor:
    """Utils for the extract shell command"""

    _extension_map: Dict[str, extraction_types] = {
        "md": "markdown",
        "html": "html",
        "txt": "text",
    }

    @classmethod
    def _convert_to_markdown(cls, body: TextHandler) -> str:
        """Convert HTML content to Markdown"""
        from markdownify import markdownify

        return markdownify(body)

    @classmethod
    def _strip_noise_tags(cls, page: Selector) -> Selector:
        """Return a copy of the Selector with noise tags removed."""
        clean_root = deepcopy(page._root)
        for element in clean_root.iter(*{"script", "style", "noscript", "svg"}):
            element.drop_tree()
        return Selector(root=clean_root, url=page.url)

    @classmethod
    def _sanitize_for_ai(cls, page: Selector) -> Selector:
        """Strip hidden content that could be used for prompt injection.

        Removes CSS-hidden elements, aria-hidden elements, <template> tags,
        HTML comments, zero-width Unicode characters, and XML-incompatible
        control characters.
        """
        clean_root = deepcopy(page._root)
        for element in cast(list, _HIDDEN_XPATH(clean_root)):
            element.drop_tree()
        for element in clean_root.iter():
            if element.text:
                element.text = _CONTROL_CHARS_PATTERN.sub("", _ZWC_PATTERN.sub("", element.text))
            if element.tail:
                element.tail = _CONTROL_CHARS_PATTERN.sub("", _ZWC_PATTERN.sub("", element.tail))
        return Selector(root=clean_root, url=page.url, keep_comments=False)

    @classmethod
    def _extract_content(
        cls,
        page: Selector,
        extraction_type: extraction_types = "markdown",
        css_selector: Optional[str] = None,
        main_content_only: bool = False,
    ) -> Generator[str, None, None]:
        """Extract the content of a Selector"""
        if not page or not isinstance(page, Selector):  # pragma: no cover
            raise TypeError("Input must be of type `Selector`")
        elif not extraction_type or extraction_type not in cls._extension_map.values():
            raise ValueError(f"Unknown extraction type: {extraction_type}")
        else:
            if main_content_only:
                page = cast(Selector, page.css("body").first) or page
                page = cls._strip_noise_tags(page)
                page = cls._sanitize_for_ai(page)

            pages = [page] if not css_selector else cast(Selectors, page.css(css_selector))
            for page in pages:
                match extraction_type:
                    case "markdown":
                        yield cls._convert_to_markdown(page.html_content)
                    case "html":
                        yield page.html_content
                    case "text":
                        txt_content = page.get_all_text(
                            strip=True, ignore_tags=("script", "style", "noscript", "svg", "iframe")
                        )
                        for s in (
                            "\n",
                            "\r",
                            "\t",
                            " ",
                        ):
                            # Remove consecutive white-spaces
                            txt_content = TextHandler(re_sub(f"[{s}]+", s, txt_content))
                        yield txt_content
            yield ""

    @classmethod
    def write_content_to_file(
        cls, page: Selector, filename: str, css_selector: Optional[str] = None, main_content_only: bool = False
    ) -> None:
        """Write a Selector's content to a file"""
        if not page or not isinstance(page, Selector):  # pragma: no cover
            raise TypeError("Input must be of type `Selector`")
        elif not filename or not isinstance(filename, str) or not filename.strip():
            raise ValueError("Filename must be provided")
        elif not filename.endswith((".md", ".html", ".txt")):
            raise ValueError("Unknown file type: filename must end with '.md', '.html', or '.txt'")
        else:
            with open(filename, "w", encoding=page.encoding) as f:
                extension = filename.split(".")[-1]
                f.write(
                    "".join(
                        cls._extract_content(
                            page,
                            cls._extension_map[extension],
                            css_selector=css_selector,
                            main_content_only=main_content_only,
                        )
                    )
                )