Update README.md

README.md

@@ -1,59 +1,58 @@
 ---
 license: apache-2.0
 base_model:
-  - Qwen/Qwen3-Embedding-0.6B
+- Qwen/Qwen3-Embedding-0.6B
 pipeline_tag: feature-extraction
 library_name: sentence-transformers
-
 language:
-  - multilingual
-
+- multilingual
 tags:
-  - text-embeddings
-  - feature-extraction
-  - sentence-transformers
-  - transformers
-  - multilingual
-  - qwen3
-  - silas
-  - alfotech
-
+- text-embeddings
+- feature-extraction
+- sentence-transformers
+- transformers
+- multilingual
+- qwen3
+- alfotech
+- silas
+- rag
+- semantic-search
+- vector-search
 model-index:
-  - name: silas-embedding-0.6b
-    results:
-      - task:
-          type: feature-extraction
-          name: Embedding Model — Easy Benchmark
-        metrics:
-          - type: recall_at_1
-            name: Recall@1
-            value: 0.967
-          - type: recall_at_5
-            name: Recall@5
-            value: 0.999
-          - type: recall_at_10
-            name: Recall@10
-            value: 1.000
-          - type: mrr_at_10
-            name: MRR@10
-            value: 0.9815
-
-      - task:
-          type: feature-extraction
-          name: Embedding Model — Hard Benchmark
-        metrics:
-          - type: recall_at_1
-            name: Recall@1
-            value: 0.614
-          - type: recall_at_5
-            name: Recall@5
-            value: 0.883
-          - type: recall_at_10
-            name: Recall@10
-            value: 0.929
-          - type: mrr_at_10
-            name: MRR@10
-            value: 0.7268
+- name: silas-embedding-0.6b
+  results:
+  - task:
+      type: feature-extraction
+      name: Embedding Model — Easy Benchmark
+    metrics:
+    - type: recall_at_1
+      value: 0.967
+      name: Recall@1
+    - type: recall_at_5
+      value: 0.999
+      name: Recall@5
+    - type: recall_at_10
+      value: 1.0
+      name: Recall@10
+    - type: mrr_at_10
+      value: 0.9815
+      name: MRR@10
+  - task:
+      type: feature-extraction
+      name: Embedding Model — Hard Benchmark
+    metrics:
+    - type: recall_at_1
+      value: 0.614
+      name: Recall@1
+    - type: recall_at_5
+      value: 0.883
+      name: Recall@5
+    - type: recall_at_10
+      value: 0.929
+      name: Recall@10
+    - type: mrr_at_10
+      value: 0.7268
+      name: MRR@10
 ---
 
 <p align="center">
@@ -64,67 +63,551 @@ model-index:
 
 <p align="center">
   <b>Production-Grade Embedding Model by Alfo Tech Industries</b><br/>
-  Multilingual · Long Context · Semantic Understanding · RAG Ready
+  Multilingual · Long Context · Semantic Understanding · AI Infrastructure
 </p>
 
 <p align="center">
   <a href="https://huggingface.co/alfotech/silas-embedding-0.6b">
-    <img src="https://img.shields.io/badge/🤗-Hugging%20Face-yellow" alt="Hugging Face">
+    <img src="https://img.shields.io/badge/🤗_Hugging_Face-Model-yellow" alt="Hugging Face"/>
   </a>
-  <img src="https://img.shields.io/badge/Parameters-0.6B-6D5BFF" alt="Parameters">
-  <img src="https://img.shields.io/badge/Context-32K-00E5FF" alt="Context">
-  <img src="https://img.shields.io/badge/Languages-100%2B-success" alt="Languages">
-  <img src="https://img.shields.io/badge/License-Apache--2.0-blue" alt="License">
+  <img src="https://img.shields.io/badge/Parameters-0.6B-6D5BFF" alt="Parameters"/>
+  <img src="https://img.shields.io/badge/Context-32K-00E5FF" alt="Context"/>
+  <img src="https://img.shields.io/badge/Embedding-1024D-8B5CF6" alt="Embedding Dimension"/>
+  <img src="https://img.shields.io/badge/Languages-100%2B-22C55E" alt="Languages"/>
+  <img src="https://img.shields.io/badge/License-Apache--2.0-2563EB" alt="License"/>
+  <img src="https://img.shields.io/badge/Status-Active_Development-F59E0B" alt="Status"/>
+</p>
+
+<p align="center">
+  <a href="#quick-start">Quick Start</a> ·
+  <a href="#benchmark-dashboard">Benchmarks</a> ·
+  <a href="#production-deployment">Deployment</a> ·
+  <a href="#intended-use">Intended Use</a> ·
+  <a href="#faq">FAQ</a> ·
+  <a href="#citation">Citation</a>
 </p>
 
 ---
 
-# Silas
+## Table of Contents
+
+- [Model Profile](#model-profile)
+- [Why Silas?](#why-silas)
+- [Embedding Capabilities](#embedding-capabilities)
+- [Benchmark Dashboard](#benchmark-dashboard)
+  - [Easy Embedding Benchmark](#easy-embedding-benchmark)
+  - [Hard Embedding Benchmark](#hard-embedding-benchmark)
+  - [Benchmark Comparison](#benchmark-comparison)
+  - [Semantic Analysis](#semantic-analysis)
+- [Development Architecture](#development-architecture)
+- [Quick Start](#quick-start)
+- [Usage Guide](#usage-guide)
+  - [Query vs. Document Embeddings](#query-vs-document-embeddings)
+  - [Variable Embedding Size (Matryoshka)](#variable-embedding-size-matryoshka)
+  - [Batch Processing](#batch-processing)
+- [Production Deployment](#production-deployment)
+  - [Recommended Production Config](#production-starting-point)
+  - [Serving Options](#serving-options)
+  - [Integration Architecture](#integration)
+- [Intended Use](#intended-use)
+- [Bias, Risks & Limitations](#bias-risks--limitations)
+- [Current Development Status](#current-development-status)
+- [Roadmap](#roadmap)
+- [Versioning](#versioning)
+- [FAQ](#faq)
+- [Reproducibility](#reproducibility)
+- [Citation](#citation)
+- [License](#license)
+- [Contact & Support](#contact--support)
 
-## An Embedding Model Built for Real-World AI Systems
+---
+
+# Model Profile
 
 **Silas Embedding 0.6B** is a production-oriented multilingual embedding model developed by **Alfo Tech Industries**.
 
-Silas is designed to convert text into dense vector representations suitable for semantic understanding, similarity search, retrieval systems, knowledge bases, and Retrieval-Augmented Generation.
+Silas converts text into dense vector representations designed for semantic understanding, similarity measurement, knowledge systems, AI search, Retrieval-Augmented Generation, and vector databases. The model is based on **Qwen3-Embedding-0.6B** and follows an evaluation-first engineering workflow focused on difficult semantic matching scenarios rather than only easy, well-separated cases.
 
-The model is built upon **Qwen3-Embedding-0.6B** and follows an evaluation-first development methodology focused on difficult semantic matching scenarios.
+| Property                   | Specification         |
+| --------------------------- | ---------------------- |
+| **Model**                   | Silas Embedding 0.6B  |
+| **Organization**            | Alfo Tech Industries  |
+| **Base Model**               | Qwen3-Embedding-0.6B  |
+| **Parameters**               | 0.6B                  |
+| **Context Window**           | 32K                   |
+| **Default Embedding Size**   | 1024 dimensions       |
+| **Supported Dimensions**     | 32–1024               |
+| **Languages**                | 100+                  |
+| **Architecture Type**        | Text Embedding        |
+| **Instruction Aware**        | Yes                   |
+| **Framework**                | Sentence Transformers |
+| **License**                  | Apache-2.0            |
 
 ---
 
-# At a Glance
+# Why Silas?
+
+A useful embedding model should do more than place related sentences close together — it should help a downstream system distinguish the *correct* semantic match from close, plausible-looking alternatives.
+
+```text
+Correct Semantic Meaning
+            │
+            ▼
+     Silas Embedding
+            │
+            ▼
+Dense Vector Representation
+            │
+     ┌──────┼────────┐
+     ▼      ▼        ▼
+   Search   RAG    Similarity
+```
+
+For this reason, Silas is evaluated on both a straightforward **Easy Benchmark** and a deliberately adversarial **Hard Benchmark** built from semantically similar candidates — the gap between the two is treated as the real signal of embedding quality, not the easy score alone.
+
+---
+
+# Embedding Capabilities
+
+| Capability             | Example                                   |
+| ----------------------- | ------------------------------------------ |
+| **Semantic Search**      | Find conceptually relevant documents      |
+| **RAG**                  | Retrieve context for language models      |
+| **Knowledge Bases**      | Search enterprise documentation           |
+| **FAQ Matching**         | Match questions with answers              |
+| **Similarity Systems**   | Compare semantic meaning                  |
+| **Code Search**          | Retrieve related programming content      |
+| **Vector Databases**     | Store and query dense vectors             |
+| **Multilingual AI**      | Represent text across supported languages |
+
+---
+
+# Benchmark Dashboard
+
+## Evaluation Overview
+
+| Benchmark                    | Corpus            | Queries | Primary Purpose                   |
+| ------------------------------ | ------------------ | ------- | ----------------------------------- |
+| **Easy Embedding Benchmark**   | Evaluation set     | 1,000   | Standard semantic matching        |
+| **Hard Embedding Benchmark**   | 72,635 documents   | 1,000   | Difficult semantic discrimination |
+
+The hard benchmark introduces semantically similar candidates, making it a more demanding test of embedding quality than corpus-level recall alone.
+
+## Easy Embedding Benchmark
+
+| Metric        | Score   |
+| -------------- | ------- |
+| **Recall@1**   | 96.70%  |
+| **Recall@5**   | 99.90%  |
+| **Recall@10**  | 100.00% |
+| **MRR@10**     | 98.15%  |
+
+**Interpretation:** The model performs strongly when the correct semantic match is relatively distinguishable from competing candidates — this reflects typical FAQ-matching and coarse retrieval workloads.
+
+## Hard Embedding Benchmark
+
+### Evaluation Setup
+
+| Parameter                 | Value                           |
+| --------------------------- | -------------------------------- |
+| **Corpus Size**              | 72,635 documents                |
+| **Evaluation Queries**       | 1,000                           |
+| **Candidate Environment**    | Semantically similar documents  |
+| **Primary Metric**           | Recall@1                        |
+| **Ranking Metric**           | MRR@10                          |
+
+### Results
+
+| Metric        | Score  |
+| -------------- | ------ |
+| **Recall@1**   | 61.40% |
+| **Recall@5**   | 88.30% |
+| **Recall@10**  | 92.90% |
+| **MRR@10**     | 72.68% |
+
+This benchmark is intentionally more difficult because incorrect candidates can be semantically close to the correct document — the kind of near-miss confusion that matters most in production RAG pipelines.
+
+## Benchmark Comparison
+
+| Metric        | Easy    | Hard   |
+| -------------- | ------- | ------ |
+| **Recall@1**   | 96.70%  | 61.40% |
+| **Recall@5**   | 99.90%  | 88.30% |
+| **Recall@10**  | 100.00% | 92.90% |
+| **MRR@10**     | 98.15%  | 72.68% |
+
+The gap between the two evaluations is a useful signal of how the model behaves once results get semantically crowded, rather than an artifact of an easy test set.
+
+## Semantic Analysis
+
+A detailed failure analysis was run on the hard benchmark results:
+
+| Analysis                            | Result  |
+| ------------------------------------- | ------- |
+| **Queries analyzed**                  | 1,000   |
+| **Semantic hard negatives**           | 386     |
+| **Average positive similarity**       | 0.6894  |
+| **Average top-negative similarity**   | 0.6534  |
+| **Average separation margin**         | 0.0360  |
+| **Potential noisy positives**         | 0       |
+| **Potential duplicates**              | 0       |
+
+**Key finding:** The average similarity margin between the positive document and the strongest negative candidate was only **0.0360**. This indicates the remaining challenge is primarily **fine-grained semantic separation**, not simply filtering out unrelated content — which directly informs the hard-negative mining plan in the [Roadmap](#roadmap).
+
+---
+
+# Development Architecture
+
+Silas follows an evaluation-driven development process:
+
+```text
+                 BASE MODEL
+                     │
+                     ▼
+            ┌─────────────────┐
+            │ Baseline Test   │
+            └────────┬────────┘
+                     │
+                     ▼
+            ┌─────────────────┐
+            │ Hard Benchmark  │
+            └────────┬────────┘
+                     │
+                     ▼
+            ┌─────────────────┐
+            │ Failure Analysis│
+            └────────┬────────┘
+                     │
+                     ▼
+            ┌─────────────────┐
+            │ Hard Negatives  │
+            └────────┬────────┘
+                     │
+                     ▼
+            ┌─────────────────┐
+            │ Fine-Tuning     │
+            └────────┬────────┘
+                     │
+                     ▼
+            ┌─────────────────┐
+            │ Validation      │
+            └────────┬────────┘
+                     │
+                     ▼
+            PRODUCTION MODEL
+```
+
+---
+
+# Quick Start
+
+## Install
+
+```bash
+pip install -U sentence-transformers
+```
+
+## Load the Model
+
+```python
+from sentence_transformers import SentenceTransformer
+
+model = SentenceTransformer(
+    "alfotech/silas-embedding-0.6b"
+)
+```
+
+## Generate Embeddings
+
+```python
+texts = [
+    "Artificial intelligence improves information retrieval.",
+    "Vector embeddings represent semantic relationships."
+]
+
+embeddings = model.encode(
+    texts,
+    normalize_embeddings=True
+)
+
+print(embeddings.shape)
+```
+
+---
+
+# Usage Guide
+
+## Query vs. Document Embeddings
+
+For retrieval-style workloads, query instructions can describe the intended task:
+
+```text
+Instruct: Given a search query, retrieve relevant passages.
+
+Query: How do I optimize transformer inference?
+```
+
+```python
+queries = [
+    "How do I optimize transformer inference?"
+]
+
+documents = [
+    "Quantization, batching, and KV-cache optimization can reduce inference latency.",
+    "The stock market closed higher after strong earnings reports."
+]
+
+query_embeddings = model.encode(
+    queries,
+    prompt_name="query",
+    normalize_embeddings=True
+)
+
+document_embeddings = model.encode(
+    documents,
+    normalize_embeddings=True
+)
+
+scores = model.similarity(
+    query_embeddings,
+    document_embeddings
+)
+
+print(scores)
+```
+
+> Use `prompt_name="query"` for the search-query side of asymmetric retrieval and leave documents un-prefixed. For symmetric tasks (e.g. clustering, deduplication), encode both sides the same way.
+
+## Variable Embedding Size (Matryoshka)
 
-| Specification | Details |
-|---|---|
-| Model | Silas Embedding 0.6B |
-| Organization | Alfo Tech Industries |
-| Base Model | Qwen3-Embedding-0.6B |
-| Parameters | **0.6B** |
-| Context Window | **32K** |
-| Default Embedding Dimension | **1024** |
-| Supported Dimensions | **32–1024** |
-| Languages | **100+** |
-| Instruction Aware | Yes |
-| Framework | Sentence Transformers |
-| License | Apache-2.0 |
+Silas supports configurable embedding dimensions so storage and latency can be traded against representation capacity without re-encoding your corpus with a different model:
+
+```python
+embeddings = model.encode(
+    ["Efficient vectors reduce storage requirements."],
+    normalize_embeddings=True,
+    truncate_dim=256
+)
+```
+
+Supported range: **32 → 1024 dimensions**.
+
+Use larger representations when maximizing representation capacity is important (e.g. hard semantic discrimination), or smaller vectors when storage and throughput are the priority (e.g. large-scale first-pass retrieval).
+
+## Batch Processing
+
+```python
+embeddings = model.encode(
+    large_text_list,
+    batch_size=32,
+    normalize_embeddings=True,
+    show_progress_bar=True
+)
+```
+
+For corpora in the millions of documents, encode in chunks and stream directly into your vector database's bulk-insert API rather than holding all vectors in memory at once.
 
 ---
 
-# What Silas Does
+# Production Deployment
+
+## Production Starting Point
+
+| Configuration           | Recommended Value |
+| ------------------------- | ------------------ |
+| **Sequence Length**        | 512                |
+| **Batch Size**              | 32                 |
+| **Normalization**           | Enabled            |
+| **Similarity Metric**       | Cosine             |
+| **Default Dimension**       | 1024               |
+
+These are recommended starting points — production workloads should be benchmarked using representative data before locking in a configuration.
+
+## Serving Options
 
-Silas transforms text into numerical representations that preserve semantic relationships.
+Silas is a standard `sentence-transformers`-compatible model, so it can be served with:
+
+- **Sentence Transformers**, directly in a Python service, for simplest integration and full control over batching.
+- **Hugging Face Text Embeddings Inference (TEI)** or similar dedicated embedding servers, for higher-throughput, lower-latency serving behind a REST/gRPC endpoint.
+- **ONNX / quantized export**, where CPU-only or edge deployment is required and GPU serving isn't available.
+
+Choice of serving stack should be validated against your own latency, throughput, and hardware constraints — figures above are configuration defaults, not deployment benchmarks.
+
+## Integration
 
 ```text
-Text
-  │
-  ▼
-Silas Embedding Model
-  │
-  ▼
-Dense Vector
-  │
-  ├── Semantic Search
-  ├── RAG
-  ├── Similarity Search
-  ├── Knowledge Retrieval
-  ├── FAQ Matching
-  └── Vector Databases
+Application
+     │
+     ▼
+Silas Embedding
+     │
+     ▼
+Vector Database
+     │
+ ┌───┼───────────────┐
+ ▼   ▼               ▼
+FAISS Qdrant      pgvector
+     │
+     ▼
+Nearest Neighbors
+     │
+     ▼
+AI Application
+```
+
+---
+
+# Intended Use
+
+Silas is intended as the **representation layer** in semantic search, RAG, knowledge-base retrieval, and similarity-matching systems, primarily where:
+
+- Text needs to be compared or retrieved by meaning rather than exact keyword match.
+- A downstream ranking, generation, or filtering step consumes the retrieved candidates (Silas returns similarity, not a final answer).
+- Multilingual input is expected, or embedding size needs to be tuned per deployment tier.
+
+**Out of scope:** Silas does not verify factual correctness, does not perform classification or generation on its own, and should not be used as a sole safety or content-moderation filter — similarity scores reflect semantic closeness, not truth or safety.
+
+---
+
+# Bias, Risks & Limitations
+
+Silas is currently evaluated primarily through embedding and semantic-matching experiments. Current limitations include:
+
+- External leaderboard evaluation has not been independently performed.
+- Long-context behavior has not been independently re-benchmarked across the entire 32K context window.
+- Performance may vary across domains not represented in the benchmark corpus.
+- Multilingual performance should be validated against the specific languages relevant to the target application — "100+ languages supported" reflects the base model's training, not per-language benchmarking by Alfo Tech Industries.
+- Similarity scores do not represent factual correctness, and retrieved-but-similar text can still be wrong, biased, or outdated relative to the query's intent.
+
+For high-stakes applications (legal, medical, financial, safety-critical), validate Silas using domain-specific evaluation datasets and appropriate system-level safeguards rather than relying on the benchmarks above alone.
+
+---
+
+# Current Development Status
+
+| Component                          | Status     |
+| ------------------------------------ | ---------- |
+| Base Model Integration               | ✅ Complete |
+| Baseline Evaluation                  | ✅ Complete |
+| Easy Benchmark                       | ✅ Complete |
+| Hard Benchmark                       | ✅ Complete |
+| Failure Analysis                     | ✅ Complete |
+| Semantic Hard-Negative Discovery     | ✅ Complete |
+| Retrieval Fine-Tuning                | 🔄 Ongoing |
+| Extended Blind Evaluation            | 🔄 Planned |
+| Production Optimization              | 🔄 Planned |
+
+---
+
+# Roadmap
+
+**Model Quality**
+- Larger curated datasets
+- Improved semantic hard-negative mining
+- Stronger domain adaptation
+- Broader multilingual evaluation
+
+**Benchmarking**
+- Blind evaluation sets
+- Additional embedding benchmarks
+- Expanded production-scale benchmarks
+- Cross-domain evaluation
+
+**Deployment**
+- Higher-throughput inference
+- Optimized vector dimensions
+- Serving infrastructure
+- Additional vector database integrations
+
+---
+
+# Versioning
+
+| Version | Status  | Notes                                                   |
+| ------- | ------- | -------------------------------------------------------- |
+| v1      | Current | Initial public release; baseline + hard-benchmark results above |
+
+Future releases that materially change benchmark numbers or the recommended production config will be tagged as new versions rather than silently overwriting these results.
+
+---
+
+# FAQ
+
+**Which dimension should I use?**
+Start at the default 1024 for best hard-case separation. Drop to a smaller `truncate_dim` (e.g. 256–384) once you've confirmed accuracy holds on your own hard-negative style data — don't shrink dimensions before benchmarking on your corpus.
+
+**Do I need the `query` prompt for every use case?**
+Only for asymmetric retrieval (short query → long document). For symmetric comparison tasks (dedup, clustering, paraphrase matching), encode both sides without the query prompt.
+
+**Why is Recall@1 so much lower on the hard benchmark?**
+Because the hard benchmark's negatives are semantically close to the correct answer by design (see [Semantic Analysis](#semantic-analysis)) — this is expected and is the metric the roadmap's hard-negative mining work targets directly.
+
+**Is this model safe to use as a standalone fact-checker or filter?**
+No — see [Intended Use](#intended-use) and [Bias, Risks & Limitations](#bias-risks--limitations).
+
+---
+
+# Reproducibility
+
+Silas follows an evaluation-first development methodology. The project tracks:
+
+- Model configuration
+- Dataset processing
+- Benchmark methodology
+- Failure analysis
+- Inference configuration
+- Production recommendations
+
+All benchmark values presented above are measured results from the current development evaluation.
+
+---
+
+# Citation
+
+## Silas Embedding
+
+```bibtex
+@misc{silas_embedding_2026,
+  title={Silas Embedding 0.6B},
+  author={Alfo Tech Industries},
+  year={2026},
+  publisher={Hugging Face},
+  url={https://huggingface.co/alfotech/silas-embedding-0.6b}
+}
+```
+
+## Qwen3 Embedding
+
+```bibtex
+@article{qwen3embedding,
+  title={Qwen3 Embedding: Advancing Text Embedding and Reranking Through Foundation Models},
+  author={Zhang, Yanzhao and Li, Mingxin and Long, Dingkun and others},
+  journal={arXiv preprint arXiv:2506.05176},
+  year={2025}
+}
+```
+
+---
+
+# License
+
+**Apache-2.0**
+
+Silas is based on Qwen3-Embedding-0.6B and follows the applicable licensing requirements of the upstream model.
+
+---
+
+# Contact & Support
+
+- **Organization:** Alfo Tech Industries — [github.com/Alfo-Tech-Lab](https://github.com/Alfo-Tech-Lab)
+- **Issues & feedback:** open an issue on the model's Hugging Face repository discussion tab, or via the GitHub organization above.
+
+---
+
+<p align="center">
+  <b>Silas Embedding 0.6B</b><br/>
+  <b>Alfo Tech Industries</b><br/>
+  <i>Production-grade embedding infrastructure for modern AI systems.</i>
+  <br/><br/>
+  <a href="https://huggingface.co/alfotech/silas-embedding-0.6b">Hugging Face</a>
+</p>