evaluation_summary.json

{
  "models": {
    "language_detector": {
      "model": "distilbert-base-multilingual-cased",
      "task": "lang_detection",
      "test_accuracy": 0.996186844613918,
      "test_f1_weighted": 0.9961852425013739,
      "test_f1_macro": 0.9966610785485943,
      "labels": [
        "AR",
        "CS",
        "EN",
        "FR"
      ]
    },
    "intent_classifier": {
      "model": "distilbert-base-multilingual-cased",
      "task": "intent",
      "test_accuracy": 0.9244604316546763,
      "test_f1_weighted": 0.9245594281209359,
      "test_f1_macro": 0.9371155856970953,
      "labels": [
        "booking",
        "complaint",
        "farewell",
        "greeting",
        "inquiry",
        "other"
      ],
      "per_language": {
        "AR": {
          "n": 736,
          "accuracy": 0.8980978260869565,
          "f1_weighted": 0.89834321233115,
          "f1_macro": 0.9153021560624438
        },
        "EN": {
          "n": 751,
          "accuracy": 0.9494007989347537,
          "f1_weighted": 0.9493381733107601,
          "f1_macro": 0.958871593735452
        },
        "FR": {
          "n": 737,
          "accuracy": 0.9253731343283582,
          "f1_weighted": 0.9252881695076642,
          "f1_macro": 0.9364015453854347
        }
      }
    },
    "ner_model": {
      "model": "distilbert-base-multilingual-cased",
      "task": "ner",
      "test_f1_micro": 0.8454746136865342,
      "test_precision_micro": 0.8299024918743229,
      "test_recall_micro": 0.8616422947131609,
      "labels": [
        "O",
        "B-PER",
        "I-PER",
        "B-LOC",
        "I-LOC",
        "B-ORG",
        "I-ORG",
        "B-DATE",
        "I-DATE"
      ],
      "per_language": {
        "AR": {
          "n": 220,
          "f1": 0.8510638297872339,
          "precision": 0.8421052631578947,
          "recall": 0.8602150537634409
        },
        "EN": {
          "n": 220,
          "f1": 0.8226059654631083,
          "precision": 0.8061538461538461,
          "recall": 0.8397435897435898
        },
        "FR": {
          "n": 220,
          "f1": 0.8641571194762685,
          "precision": 0.8434504792332268,
          "recall": 0.8859060402684564
        }
      },
      "per_entity_f1": {
        "DATE": 0.9923664122137404,
        "LOC": 0.8789808917197451,
        "ORG": 0.7246376811594203,
        "PER": 0.8771929824561404
      }
    }
  },
  "e2e": {
    "n_cases": 15,
    "language_accuracy": 1.0,
    "intent_accuracy_in_set": 0.9333333333333333,
    "retrieval_triggered": 8,
    "nonempty_reply_rate": 1.0,
    "latency_seconds": {
      "mean": 1.619,
      "median": 1.78,
      "p95": 3.282,
      "max": 3.427
    },
    "cases": [
      {
        "i": 1,
        "text": "Hello there!",
        "expected_language": "EN",
        "predicted_language": "EN",
        "language_ok": true,
        "expected_intents": [
          "greeting"
        ],
        "predicted_intent": "greeting",
        "intent_confidence": 0.9989830851554871,
        "intent_ok": true,
        "n_entities": 1,
        "n_retrieved": 0,
        "generator_used": false,
        "reply": "Hello! How can I help you today?",
        "latency_s": 0.139
      },
      {
        "i": 2,
        "text": "How can I book a hotel in Paris?",
        "expected_language": "EN",
        "predicted_language": "EN",
        "language_ok": true,
        "expected_intents": [
          "booking",
          "inquiry"
        ],
        "predicted_intent": "booking",
        "intent_confidence": 0.9978996515274048,
        "intent_ok": true,
        "n_entities": 1,
        "n_retrieved": 3,
        "generator_used": true,
        "reply": "To book a hotel in Paris, you need to select your destination, check-in and checkout dates, and the number of guests. You can either book through our website or mobile app, which allows you to make reservations at your desired hotel.",
        "latency_s": 3.096
      },
      {
        "i": 3,
        "text": "What payment methods do you accept?",
        "expected_language": "EN",
        "predicted_language": "EN",
        "language_ok": true,
        "expected_intents": [
          "inquiry"
        ],
        "predicted_intent": "inquiry",
        "intent_confidence": 0.988635778427124,
        "intent_ok": true,
        "n_entities": 0,
        "n_retrieved": 3,
        "generator_used": true,
        "reply": "We accept Visa, Mastercard, Amex, PayPal, and bank transfers as payment methods. For refund requests, we can process them through your account dashboard within 5–10 business days.",
        "latency_s": 2.369
      },
      {
        "i": 4,
        "text": "My order arrived damaged, I want a refund.",
        "expected_language": "EN",
        "predicted_language": "EN",
        "language_ok": true,
        "expected_intents": [
          "complaint"
        ],
        "predicted_intent": "complaint",
        "intent_confidence": 0.9996634721755981,
        "intent_ok": true,
        "n_entities": 0,
        "n_retrieved": 3,
        "generator_used": true,
        "reply": "To resolve this issue, you need to contact our support team. Please provide details such as the product/service name, shipping address, and date of delivery so we can investigate further.",
        "latency_s": 2.2
      },
      {
        "i": 5,
        "text": "Goodbye, see you next time.",
        "expected_language": "EN",
        "predicted_language": "EN",
        "language_ok": true,
        "expected_intents": [
          "farewell"
        ],
        "predicted_intent": "farewell",
        "intent_confidence": 0.9995536208152771,
        "intent_ok": true,
        "n_entities": 1,
        "n_retrieved": 0,
        "generator_used": false,
        "reply": "Goodbye! Have a great day.",
        "latency_s": 0.058
      },
      {
        "i": 6,
        "text": "مرحباً، كيف حالك؟",
        "expected_language": "AR",
        "predicted_language": "AR",
        "language_ok": true,
        "expected_intents": [
          "greeting"
        ],
        "predicted_intent": "greeting",
        "intent_confidence": 0.9991914629936218,
        "intent_ok": true,
        "n_entities": 1,
        "n_retrieved": 0,
        "generator_used": false,
        "reply": "مرحباً! كيف يمكنني مساعدتك اليوم؟",
        "latency_s": 0.073
      },
      {
        "i": 7,
        "text": "أحتاج أحجز فندقاً في مكة المكرمة",
        "expected_language": "AR",
        "predicted_language": "AR",
        "language_ok": true,
        "expected_intents": [
          "booking",
          "inquiry"
        ],
        "predicted_intent": "booking",
        "intent_confidence": 0.9989546537399292,
        "intent_ok": true,
        "n_entities": 1,
        "n_retrieved": 3,
        "generator_used": true,
        "reply": "لحل هذه الخدمة، قم بتحديد وجهة الطيران المناسبة التي ترغب في العودة منها إلى مكة المكرمة. ثم، قم بإضافة عدد النزلاء الذين يأتون معك للمشاركة في الطاقم وتحديد تاريخ الوصول والغروب.",
        "latency_s": 3.427
      },
      {
        "i": 8,
        "text": "ما هي طرق الدفع المتاحة لديكم؟",
        "expected_language": "AR",
        "predicted_language": "AR",
        "language_ok": true,
        "expected_intents": [
          "inquiry"
        ],
        "predicted_intent": "inquiry",
        "intent_confidence": 0.9935346841812134,
        "intent_ok": true,
        "n_entities": 2,
        "n_retrieved": 3,
        "generator_used": true,
        "reply": "الطرق المتاحة للدفع في هذا السياق تشمل البطاقات الائتمانية (فيفا، ماستركارد، أمريكان إكسبريس) وطبقة الخصم، وبنك转账، بالإضافة إلى التحويلات المالية.",
        "latency_s": 3.047
      },
      {
        "i": 9,
        "text": "لدي شكوى بخصوص طلبي الأخير",
        "expected_language": "AR",
        "predicted_language": "AR",
        "language_ok": true,
        "expected_intents": [
          "complaint"
        ],
        "predicted_intent": "other",
        "intent_confidence": 0.9814960360527039,
        "intent_ok": false,
        "n_entities": 1,
        "n_retrieved": 0,
        "generator_used": true,
        "reply": "نعم، أستطيع تقديم المعلومات اللازمة للمساهمة في حل المشكلة. كيف يمكنني مساعدتك؟",
        "latency_s": 1.46
      },
      {
        "i": 10,
        "text": "إلى اللقاء، شكراً لكم",
        "expected_language": "AR",
        "predicted_language": "AR",
        "language_ok": true,
        "expected_intents": [
          "farewell"
        ],
        "predicted_intent": "farewell",
        "intent_confidence": 0.9864930510520935,
        "intent_ok": true,
        "n_entities": 0,
        "n_retrieved": 0,
        "generator_used": false,
        "reply": "إلى اللقاء! يومٌ سعيد.",
        "latency_s": 0.052
      },
      {
        "i": 11,
        "text": "Bonjour, comment allez-vous ?",
        "expected_language": "FR",
        "predicted_language": "FR",
        "language_ok": true,
        "expected_intents": [
          "greeting"
        ],
        "predicted_intent": "greeting",
        "intent_confidence": 0.9992755055427551,
        "intent_ok": true,
        "n_entities": 2,
        "n_retrieved": 0,
        "generator_used": false,
        "reply": "Bonjour ! Comment puis-je vous aider aujourd'hui ?",
        "latency_s": 0.047
      },
      {
        "i": 12,
        "text": "Comment puis-je réserver un vol pour Lyon ?",
        "expected_language": "FR",
        "predicted_language": "FR",
        "language_ok": true,
        "expected_intents": [
          "booking",
          "inquiry"
        ],
        "predicted_intent": "inquiry",
        "intent_confidence": 0.968703031539917,
        "intent_ok": true,
        "n_entities": 1,
        "n_retrieved": 3,
        "generator_used": true,
        "reply": "Pour réserver un vol à Lyon, vous pouvez utiliser notre site web ou notre application mobile pour choisir votre destination, vos dates et le nombre de passagers. Vous disposez également des options de désinscription jusqu'à 24 heures avant le départ pour obtenir un remboursement complet.",
        "latency_s": 3.192
      },
      {
        "i": 13,
        "text": "Quels sont vos horaires d'ouverture ?",
        "expected_language": "FR",
        "predicted_language": "FR",
        "language_ok": true,
        "expected_intents": [
          "inquiry"
        ],
        "predicted_intent": "inquiry",
        "intent_confidence": 0.9928333759307861,
        "intent_ok": true,
        "n_entities": 2,
        "n_retrieved": 3,
        "generator_used": true,
        "reply": "Les horaires d'ouverture de notre service client sont de 24 heures sur 24 et 7 jours sur 7. Les bureaux sont ouverts des lundis au vendredis de 9 h à 18 h en heure locale.",
        "latency_s": 3.282
      },
      {
        "i": 14,
        "text": "Je voudrais déposer une réclamation.",
        "expected_language": "FR",
        "predicted_language": "FR",
        "language_ok": true,
        "expected_intents": [
          "complaint"
        ],
        "predicted_intent": "complaint",
        "intent_confidence": 0.9997690320014954,
        "intent_ok": true,
        "n_entities": 0,
        "n_retrieved": 3,
        "generator_used": true,
        "reply": "Désolé, mais je ne peux pas vous aider avec cette demande.",
        "latency_s": 1.78
      },
      {
        "i": 15,
        "text": "Au revoir et bonne journée",
        "expected_language": "FR",
        "predicted_language": "FR",
        "language_ok": true,
        "expected_intents": [
          "farewell"
        ],
        "predicted_intent": "farewell",
        "intent_confidence": 0.9399925470352173,
        "intent_ok": true,
        "n_entities": 1,
        "n_retrieved": 0,
        "generator_used": false,
        "reply": "Au revoir ! Bonne journée.",
        "latency_s": 0.056
      }
    ]
  }
}