{
  "summary": {
    "keyword_accuracy": 0.7647058823529411,
    "format_accuracy": 0.0,
    "precision": 1.0,
    "recall": 0.7647058823529411,
    "f1": 0.8666666666666667,
    "avg_rouge1": 0.03211764705882353,
    "avg_rouge2": 0.006058823529411765,
    "avg_rougeL": 0.030941176470588236,
    "avg_response_time": 16.512235294117644
  },
  "per_category": {
    "DS Logic": {
      "accuracy": 1.0,
      "format": 0.0,
      "rougeL": 0.022,
      "avg_time": 15.978,
      "n": 2
    },
    "LangChain": {
      "accuracy": 0.25,
      "format": 0.0,
      "rougeL": 0.022,
      "avg_time": 17.225,
      "n": 4
    },
    "ML": {
      "accuracy": 1.0,
      "format": 0.0,
      "rougeL": 0.031,
      "avg_time": 22.483,
      "n": 3
    },
    "Pandas": {
      "accuracy": 1.0,
      "format": 0.0,
      "rougeL": 0.031,
      "avg_time": 17.45,
      "n": 4
    },
    "R": {
      "accuracy": 0.5,
      "format": 0.0,
      "rougeL": 0.039,
      "avg_time": 12.786,
      "n": 2
    },
    "Viz": {
      "accuracy": 1.0,
      "format": 0.0,
      "rougeL": 0.05,
      "avg_time": 8.516,
      "n": 2
    }
  },
  "per_sample": [
    {
      "category": "LangChain",
      "prompt": "What function limits the number of messages passed to a model by counting tokens in Langchain? in Python ?",
      "keyword": "trim_messages",
      "library": "",
      "code": "",
      "math": "",
      "rouge1": 0.0,
      "rouge2": 0.0,
      "rougeL": 0.0,
      "keyword_match": 0,
      "response_time": 20.159,
      "output_tokens": 190,
      "format_correct": 0
    },
    {
      "category": "LangChain",
      "prompt": "What argument enables jq schema parsing for content_key in JSONLoader in Langchain? in Python ?",
      "keyword": "is_content_key_jq_parsable",
      "library": "",
      "code": "",
      "math": "",
      "rouge1": 0.081,
      "rouge2": 0.021,
      "rougeL": 0.061,
      "keyword_match": 0,
      "response_time": 17.154,
      "output_tokens": 137,
      "format_correct": 0
    },
    {
      "category": "LangChain",
      "prompt": "What callback handler tracks token usage in Langchain? in Python ?",
      "keyword": "UsageMetadataCallbackHandler",
      "library": "",
      "code": "",
      "math": "",
      "rouge1": 0.0,
      "rouge2": 0.0,
      "rougeL": 0.0,
      "keyword_match": 0,
      "response_time": 19.01,
      "output_tokens": 183,
      "format_correct": 0
    },
    {
      "category": "LangChain",
      "prompt": "What taxonomy is used on GitHub Issues for sorting and discovery in Langchain? in Python ?",
      "keyword": "Labels",
      "library": "",
      "code": "",
      "math": "",
      "rouge1": 0.028,
      "rouge2": 0.0,
      "rougeL": 0.028,
      "keyword_match": 1,
      "response_time": 12.577,
      "output_tokens": 123,
      "format_correct": 0
    },
    {
      "category": "Pandas",
      "prompt": "How to drop missing values in a pandas DataFrame? in Python ?",
      "keyword": "dropna",
      "library": "",
      "code": "",
      "math": "",
      "rouge1": 0.021,
      "rouge2": 0.0,
      "rougeL": 0.021,
      "keyword_match": 1,
      "response_time": 20.368,
      "output_tokens": 200,
      "format_correct": 0
    },
    {
      "category": "Pandas",
      "prompt": "How to read a CSV file using pandas? in Python ?",
      "keyword": "read_csv",
      "library": "",
      "code": "",
      "math": "",
      "rouge1": 0.061,
      "rouge2": 0.031,
      "rougeL": 0.061,
      "keyword_match": 1,
      "response_time": 10.138,
      "output_tokens": 98,
      "format_correct": 0
    },
    {
      "category": "Pandas",
      "prompt": "How to rename a column in pandas? in Python ?",
      "keyword": "rename",
      "library": "",
      "code": "",
      "math": "",
      "rouge1": 0.023,
      "rouge2": 0.0,
      "rougeL": 0.023,
      "keyword_match": 1,
      "response_time": 19.195,
      "output_tokens": 182,
      "format_correct": 0
    },
    {
      "category": "Pandas",
      "prompt": "How to group rows and compute aggregate in pandas? in Python ?",
      "keyword": "groupby",
      "library": "",
      "code": "",
      "math": "",
      "rouge1": 0.019,
      "rouge2": 0.0,
      "rougeL": 0.019,
      "keyword_match": 1,
      "response_time": 20.097,
      "output_tokens": 200,
      "format_correct": 0
    },
    {
      "category": "ML",
      "prompt": "How to train a decision tree classifier in sklearn? in Python ?",
      "keyword": "DecisionTreeClassifier",
      "library": "",
      "code": "",
      "math": "",
      "rouge1": 0.019,
      "rouge2": 0.0,
      "rougeL": 0.019,
      "keyword_match": 1,
      "response_time": 20.672,
      "output_tokens": 200,
      "format_correct": 0
    },
    {
      "category": "ML",
      "prompt": "How to split dataset into train and test in sklearn? in Python ?",
      "keyword": "train_test_split",
      "library": "",
      "code": "",
      "math": "",
      "rouge1": 0.048,
      "rouge2": 0.033,
      "rougeL": 0.048,
      "keyword_match": 1,
      "response_time": 20.794,
      "output_tokens": 200,
      "format_correct": 0
    },
    {
      "category": "ML",
      "prompt": "How to normalize features using StandardScaler in sklearn? in Python ?",
      "keyword": "StandardScaler",
      "library": "",
      "code": "",
      "math": "",
      "rouge1": 0.025,
      "rouge2": 0.0,
      "rougeL": 0.025,
      "keyword_match": 1,
      "response_time": 25.984,
      "output_tokens": 136,
      "format_correct": 0
    },
    {
      "category": "Viz",
      "prompt": "How to plot a heatmap using seaborn? in Python ?",
      "keyword": "heatmap",
      "library": "",
      "code": "",
      "math": "",
      "rouge1": 0.043,
      "rouge2": 0.0,
      "rougeL": 0.043,
      "keyword_match": 1,
      "response_time": 9.108,
      "output_tokens": 87,
      "format_correct": 0
    },
    {
      "category": "Viz",
      "prompt": "How to create a histogram using matplotlib? in Python ?",
      "keyword": "hist",
      "library": "",
      "code": "",
      "math": "",
      "rouge1": 0.057,
      "rouge2": 0.0,
      "rougeL": 0.057,
      "keyword_match": 1,
      "response_time": 7.924,
      "output_tokens": 77,
      "format_correct": 0
    },
    {
      "category": "R",
      "prompt": "How to get the maximum value of a vector in R ?",
      "keyword": "max(",
      "library": "",
      "code": "",
      "math": "",
      "rouge1": 0.043,
      "rouge2": 0.0,
      "rougeL": 0.043,
      "keyword_match": 1,
      "response_time": 8.151,
      "output_tokens": 75,
      "format_correct": 0
    },
    {
      "category": "R",
      "prompt": "How to read a CSV file in R ?",
      "keyword": "read.csv",
      "library": "",
      "code": "",
      "math": "",
      "rouge1": 0.035,
      "rouge2": 0.018,
      "rougeL": 0.035,
      "keyword_match": 0,
      "response_time": 17.421,
      "output_tokens": 172,
      "format_correct": 0
    },
    {
      "category": "DS Logic",
      "prompt": "What does overfitting mean in machine learning? in Python ?",
      "keyword": "overfit",
      "library": "",
      "code": "",
      "math": "",
      "rouge1": 0.026,
      "rouge2": 0.0,
      "rougeL": 0.026,
      "keyword_match": 1,
      "response_time": 11.186,
      "output_tokens": 106,
      "format_correct": 0
    },
    {
      "category": "DS Logic",
      "prompt": "What is gradient descent in machine learning? in Python ?",
      "keyword": "gradient",
      "library": "",
      "code": "",
      "math": "",
      "rouge1": 0.017,
      "rouge2": 0.0,
      "rougeL": 0.017,
      "keyword_match": 1,
      "response_time": 20.77,
      "output_tokens": 196,
      "format_correct": 0
    }
  ]
}