| { |
| "summary": { |
| "keyword_accuracy": 0.7647058823529411, |
| "format_accuracy": 0.0, |
| "precision": 1.0, |
| "recall": 0.7647058823529411, |
| "f1": 0.8666666666666667, |
| "avg_rouge1": 0.03211764705882353, |
| "avg_rouge2": 0.006058823529411765, |
| "avg_rougeL": 0.030941176470588236, |
| "avg_response_time": 16.512235294117644 |
| }, |
| "per_category": { |
| "DS Logic": { |
| "accuracy": 1.0, |
| "format": 0.0, |
| "rougeL": 0.022, |
| "avg_time": 15.978, |
| "n": 2 |
| }, |
| "LangChain": { |
| "accuracy": 0.25, |
| "format": 0.0, |
| "rougeL": 0.022, |
| "avg_time": 17.225, |
| "n": 4 |
| }, |
| "ML": { |
| "accuracy": 1.0, |
| "format": 0.0, |
| "rougeL": 0.031, |
| "avg_time": 22.483, |
| "n": 3 |
| }, |
| "Pandas": { |
| "accuracy": 1.0, |
| "format": 0.0, |
| "rougeL": 0.031, |
| "avg_time": 17.45, |
| "n": 4 |
| }, |
| "R": { |
| "accuracy": 0.5, |
| "format": 0.0, |
| "rougeL": 0.039, |
| "avg_time": 12.786, |
| "n": 2 |
| }, |
| "Viz": { |
| "accuracy": 1.0, |
| "format": 0.0, |
| "rougeL": 0.05, |
| "avg_time": 8.516, |
| "n": 2 |
| } |
| }, |
| "per_sample": [ |
| { |
| "category": "LangChain", |
| "prompt": "What function limits the number of messages passed to a model by counting tokens in Langchain? in Python ?", |
| "keyword": "trim_messages", |
| "library": "", |
| "code": "", |
| "math": "", |
| "rouge1": 0.0, |
| "rouge2": 0.0, |
| "rougeL": 0.0, |
| "keyword_match": 0, |
| "response_time": 20.159, |
| "output_tokens": 190, |
| "format_correct": 0 |
| }, |
| { |
| "category": "LangChain", |
| "prompt": "What argument enables jq schema parsing for content_key in JSONLoader in Langchain? in Python ?", |
| "keyword": "is_content_key_jq_parsable", |
| "library": "", |
| "code": "", |
| "math": "", |
| "rouge1": 0.081, |
| "rouge2": 0.021, |
| "rougeL": 0.061, |
| "keyword_match": 0, |
| "response_time": 17.154, |
| "output_tokens": 137, |
| "format_correct": 0 |
| }, |
| { |
| "category": "LangChain", |
| "prompt": "What callback handler tracks token usage in Langchain? in Python ?", |
| "keyword": "UsageMetadataCallbackHandler", |
| "library": "", |
| "code": "", |
| "math": "", |
| "rouge1": 0.0, |
| "rouge2": 0.0, |
| "rougeL": 0.0, |
| "keyword_match": 0, |
| "response_time": 19.01, |
| "output_tokens": 183, |
| "format_correct": 0 |
| }, |
| { |
| "category": "LangChain", |
| "prompt": "What taxonomy is used on GitHub Issues for sorting and discovery in Langchain? in Python ?", |
| "keyword": "Labels", |
| "library": "", |
| "code": "", |
| "math": "", |
| "rouge1": 0.028, |
| "rouge2": 0.0, |
| "rougeL": 0.028, |
| "keyword_match": 1, |
| "response_time": 12.577, |
| "output_tokens": 123, |
| "format_correct": 0 |
| }, |
| { |
| "category": "Pandas", |
| "prompt": "How to drop missing values in a pandas DataFrame? in Python ?", |
| "keyword": "dropna", |
| "library": "", |
| "code": "", |
| "math": "", |
| "rouge1": 0.021, |
| "rouge2": 0.0, |
| "rougeL": 0.021, |
| "keyword_match": 1, |
| "response_time": 20.368, |
| "output_tokens": 200, |
| "format_correct": 0 |
| }, |
| { |
| "category": "Pandas", |
| "prompt": "How to read a CSV file using pandas? in Python ?", |
| "keyword": "read_csv", |
| "library": "", |
| "code": "", |
| "math": "", |
| "rouge1": 0.061, |
| "rouge2": 0.031, |
| "rougeL": 0.061, |
| "keyword_match": 1, |
| "response_time": 10.138, |
| "output_tokens": 98, |
| "format_correct": 0 |
| }, |
| { |
| "category": "Pandas", |
| "prompt": "How to rename a column in pandas? in Python ?", |
| "keyword": "rename", |
| "library": "", |
| "code": "", |
| "math": "", |
| "rouge1": 0.023, |
| "rouge2": 0.0, |
| "rougeL": 0.023, |
| "keyword_match": 1, |
| "response_time": 19.195, |
| "output_tokens": 182, |
| "format_correct": 0 |
| }, |
| { |
| "category": "Pandas", |
| "prompt": "How to group rows and compute aggregate in pandas? in Python ?", |
| "keyword": "groupby", |
| "library": "", |
| "code": "", |
| "math": "", |
| "rouge1": 0.019, |
| "rouge2": 0.0, |
| "rougeL": 0.019, |
| "keyword_match": 1, |
| "response_time": 20.097, |
| "output_tokens": 200, |
| "format_correct": 0 |
| }, |
| { |
| "category": "ML", |
| "prompt": "How to train a decision tree classifier in sklearn? in Python ?", |
| "keyword": "DecisionTreeClassifier", |
| "library": "", |
| "code": "", |
| "math": "", |
| "rouge1": 0.019, |
| "rouge2": 0.0, |
| "rougeL": 0.019, |
| "keyword_match": 1, |
| "response_time": 20.672, |
| "output_tokens": 200, |
| "format_correct": 0 |
| }, |
| { |
| "category": "ML", |
| "prompt": "How to split dataset into train and test in sklearn? in Python ?", |
| "keyword": "train_test_split", |
| "library": "", |
| "code": "", |
| "math": "", |
| "rouge1": 0.048, |
| "rouge2": 0.033, |
| "rougeL": 0.048, |
| "keyword_match": 1, |
| "response_time": 20.794, |
| "output_tokens": 200, |
| "format_correct": 0 |
| }, |
| { |
| "category": "ML", |
| "prompt": "How to normalize features using StandardScaler in sklearn? in Python ?", |
| "keyword": "StandardScaler", |
| "library": "", |
| "code": "", |
| "math": "", |
| "rouge1": 0.025, |
| "rouge2": 0.0, |
| "rougeL": 0.025, |
| "keyword_match": 1, |
| "response_time": 25.984, |
| "output_tokens": 136, |
| "format_correct": 0 |
| }, |
| { |
| "category": "Viz", |
| "prompt": "How to plot a heatmap using seaborn? in Python ?", |
| "keyword": "heatmap", |
| "library": "", |
| "code": "", |
| "math": "", |
| "rouge1": 0.043, |
| "rouge2": 0.0, |
| "rougeL": 0.043, |
| "keyword_match": 1, |
| "response_time": 9.108, |
| "output_tokens": 87, |
| "format_correct": 0 |
| }, |
| { |
| "category": "Viz", |
| "prompt": "How to create a histogram using matplotlib? in Python ?", |
| "keyword": "hist", |
| "library": "", |
| "code": "", |
| "math": "", |
| "rouge1": 0.057, |
| "rouge2": 0.0, |
| "rougeL": 0.057, |
| "keyword_match": 1, |
| "response_time": 7.924, |
| "output_tokens": 77, |
| "format_correct": 0 |
| }, |
| { |
| "category": "R", |
| "prompt": "How to get the maximum value of a vector in R ?", |
| "keyword": "max(", |
| "library": "", |
| "code": "", |
| "math": "", |
| "rouge1": 0.043, |
| "rouge2": 0.0, |
| "rougeL": 0.043, |
| "keyword_match": 1, |
| "response_time": 8.151, |
| "output_tokens": 75, |
| "format_correct": 0 |
| }, |
| { |
| "category": "R", |
| "prompt": "How to read a CSV file in R ?", |
| "keyword": "read.csv", |
| "library": "", |
| "code": "", |
| "math": "", |
| "rouge1": 0.035, |
| "rouge2": 0.018, |
| "rougeL": 0.035, |
| "keyword_match": 0, |
| "response_time": 17.421, |
| "output_tokens": 172, |
| "format_correct": 0 |
| }, |
| { |
| "category": "DS Logic", |
| "prompt": "What does overfitting mean in machine learning? in Python ?", |
| "keyword": "overfit", |
| "library": "", |
| "code": "", |
| "math": "", |
| "rouge1": 0.026, |
| "rouge2": 0.0, |
| "rougeL": 0.026, |
| "keyword_match": 1, |
| "response_time": 11.186, |
| "output_tokens": 106, |
| "format_correct": 0 |
| }, |
| { |
| "category": "DS Logic", |
| "prompt": "What is gradient descent in machine learning? in Python ?", |
| "keyword": "gradient", |
| "library": "", |
| "code": "", |
| "math": "", |
| "rouge1": 0.017, |
| "rouge2": 0.0, |
| "rougeL": 0.017, |
| "keyword_match": 1, |
| "response_time": 20.77, |
| "output_tokens": 196, |
| "format_correct": 0 |
| } |
| ] |
| } |
