Text Generation
PEFT
Safetensors
lora
trl
grpo
gdpo
dpo
divpo
rlhf
diversity
creative-writing
mode-collapse
Instructions to use Mercity/creative-writing-llm with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use Mercity/creative-writing-llm with PEFT:
Task type is invalid.
- Notebooks
- Google Colab
- Kaggle
add held-out eval results (base/E0/E1)
Browse files- outputs/eval/results.csv +4 -0
- outputs/eval/rows/E0-baseline.json +18 -0
- outputs/eval/rows/E1-div-individual.json +18 -0
- outputs/eval/rows/base.json +18 -0
- outputs/eval/samples/E0-baseline_examples.json +0 -0
- outputs/eval/samples/E0-baseline_full.json +0 -0
- outputs/eval/samples/E1-div-individual_examples.json +0 -0
- outputs/eval/samples/E1-div-individual_full.json +0 -0
- outputs/eval/samples/base_examples.json +0 -0
- outputs/eval/samples/base_full.json +0 -0
outputs/eval/results.csv
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
model,quality,quality_sd,novelty,gate_pass,ends_cleanly,pairwise,logdet,distinct4,self_bleu,n_clusters,eff_rank,tok_entropy,words,n_stories,n_judged
|
| 2 |
+
base,6.472222222222222,0.8014928497648134,5.645299145299146,0.975,0.975,0.11791951073656079,-32.46068154337137,0.9378452369685965,0.28749261995127673,1.0333333333333334,1.891051140465491,0.8925785425728144,384.61041666666665,480,468
|
| 3 |
+
E0-baseline,6.716386554621849,0.7814984863638379,6.1155462184873945,0.9916666666666667,0.99375,0.12589213987738918,-31.587273885630157,0.9485256140869082,0.2634834797638087,1.0,1.9566844143260946,0.8949459429529177,388.26666666666665,480,476
|
| 4 |
+
E1-div-individual,6.86411889596603,0.6854626490105716,6.43099787685775,0.98125,0.98125,0.13901946444144617,-30.080998675749,0.9463888726655647,0.26248418441736415,1.0,2.0806474609092787,0.9170640079119534,392.24791666666664,480,471
|
outputs/eval/rows/E0-baseline.json
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "E0-baseline",
|
| 3 |
+
"quality": 6.716386554621849,
|
| 4 |
+
"quality_sd": 0.7814984863638379,
|
| 5 |
+
"novelty": 6.1155462184873945,
|
| 6 |
+
"gate_pass": 0.9916666666666667,
|
| 7 |
+
"ends_cleanly": 0.99375,
|
| 8 |
+
"pairwise": 0.12589213987738918,
|
| 9 |
+
"logdet": -31.587273885630157,
|
| 10 |
+
"distinct4": 0.9485256140869082,
|
| 11 |
+
"self_bleu": 0.2634834797638087,
|
| 12 |
+
"n_clusters": 1.0,
|
| 13 |
+
"eff_rank": 1.9566844143260946,
|
| 14 |
+
"tok_entropy": 0.8949459429529177,
|
| 15 |
+
"words": 388.26666666666665,
|
| 16 |
+
"n_stories": 480,
|
| 17 |
+
"n_judged": 476
|
| 18 |
+
}
|
outputs/eval/rows/E1-div-individual.json
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "E1-div-individual",
|
| 3 |
+
"quality": 6.86411889596603,
|
| 4 |
+
"quality_sd": 0.6854626490105716,
|
| 5 |
+
"novelty": 6.43099787685775,
|
| 6 |
+
"gate_pass": 0.98125,
|
| 7 |
+
"ends_cleanly": 0.98125,
|
| 8 |
+
"pairwise": 0.13901946444144617,
|
| 9 |
+
"logdet": -30.080998675749,
|
| 10 |
+
"distinct4": 0.9463888726655647,
|
| 11 |
+
"self_bleu": 0.26248418441736415,
|
| 12 |
+
"n_clusters": 1.0,
|
| 13 |
+
"eff_rank": 2.0806474609092787,
|
| 14 |
+
"tok_entropy": 0.9170640079119534,
|
| 15 |
+
"words": 392.24791666666664,
|
| 16 |
+
"n_stories": 480,
|
| 17 |
+
"n_judged": 471
|
| 18 |
+
}
|
outputs/eval/rows/base.json
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "base",
|
| 3 |
+
"quality": 6.472222222222222,
|
| 4 |
+
"quality_sd": 0.8014928497648134,
|
| 5 |
+
"novelty": 5.645299145299146,
|
| 6 |
+
"gate_pass": 0.975,
|
| 7 |
+
"ends_cleanly": 0.975,
|
| 8 |
+
"pairwise": 0.11791951073656079,
|
| 9 |
+
"logdet": -32.46068154337137,
|
| 10 |
+
"distinct4": 0.9378452369685965,
|
| 11 |
+
"self_bleu": 0.28749261995127673,
|
| 12 |
+
"n_clusters": 1.0333333333333334,
|
| 13 |
+
"eff_rank": 1.891051140465491,
|
| 14 |
+
"tok_entropy": 0.8925785425728144,
|
| 15 |
+
"words": 384.61041666666665,
|
| 16 |
+
"n_stories": 480,
|
| 17 |
+
"n_judged": 468
|
| 18 |
+
}
|
outputs/eval/samples/E0-baseline_examples.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
outputs/eval/samples/E0-baseline_full.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
outputs/eval/samples/E1-div-individual_examples.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
outputs/eval/samples/E1-div-individual_full.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
outputs/eval/samples/base_examples.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|
outputs/eval/samples/base_full.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|