diff --git a/.gitattributes b/.gitattributes index df5889b11cddf35bf986ad2e98cf050dbc99f624..faa9a4606da64ff543a887372e8aa9d75a2803c3 100644 --- a/.gitattributes +++ b/.gitattributes @@ -52,3 +52,22 @@ llama2:13b-chat/all_20k_uniform_2choice/choice/probability/wandb/offline-run-202 llama2:13b-chat/winogrande/choice/correctness/wandb/offline-run-20250309_172205-uagejaxj/run-uagejaxj.wandb filter=lfs diff=lfs merge=lfs -text llama2:13b-chat/winogrande/choice/probabilities/checkpoint-10000/eval_winogrande_probabilities/wandb/offline-run-20250306_225838-4g5riji6/run-4g5riji6.wandb filter=lfs diff=lfs merge=lfs -text llama2:13b-chat/winogrande/choice/probability/wandb/offline-run-20250309_172801-ne6ggu1n/run-ne6ggu1n.wandb filter=lfs diff=lfs merge=lfs -text +cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250308_234315-132ixfh6/run-132ixfh6.wandb filter=lfs diff=lfs merge=lfs -text +cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250309_001704-olsr0k56/run-olsr0k56.wandb filter=lfs diff=lfs merge=lfs -text +cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_211847-wpm44dvi/run-wpm44dvi.wandb filter=lfs diff=lfs merge=lfs -text +cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_212521-bkazmwmf/run-bkazmwmf.wandb filter=lfs diff=lfs merge=lfs -text +cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_213403-pp1ltnvm/run-pp1ltnvm.wandb filter=lfs diff=lfs merge=lfs -text +cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_214217-9zyowpty/run-9zyowpty.wandb filter=lfs diff=lfs merge=lfs -text +cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_214844-tmkoufpd/run-tmkoufpd.wandb filter=lfs diff=lfs merge=lfs -text +cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_221649-yxmxbw23/run-yxmxbw23.wandb filter=lfs diff=lfs merge=lfs -text +cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222015-e0wgt1xk/run-e0wgt1xk.wandb filter=lfs diff=lfs merge=lfs -text +cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/run-49mgyijh.wandb filter=lfs diff=lfs merge=lfs -text +cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233926-lvxg4rg8/run-lvxg4rg8.wandb filter=lfs diff=lfs merge=lfs -text +cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234031-htonl59j/run-htonl59j.wandb filter=lfs diff=lfs merge=lfs -text +cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234033-nuxm0lk5/run-nuxm0lk5.wandb filter=lfs diff=lfs merge=lfs -text +cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250309_001852-ojcexm6h/run-ojcexm6h.wandb filter=lfs diff=lfs merge=lfs -text +cached_results/llama2:13b-chat/all_20k_uniform_2choice/choice/correctness/wandb/offline-run-20250313_211122-w5ht7ylx/run-w5ht7ylx.wandb filter=lfs diff=lfs merge=lfs -text +cached_results/llama2:13b-chat/all_20k_uniform_2choice/choice/probability/wandb/offline-run-20250313_211206-7l6msfdp/run-7l6msfdp.wandb filter=lfs diff=lfs merge=lfs -text +cached_results/llama2:13b-chat/winogrande/choice/correctness/wandb/offline-run-20250309_172205-uagejaxj/run-uagejaxj.wandb filter=lfs diff=lfs merge=lfs -text +cached_results/llama2:13b-chat/winogrande/choice/probabilities/checkpoint-10000/eval_winogrande_probabilities/wandb/offline-run-20250306_225838-4g5riji6/run-4g5riji6.wandb filter=lfs diff=lfs merge=lfs -text +cached_results/llama2:13b-chat/winogrande/choice/probability/wandb/offline-run-20250309_172801-ne6ggu1n/run-ne6ggu1n.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/cached_results/all_results.csv b/cached_results/all_results.csv new file mode 100644 index 0000000000000000000000000000000000000000..4647fae6d8edc807ff6a84ede8f965d3bcdb0afb --- /dev/null +++ b/cached_results/all_results.csv @@ -0,0 +1,7 @@ +model_name,dataset,training_target,eval_dataset,eval_split,eval_dataset_len,qa_accuracy,correctness_accuracy,correctness_auroc,correctness_macro-f1,ece,probability_accuracy,probability_macro-f1,entropy_accuracy,entropy_macro-f1 +llama2:13b-chat,all_20k_uniform_2choice,correctness,all_20k_uniform_2choice,train,16400,0.2821,0.9179,0.9733,0.8996,0.0044,,,, +llama2:13b-chat,all_20k_uniform_2choice,correctness,all_20k_uniform_2choice,validation,2000,0.1275,0.969,0.9924,0.9303,0.0081,,,, +llama2:13b-chat,all_20k_uniform_2choice,probability,all_20k_uniform_2choice,train,16400,,,,,,0.8552,0.5829,, +llama2:13b-chat,all_20k_uniform_2choice,probability,all_20k_uniform_2choice,validation,2000,,,,,,0.9425,0.5201,, +llama2:13b-chat,all_20k_uniform_2choice,entropy,all_20k_uniform_2choice,train,16400,,,,,,,,0.3876,0.0694 +llama2:13b-chat,all_20k_uniform_2choice,entropy,all_20k_uniform_2choice,validation,2000,,,,,,,,0.624,0.0786 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/metrics.pkl b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/metrics.pkl new file mode 100644 index 0000000000000000000000000000000000000000..fc94e77e2b0de22c38047925588e865d76c31da0 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/metrics.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c00627f03fa8bd0be4e55ec186f60a082d4d28770c2fd3a39760e802b732559c +size 16434 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/abstract_algebra/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/abstract_algebra/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..2595208da9adce98df6338ec1fc5c63f04c4ab9b --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/abstract_algebra/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87dc37bde5866b5485dc34914d375f8ce0449a4bb1a796e236111b8ac7976411 +size 2696 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/anatomy/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/anatomy/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..28614e4bf653bdf76d3bad27730f5841915e3887 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/anatomy/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4592496399bdea0453b1c6376e32035f42d7ec2cc9c73445c679db89ed44f48 +size 3144 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/astronomy/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/astronomy/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..77eb10625ade33b9eec36aae28f922fb3f710618 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/astronomy/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd73a10f23c310fbd05e7d3dc5ed1acad4e384f20553bacd2f910f8ef36314f1 +size 3336 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/business_ethics/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/business_ethics/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..13d8326a9c77568c5b6827494737f86d78688076 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/business_ethics/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf6adb48b1e63660c5fe303cffcad8449124480eda4f03d4a7562fc24e844302 +size 2696 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..4d5f8be210dd69c8387705d6d05f8af9e7784cee --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97739fcad2bd3a805d0d1ecba1b4b8fb5ce4c4c76aabc6c9a37f2f4d0f8b423b +size 169992 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/clinical_knowledge/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/clinical_knowledge/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..b098d3e76df26e7e02e941aa8561c6910d795138 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/clinical_knowledge/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f74e5eedaae6587d3fb8ab16580f383dc3c47aa4d97ebfb12cc5b408579dac5e +size 4680 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/college_biology/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/college_biology/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..a6fdaf40941f3646b84648411ebdcf2400182550 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/college_biology/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:917026c7ffa6fb5b5dfcc40b9f1f2eeb25077681d2ea82ebd94b451f9df43bac +size 3272 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/college_chemistry/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/college_chemistry/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..15ad6310b647afc0340c83e66070cd199e32c23e --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/college_chemistry/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bba67daae2e2ec7deae3484d25caa435ccd8b5aa19fc737912c2bbad9bf6435d +size 2696 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/college_computer_science/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/college_computer_science/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..c0b26d5585b6ad1e92658f45532b9294a3f4f3fc --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/college_computer_science/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29715a9b84fadb26bebcbc4b6de3ebc80962c76eb18540262a284be1db7a0432 +size 2696 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/college_mathematics/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/college_mathematics/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..e2c4ccc897779988479c262f84a4295a00e2fdfc --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/college_mathematics/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c08c73c6936d6cd979d1dfec4f7db4559977b95f75209b12bcb8a083387fb865 +size 2696 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/college_medicine/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/college_medicine/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..3b40db8e4c4c0699a2c74b03707a81733c8db473 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/college_medicine/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35f6a3fda868509544b7299e0d5da3afcffd7fc68b02b7599ac23d72073a0067 +size 3528 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/college_physics/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/college_physics/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..4d201581c4d0ce0d40c41155a2054c70ee4b35b5 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/college_physics/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07ea1f0946893a275c06c02267194903812f0343e04bbe6fd0219213709389b2 +size 2696 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/computer_security/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/computer_security/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..f94461a3c5c038626904226f5359a1ca96112ecd --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/computer_security/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9bd91180c72b4a6867c46cb69cce759038624480a743547c2ae37ffed4f75a0 +size 2696 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/conceptual_physics/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/conceptual_physics/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..1d878f262f046fba2002a93f2b12530ba2f9ee86 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/conceptual_physics/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31526f17a11ffee6251d563bb8fd70dd103980d1efd8eb9aef3baea3ce779f11 +size 4296 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/econometrics/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/econometrics/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..f070dfcfd8dc3b230287add3faac71d990a2e5e9 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/econometrics/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ab851a901babae06fb2e5769a14a9a348a88ac0719ad201e968f31ccc1438e2 +size 2888 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/electrical_engineering/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/electrical_engineering/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..ffd6f4cf9364f95d2ae0362ac01deab021145920 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/electrical_engineering/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6d6757f68e0085a6bc7a1291b321b40e8a00d74c770f766116afc09936d8bcd +size 3272 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/elementary_mathematics/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/elementary_mathematics/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..bcc7edae384a37bc43410707f868364323b509d1 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/elementary_mathematics/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a67e95444862824c2fe1523f06c1f57021d5fa0b4656fd7f21624305d698cbd8 +size 6024 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/formal_logic/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/formal_logic/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..77ff12a1f81c8ef68e04e137974866ac37a8281a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/formal_logic/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2df0f1e65fb4ccf305498696d444da6430ff809121c058206daa071f80517e1 +size 2952 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/global_facts/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/global_facts/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..50c0c425bb8165cc8ad41451c497c184ce6e3d58 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/global_facts/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6dc5153a9b6400372ea9dcf3b91bfdbc4badf16be04890f82c9d02768a780139 +size 2696 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_biology/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_biology/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..cd7f6e422c5330604f4827675bd1446882b69c4a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_biology/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:738c9bded47d9e8849fc2c264e301459884daa553c23b678e5d887241b424c8d +size 5192 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_chemistry/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_chemistry/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..2a4daad8deeb1f87d0313c62d97b4c7db12cefe6 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_chemistry/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e22c6b641873cf6e73084f016b143832523092dba59c39b05013b8ef8ac1774 +size 3912 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_computer_science/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_computer_science/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..878fcd0885af2109feaa3db36ba0512e20dc98e2 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_computer_science/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e92444be8625f730d37c723c872df7388982d351e3cb5ef54e3132f5e8f99dbc +size 2696 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_european_history/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_european_history/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..b170ae961a4630332e0d798506a7bc9d00dc0a63 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_european_history/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfe40fd53827840b7c2595cf07a789b5e0c6af71c05af03d273535cc620dbf8c +size 3464 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_geography/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_geography/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..8b31e55dea9ae768537a6f030cecea116c9b800f --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_geography/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:123009f94874485a5d3d2d5d28d9fbd4525d3e23167de79e9dc7e55601a3baa4 +size 3848 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_government_and_politics/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_government_and_politics/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..bda942f90f00e3df8702a36fff8fc29d44f9fd5f --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_government_and_politics/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dc4e53078048d5d0e6f0d97746177ee3b41c8d61e1de5ee353b50bf048ce8ed +size 3848 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_macroeconomics/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_macroeconomics/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..94ed31c8bdb6d0cadb2bdec58becd10d3fc24dd0 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_macroeconomics/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc954fb0ae0125f9e53210dcf6aa9588a6faaf3d2f578e4cd0996f979ceb39bd +size 6152 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_mathematics/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_mathematics/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..00155e7b8eac099cd591019269ea27fc5f834066 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_mathematics/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73aa5d0fbc7ee3bfabe074ccc3098b2c910458740a27bce7830e7a75e574e025 +size 4680 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_microeconomics/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_microeconomics/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..32dc0b1df5f24775e9fb91bff17f3016e7e002cc --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_microeconomics/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aab98830d1b94a5b0d22f7c6cd4450a11a3f9a97876efe0c178fc632db843536 +size 4296 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_physics/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_physics/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..cd8bd14b4aeca05aceb20a22992f2f8fb016c354 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_physics/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f839e3534d215c73ab1d96ec64089668da50b71fad93fb82e967dce60e5e5a27 +size 3336 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_psychology/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_psychology/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..d9f01888e0010617dee24ed721920fb5c442a415 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_psychology/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c1b95441337fcfa34c0df35b0b4ffbfd00275998076dbce2b5e9b0b5f5e72d6 +size 8072 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_statistics/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_statistics/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..320809cf55773e23d7c8e9a3db444a8d09abffda --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_statistics/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec0d35503b2a97289200556ced0d00e31b564c2487c0a2151caf9d17519eeb8f +size 4104 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_us_history/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_us_history/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..63fb1bb35ea8b3bb44ed624487056ce38549971a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_us_history/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca750468ce202322972fd5ff4b042012f48a56b43fb3f533cd3660bc91b81aa7 +size 3912 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_world_history/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_world_history/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..08797cdcbd2334a72973a6d293f1b7fef68c50a2 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/high_school_world_history/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c79c8fa950cfedd9bb328db0ea75a56222daa8cc4a4f103b49028edfa06032fc +size 4296 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/human_aging/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/human_aging/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..06ca502456daa1b619e14ea40fd35e8606aec20c --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/human_aging/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce14f5f15cfffb0f3907d0240b1bb06cf077ff23ab6cbad6ef2ebcd44d5a7085 +size 4232 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/human_sexuality/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/human_sexuality/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..b924b5f21f17ae31ff5f14721bfc2956fe9707cc --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/human_sexuality/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:910c9b1df419cc05a369f71264dd620b029810525256f23cc5949a2edf6457de +size 3080 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/international_law/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/international_law/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..dc2d47c3f01737c82f4d42cb91a2377e9f3ded21 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/international_law/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6941627d644174d7b544b74feed88ff96cad3b19ed299dac66ea401abc61a1b +size 2952 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/jurisprudence/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/jurisprudence/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..f1608a950236c0a23fa6851fb7cbb3d9338a9465 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/jurisprudence/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03355d21dde566f2ffba12b07228a0ff2e79c8ba943bbb0ff6b61c7b85b4b627 +size 2760 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/logical_fallacies/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/logical_fallacies/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..887436868815895a78c6969c17266b18b707761e --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/logical_fallacies/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d7916728a4c559035780b918d269275b09af2530391e23de98300a3c62e8cd1 +size 3464 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/machine_learning/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/machine_learning/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..ec9f2e693461d6f24badad2705438a683bb43726 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/machine_learning/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06af6d411b6c4e89a3dc30d4b4a5c75c9b8e16e6eed5957b6d7147b4f23581ba +size 2888 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/management/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/management/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..377d705fc4182695969197b6fe5648220ab6f272 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/management/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8b2be6ca7b9ba67d574bccc21a4846244cf8924ab9aa2e94132cc90ba86a16b +size 2760 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/marketing/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/marketing/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..529933224ce38f20eaa10acd8c97e3ec823190cf --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/marketing/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ad2ff25f9f14cf08bad217c7bb039a08d323bfcde730c4ee0dde2b4e3a59b30 +size 4296 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/medical_genetics/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/medical_genetics/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..9697e8122637448c7e60377eb8cbdecb170b4c54 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/medical_genetics/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da37f8a98ab714620f7b7af98c177cdbb394cfbb6e9fa5fab893bd1cbccb2110 +size 2696 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/miscellaneous/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/miscellaneous/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..41b1e3ab5b0fa12c7234211690d2a7be2dc66782 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/miscellaneous/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:532d0896829fab2a9abf160191cfc5038f271b2ef880d7b818d11fbb8657a7a8 +size 10952 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/moral_disputes/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/moral_disputes/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..e9c0644fe523e0489b9f9c5fb12788b5ad2ffd51 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/moral_disputes/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2efc4f51417997c89b9d709b313acfb04fcf0b6544c8bb40204aeb869f23646 +size 5640 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/moral_scenarios/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/moral_scenarios/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..ef11cf90892d54006481a884b8fa90f8acbedb09 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/moral_scenarios/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a26bfb373dba83132599bea1334df069b73ed33a03477c1c940d130a1a67a37 +size 12296 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/nutrition/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/nutrition/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..40984137a0ae854f0a991ac3a2cfa8eb72259eae --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/nutrition/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b68170a1feff09f8a065996b8ab7d9f025770dc4df1e5560ab24c6f76f1635c +size 5192 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/philosophy/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/philosophy/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..744be2206c43dbfea566fa71175147c9c08333e5 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/philosophy/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be5efe3b5a46dd3a540a78f3ca266cd2ccba8f254087c92cd4f50217a85c2c49 +size 5256 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/prehistory/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/prehistory/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..7488f3b03d577b5789a894fbde1fd64c310d951e --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/prehistory/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e02961249ecb78e5d2ba7f10e2d4996fbedb93f995a307b3303f5d4c9707b71 +size 5384 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/professional_accounting/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/professional_accounting/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..38a1161a73bf9bc9bcd7275a606722a6ed17cefc --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/professional_accounting/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a177c002bce11003847fc487fb9024ad06991c5022ded09f1dd1e35918adbd71 +size 4872 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/professional_law/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/professional_law/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..343ff1ed50941f54062462b4dfd917c9e5eec4e8 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/professional_law/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:773ebff86f354cba48946fec3f578eb42002a6a329252b165194fb7f5fed48c7 +size 19848 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/professional_medicine/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/professional_medicine/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..96084cfa82ba4b5ad1a9022f65d750abe20c124d --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/professional_medicine/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0176fa4c08781e88738df2d9db62871fbd93418e70a0c2d74f1fa8cfe412c62 +size 4808 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/professional_psychology/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/professional_psychology/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..631439b9e820fc9d52fc7af6ea9ead063268284e --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/professional_psychology/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c939030e25c4bf000791330f088b9bc2b3aa155ef5d9598ba181ac5f7938443 +size 8840 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/public_relations/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/public_relations/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..394bb6c4c93c75d5f1825f98bef6579ea92b03a3 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/public_relations/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9dc2cd2684508153a42eba1b7afaca5bdaf54149f5a95dc9979c38af20b23fcb +size 2760 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/security_studies/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/security_studies/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..8164b10b28ea141276381080f0e0674ec990a7f7 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/security_studies/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd5e82e81c8858b8c4093e0e7be525b30a608db9c90b1d454d82fc815b0b2a17 +size 4424 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/sociology/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/sociology/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..6edb906d614192e2aa34b2ea1eeb84a404873ca0 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/sociology/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8952026456522abdc36a4b7eae7f109674b809b464b657bdb4de00b58792ac5b +size 3912 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/us_foreign_policy/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/us_foreign_policy/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..42ed599d32893e2cd03d48f7073cb3f5eac5f1ef --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/us_foreign_policy/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0b429a1853259e2615e6520e983482e37f1a2cd9aead7eed3bdf08749d6d87f +size 2696 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/virology/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/virology/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..4ac0892c5f234c9b4be21f6d446cf88971dc5a51 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/virology/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b74e0c2b32f32b5ea44a015ac4183cf2b1bf33315173d95996087e8b113e9feb +size 3464 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/world_religions/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/world_religions/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..e57c9e9a63efc5d53898160047ace1e87859395b --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/test/world_religions/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9b19f644ec3ee5df226c8032a8fb92f57045cce0e000dd35e3499f21559c4f0 +size 3528 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/abstract_algebra/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/abstract_algebra/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..97a2a6ce45c07d4816675d759b96d9cebabb6005 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/abstract_algebra/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b610cd0b120852137801578a027aa42ae0491f376702656f6eddef9d408a21d6 +size 1608 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/anatomy/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/anatomy/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..af2398e3e9b258b3fa02f57367ad1891ef792f9a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/anatomy/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d1eaffa13ad384a9b390b9c9c05797d8dba3c952ee8c24857136b52c2043f38 +size 1608 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/astronomy/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/astronomy/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..6dd13cea102bc1bd4118fd8f9594d1110d3d8ee5 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/astronomy/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ec4b2094d6bbba792428736bca6c276e7ffff035f257a9ecc796a3f95bde64c +size 1736 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/business_ethics/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/business_ethics/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..b80e0a263dda073943f726515c570c28ce3a151c --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/business_ethics/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:435ccbc295aca40573307fae68bb7ea7cea1c68e369ecde4d80d768bd08399d4 +size 1608 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..643377952680d4e3fdb58223b1c9ca037cf0d861 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c57c1dcb83cc279c0013ccdfc4d874912436b0848ce40265b36623ab1ff4700 +size 19848 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/clinical_knowledge/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/clinical_knowledge/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..2cc0beeb66ea153d3a88b3fed4fa6d2dc75af67b --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/clinical_knowledge/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2a44946cb0efcdbc447d47b3c53ca3c4d953f1648bd9f2fa6bd6b4262ddcb6c +size 1800 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/college_biology/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/college_biology/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..1f265d348caa43329139a2bfcfa33e93f0423ec2 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/college_biology/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab90724a0080c6f63a33356e4ce4b643f0d655526d423c009bbc5de8683526f9 +size 1736 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/college_chemistry/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/college_chemistry/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..73f6fc89578f047365afc83c02ed36a2f4de6a32 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/college_chemistry/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a4101a3454061460afe92e6506ec165005cffdcfe2531b0712443ffc7803c5c +size 1608 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/college_computer_science/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/college_computer_science/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..7be153bd9db73798ace2ccca1ac6d8ddcafbe1b3 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/college_computer_science/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfc560d9e21e65493c2999b59b474cb2e15ec2c7785bce62b150c3837ddc2d0c +size 1608 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/college_mathematics/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/college_mathematics/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..fb8ab463e8846c1708f89f353ff9093ca71da69f --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/college_mathematics/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2fd754c62bdfb0c81ec591c71265b6072d24eed3e540152908aaca6ca320538 +size 1608 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/college_medicine/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/college_medicine/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..2e790fe8ddc9459ceca9049a9cd7ecd7c962ff2c --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/college_medicine/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32441a2f1ee66f464ef71efcfe94ca12a5f6d1fcf1f8a0f0badb9d1630cf547a +size 1736 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/college_physics/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/college_physics/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..1677b0afacd88003b690fecb77995a7bd545d499 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/college_physics/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fac153001a762352f1341ea33c8ef8d037627aeeb9f86e667fedc6af5b97025 +size 1608 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/computer_security/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/computer_security/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..78a461201c6052e9f8db52f7a48111b11e585277 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/computer_security/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c9c79bd8b323ef18d155b67997e9728b00c14169f025f9d4afb5505482be31c +size 1608 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/conceptual_physics/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/conceptual_physics/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..e96b2c011ce700a055533cf320f3222d3bf55ed6 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/conceptual_physics/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ed2ae0e3e5d033afd424d23d52c8acec8d10fe0c06b90b9cfaf0073780d504e +size 1800 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/econometrics/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/econometrics/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..2def162bdab2aa0e403b41ef60d1f592c356a41b --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/econometrics/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd3e31c3709114caa156cff2cab69dfd6df77eb0933cc417e57b3f0a4361e9de +size 1608 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/electrical_engineering/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/electrical_engineering/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..a7b18c977c3c60da2c48922b6c773d49552a881d --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/electrical_engineering/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0cbee79ed5dfb7fcd5cf132c1469fbf7932916e115b226d23a8cb751fbb805f +size 1736 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/elementary_mathematics/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/elementary_mathematics/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..e87d45e06135a1ac077cea0272149f439662b5f8 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/elementary_mathematics/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58d5b87644e405d50ef892c1fc36825bd029df042f4d5bb9c6a39ece7757fc3d +size 1992 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/formal_logic/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/formal_logic/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..bd5062b355f91eb41852c35dc077b71fe5c24964 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/formal_logic/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e53b9053581d9b0792176a2b61606ec6986dfce152b1bc3cd1bf8563667f0376 +size 1608 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/global_facts/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/global_facts/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..0a906120a10e570fae3eaf2c81027785e772c345 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/global_facts/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b46791b6f56c3b1f72e59e76076440c38412cec05eb0ddc4a5a4698b840d008 +size 1608 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_biology/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_biology/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..26a61b2d9cbcf0d48c74a3f5d4c1bd020507645e --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_biology/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1cc79615c479aa7ccde42860b8121e472912666c0f507ffd3da69b9b573ff113 +size 1928 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_chemistry/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_chemistry/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..2263ff611460b30474dc474abac723a7a45faaaf --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_chemistry/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1000dac42dec8c70f03828d13f6e370eca41cdc449f899fe7287bdf10b1502b5 +size 1736 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_computer_science/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_computer_science/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..7fc081378123f06596058667a8d0440e287115f0 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_computer_science/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfe8b8e3d192f37ce80fcf9cfe7c8598d2d4cba47e98337d7a1c78c92ebfd0a4 +size 1608 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_european_history/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_european_history/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..e6d5b8ba38993b4ebbf62369546f0502167aa4de --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_european_history/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9709c834eb487e68dd5270ba56ddf89810c978263dbe9b2f1885463ee821e6b5 +size 1736 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_geography/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_geography/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..056a93f7be69c8dd6e20a23357784db558ee88e4 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_geography/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a6afc34c5e6389c63d8d3170080dc75e8d50473034a7a01519336ed2d57ab61 +size 1736 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_government_and_politics/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_government_and_politics/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..7915ac99375fcbaecf179e2228259d2647466dbc --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_government_and_politics/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:536b769a47524f0e0b3828f3fc55cd15e3d2ef19ef457c23190fb38351319d0f +size 1736 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_macroeconomics/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_macroeconomics/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..5f42471a9e9f32615faae565d10a04fad4f0761d --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_macroeconomics/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19221ae902584dcb741dcaa656aefccd5de2551c3b4e43a4843ee2f120796015 +size 1992 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_mathematics/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_mathematics/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..0b1f051ff275a00ccf9eaaf03bede3c6e6008a14 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_mathematics/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ce0ca7feb835248ecb33e84e42750382b638e45f9639dd0b959f5db6f607292 +size 1800 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_microeconomics/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_microeconomics/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..002be3f4ca3c8b0e9698773c71c1e7b34fdb205f --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_microeconomics/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2472910fa5a17a38cb03c444412bb78ed7f88cc8678173f3b75cb7d4bc93376e +size 1800 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_physics/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_physics/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..e7002c44893471605892f0534485a882b3de565f --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_physics/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c459099f2a6eccc363e4c962b1869460eb58796ca6c4083ddc116847f467019 +size 1736 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_psychology/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_psychology/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..1e41277e93aa5ea3eea692dfe8f30b6e0ccf54d6 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_psychology/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfdd6e01a0da5a2800dfa98b11ec59e6228165a4195d9b19a0a6aff11adaaa2f +size 2184 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_statistics/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_statistics/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..6683e0e6dd62b3d5113079200a5c26d8be615c44 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_statistics/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de761e62120bb91dcbfba4e9168b26bff989f678bfefaa01c53ad4f87e324763 +size 1800 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_us_history/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_us_history/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..fab3d375dd004308c2dacb7da3e365cb2470a374 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_us_history/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8dd56ddbbf26ad90675edaebd6e1135de35439b729b1deafdff0be24041f2e4f +size 1736 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_world_history/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_world_history/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..e96308f3c0f54e2842398b1a2a3774d33354744c --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/high_school_world_history/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9be2993772553f2bd937f7659178d144a151ab70bc89fab5f24a7bcdc4d9a36 +size 1800 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/human_aging/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/human_aging/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..eaa1121637f417c383c1ec0aa264ec9eb2eefe10 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/human_aging/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d37fc5b80f43223094e019a82de8901464563454ce695e378394e8de7fce647a +size 1800 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/human_sexuality/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/human_sexuality/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..0cb2756507ef85038ac86f424cdab52827954d72 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/human_sexuality/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9dd389e4cddb8700938ffef766f937ca7cd8397e0386c24ddf4de61a146def85 +size 1608 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/international_law/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/international_law/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..f1a69c63745d8b09af5abb2f530b4304aed22abc --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/international_law/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00e00adf4f53a81a4fa98afc68da58be174532c8516ec3fa705e751b91433c14 +size 1608 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/jurisprudence/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/jurisprudence/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..03125763b61cae1d64b7d580d59601ba173d09c0 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/jurisprudence/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80969f4d121d71eb0b39e3cf44a8bd0bbacaf2d0736d12750d4ef32b01912122 +size 1608 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/logical_fallacies/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/logical_fallacies/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..e97af6c3cd7664c1dc3751ade74d897771889a16 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/logical_fallacies/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f4e38297ccde4cbb0cb6e4dbff877caa8cf83fb63ce0dd22cc0d463890aa85c +size 1736 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/machine_learning/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/machine_learning/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..f41c1390abdfb11883c64496a3bf93dc689f6ab3 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/machine_learning/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cb2c121d4a388fcc7b25a229ba7fb21c23d6516709de3f80bcdbf5b0abe655b +size 1608 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/management/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/management/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..7f9970cea53291e2ae01d3f2c475b45c21bf4df8 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/management/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1eb912e2a5e5fddd8975881d1430b44028cd0841ae50a452a1ced7de4bb060c +size 1608 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/marketing/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/marketing/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..28195b81309ee81442b05d9cc837fbeb18c1b6d5 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/marketing/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08909c9aa3e5b140c7737979f1a1c32414b8945199b45cd7559cd6b75096d456 +size 1800 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/medical_genetics/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/medical_genetics/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..0ab42b8960d77e42744af624daa7e13f02c28b97 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/medical_genetics/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f03b69b7cbbb79d8d8f8a77881e1fe4bcfe4c85b0b4ca0eade2e9e963a3a9029 +size 1608 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/miscellaneous/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/miscellaneous/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..a36aa0ca907c415f13bf80e3295214155e919670 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/miscellaneous/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0dbe22fd7154524d4b8b8eb330f568397e9ed3372150d4371a336407abc7303 +size 2504 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/moral_disputes/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/moral_disputes/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..94b8a140815eadb17a25f23511ee7ddf55b0cd42 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/moral_disputes/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0459888d4bc2322cd8444f7bf1c781fcdfb97093d82a9dad31c7bf071dbe5404 +size 1928 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/moral_scenarios/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/moral_scenarios/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..f8a535cdf664bee70a66508ab7870aa9a2073956 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/moral_scenarios/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a03f21c4bfb6365fd49c45fb0a55d7df09377099bf1c03b16e41339b1b21cfe +size 2696 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/nutrition/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/nutrition/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..559089ff47ea2bcf55a3497b48fc2de1b511d9ed --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/nutrition/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cf352026cf3743d4f4865ec793f2d3a929b5e9dd144ab5526d64ae10419539c +size 1928 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/philosophy/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/philosophy/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..47ca06e441beb518e7c49c85acd1787fe3b1d28b --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/philosophy/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd3a56bf1c2853da8bf48534bb95b94556d4e4d856c4275ca28aad48c29ae74b +size 1928 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/prehistory/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/prehistory/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..a49db510e512b97eb0a63a708c660f7a7d8406bd --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/prehistory/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b39bc7e4869e87f1f1fab2696477e5ea0f0aa1f07c4cd077620ad210240c0b04 +size 1928 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/prehistory/test/prehistory/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/prehistory/test/prehistory/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..b8490a51331495f5992299bcd8297a4af17da4ff --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/prehistory/test/prehistory/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c8b9f874971ef319ff5b777e7c81e0f592348ddc42e0c730a886d7df5e5cc5f +size 5384 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/professional_accounting/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/professional_accounting/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..0e1e55dbb376aff0d72ba2403ff13b95859aed2c --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/professional_accounting/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2528efad1f30ff855700a6e0c0e9f07aae2e18c2e5e28b1018577baa3ed3ddcd +size 1928 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/professional_accounting/test/professional_accounting/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/professional_accounting/test/professional_accounting/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..27189b18f11b3ade7d54bd5d1b9ed2290adc7100 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/professional_accounting/test/professional_accounting/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7b8d30ba992e5ce537a9c4346bda2f91729c9c0f595d886887f439f7a80d3ac +size 4872 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/professional_law/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/professional_law/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..998fbb30695f50006a3a441c2992ea659cdb5c96 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/professional_law/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9eb646660a4e390c01a52e66a376e0131a04e94bdbc430c5f01b6aa44f19a33 +size 3528 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/professional_law/test/professional_law/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/professional_law/test/professional_law/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..9e0dac4e9265d453e29e88f9f4f88495d43e353b --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/professional_law/test/professional_law/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abce89698c6529857ea034545fa3446edb640d8b8cdb157853254e1677617fe2 +size 19848 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/professional_medicine/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/professional_medicine/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..7fe4dadc74f010627adca821ca79dfb62d2a1d39 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/professional_medicine/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c695b226088ab10852dc7cbad69e6afdfda58c76a4df28e54707ce5e393a862 +size 1928 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/professional_medicine/test/professional_medicine/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/professional_medicine/test/professional_medicine/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..c761bb0da5b0579f7f54961fe4315a8bd0ed9398 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/professional_medicine/test/professional_medicine/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d83e654d3c7909db7a06cc7b04babd0125fb9c4ccee3cf0b9bce65274607a2b1 +size 4808 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/professional_psychology/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/professional_psychology/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..e83274f8b3d8c19a5a9d24ad6a8e37673f061cfa --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/professional_psychology/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2051429f75d38818bcf150e42ba405c8424889e3a596cbb41a6bf8b10a55a95 +size 2312 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/professional_psychology/test/professional_psychology/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/professional_psychology/test/professional_psychology/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..bcac5a4063ae6215ea1e45c7a843fc7f65aab16f --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/professional_psychology/test/professional_psychology/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2543de5f573eeb91e9a6a92aaaa94e905cb28ebf205f22ee2cc5c64893e577b2 +size 8840 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/public_relations/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/public_relations/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..f1ca693e94ee8375da42c889fdd4762efe21b9d0 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/public_relations/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:739647381e3ebc211fba867814180028e472958cf3acfa256db341446d9acc61 +size 1608 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/public_relations/test/public_relations/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/public_relations/test/public_relations/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..c4eeffc0e90108e6cf5aee2bab33c39bede3328c --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/public_relations/test/public_relations/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:848afcda94bc2eddcb19e4333e80e20427c8374bbff06c7a6c29b21fa8ad45ba +size 2760 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/security_studies/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/security_studies/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..ca0d6a00c608dd4b3f56486d8ff358b08ae5056e --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/security_studies/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0475d4ea8880949dd7f902aabc19e1e80fcbb8cfebc06df74d6863547e6cce41 +size 1800 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/security_studies/test/security_studies/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/security_studies/test/security_studies/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..8160a8e28dde3ea04d249553443e1b0db2f0c758 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/security_studies/test/security_studies/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c89811b8d69bc00c1eb205f046e3b4a034bf2ed326db9108b995fc2128d29ec9 +size 4424 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/sociology/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/sociology/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..8b665a6a27bed28eeeaff8b396b5024f486a2913 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/sociology/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48f1c51fc49673eab2856c16ae9e3304812e04f5f99e63d0fa50d40477fb4d66 +size 1736 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/sociology/test/sociology/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/sociology/test/sociology/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..8cba449b887a0cb8d40b4a358e40a74c7d90eb68 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/sociology/test/sociology/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdaaa6821fe04e07c4ac0b373a1aa5a261f157254a5d526b27a693bf925995fe +size 3912 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/us_foreign_policy/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/us_foreign_policy/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..5ae655b30861f2c9b1bb2a6121a27ec047ab49cf --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/us_foreign_policy/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83498f95a1740a8f1ca3f7a18c263b8a29d9fcf628f496ce8551dd3526ea594e +size 1608 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/us_foreign_policy/test/us_foreign_policy/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/us_foreign_policy/test/us_foreign_policy/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..34c2a916630e330faf4c56994a0a24b6d8f89af6 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/us_foreign_policy/test/us_foreign_policy/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86e98d104e545f77936d37899a6a9eaa90df2d251cf7b15c582c8d570ee4a475 +size 2696 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/virology/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/virology/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..13827cdbd462fe88c783606eb2e27857c5562f9b --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/virology/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:662b87c92595738347efd494d5017c0cfcd878588673f233d04c84b44f32d906 +size 1736 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/virology/test/virology/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/virology/test/virology/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..3c486048436ccd2c45e4398b6f42335150a107fe --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/virology/test/virology/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81c94b4f80c608a1998a96a89dfc4f92305755c4df037f315554c913a39cb87b +size 3464 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/world_religions/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/world_religions/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..94e0076008e6ec07b87a96fdf87a6e3d42cfd170 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/world_religions/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:beef9d837dff3ee2e4a71d1c3a3ec719a2e18e48889b79c1c87f956f13a9bafa +size 1736 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/world_religions/test/world_religions/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/world_religions/test/world_religions/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..69cf89ec25c9d87b89acb4e0e957451f20e39882 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/eval_dataset-mmlu_all/validation/world_religions/test/world_religions/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bb7c6d5ec15fdd8f22e52a343d42560ac1beb5703d02c658d381a0fe2583393 +size 3528 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/metrics.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/metrics.json new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/train/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/train/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..c229d4f19eab75b023e2a475b002b0f4c7121338 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/train/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1dd3dfabed061fbf1a88f7ab6abb297c5ce7785a15945e489b9bb78e99ca8551 +size 223880 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/validation/classifier_data.bin b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/validation/classifier_data.bin new file mode 100644 index 0000000000000000000000000000000000000000..1aa804576cceb627d906601222b281e7c7e57b27 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/validation/classifier_data.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8eafadccdc5b94fb1abd5ac46505bb4b98236dd0921ab7de588478584173818 +size 25544 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..bedf327a83f830e5e5df61f3a524fa1fda025272 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/debug-internal.log @@ -0,0 +1,7 @@ +{"time":"2025-03-13T00:50:38.691400557-05:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch2/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250313_005038-73nxdj42/logs/debug-core.log"} +{"time":"2025-03-13T00:50:38.825268799-05:00","level":"INFO","msg":"created new stream","id":"73nxdj42"} +{"time":"2025-03-13T00:50:38.825325589-05:00","level":"INFO","msg":"stream: started","id":"73nxdj42"} +{"time":"2025-03-13T00:50:38.825351869-05:00","level":"INFO","msg":"writer: Do: started","stream_id":"73nxdj42"} +{"time":"2025-03-13T00:50:38.825374989-05:00","level":"INFO","msg":"handler: started","stream_id":"73nxdj42"} +{"time":"2025-03-13T00:50:38.825367379-05:00","level":"INFO","msg":"sender: started","stream_id":"73nxdj42"} +{"time":"2025-03-13T00:50:38.8342329-05:00","level":"INFO","msg":"Starting system monitor"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250308_234315-132ixfh6/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250308_234315-132ixfh6/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250308_234315-132ixfh6/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250308_234315-132ixfh6/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250308_234315-132ixfh6/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250308_234315-132ixfh6/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250308_234315-132ixfh6/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f6e325afb3f57570ea3c8233381acf1c54772951 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250308_234315-132ixfh6/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T05:43:16.341758Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/correctness", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/correctness/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128073453568" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250308_234315-132ixfh6/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250308_234315-132ixfh6/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..dfe5b01f5a63012d4030878ed40649a096929de7 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250308_234315-132ixfh6/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-08T23:43:16.213595997-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmp6w8bx43z/port-2797240.txt","pid":2797240,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T23:43:16.216636388-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2797240} +{"time":"2025-03-08T23:43:16.216620708-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":43541,"Zone":""}} +{"time":"2025-03-08T23:43:16.301319001-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:57634"} +{"time":"2025-03-08T23:43:16.343395834-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"132ixfh6","id":"127.0.0.1:57634"} +{"time":"2025-03-08T23:43:16.74038102-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"132ixfh6","id":"127.0.0.1:57634"} +{"time":"2025-03-09T00:13:58.903476538-06:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:57634"} +{"time":"2025-03-09T00:13:58.903918664-06:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-09T00:13:58.903902904-06:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:57634"} +{"time":"2025-03-09T00:13:58.904076953-06:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:57634"} +{"time":"2025-03-09T00:13:58.905520831-06:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:57634"} +{"time":"2025-03-09T00:13:58.905530021-06:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:57634"} +{"time":"2025-03-09T00:13:58.905536001-06:00","level":"INFO","msg":"server is closed"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250308_234315-132ixfh6/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250308_234315-132ixfh6/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a0b74380e84b16343b1ce9ba4f996e54220c29f6 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250308_234315-132ixfh6/logs/debug-internal.log @@ -0,0 +1,14 @@ +{"time":"2025-03-08T23:43:16.612834539-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250308_234315-132ixfh6/logs/debug-core.log"} +{"time":"2025-03-08T23:43:16.740257731-06:00","level":"INFO","msg":"created new stream","id":"132ixfh6"} +{"time":"2025-03-08T23:43:16.740358761-06:00","level":"INFO","msg":"stream: started","id":"132ixfh6"} +{"time":"2025-03-08T23:43:16.74037487-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"132ixfh6"} +{"time":"2025-03-08T23:43:16.74039271-06:00","level":"INFO","msg":"handler: started","stream_id":"132ixfh6"} +{"time":"2025-03-08T23:43:16.74050418-06:00","level":"INFO","msg":"sender: started","stream_id":"132ixfh6"} +{"time":"2025-03-08T23:43:16.753675659-06:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-09T00:13:58.903930074-06:00","level":"INFO","msg":"stream: closing","id":"132ixfh6"} +{"time":"2025-03-09T00:13:58.904030403-06:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-09T00:13:58.904732468-06:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-09T00:13:58.904790227-06:00","level":"INFO","msg":"handler: closed","stream_id":"132ixfh6"} +{"time":"2025-03-09T00:13:58.904799857-06:00","level":"INFO","msg":"writer: Close: closed","stream_id":"132ixfh6"} +{"time":"2025-03-09T00:13:58.904815257-06:00","level":"INFO","msg":"sender: closed","stream_id":"132ixfh6"} +{"time":"2025-03-09T00:13:58.905461182-06:00","level":"INFO","msg":"stream: closed","id":"132ixfh6"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250308_234315-132ixfh6/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250308_234315-132ixfh6/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250308_234315-132ixfh6/run-132ixfh6.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250308_234315-132ixfh6/run-132ixfh6.wandb new file mode 100644 index 0000000000000000000000000000000000000000..07a804e34f47e003dc728e2286eb03f870c7617f --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250308_234315-132ixfh6/run-132ixfh6.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a23badbd396cda495925a7da234f9372d608e1b9919f14a08c942e87334dd64b +size 5977437 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250309_001704-olsr0k56/files/config.yaml b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250309_001704-olsr0k56/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c81dd747e2c804dfe7e470bf80f76ae7bfcc9b01 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250309_001704-olsr0k56/files/config.yaml @@ -0,0 +1,40 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + python_version: 3.12.5 + cli_version: 0.19.8 + framework: huggingface + huggingface_version: 4.49.0 + is_jupyter_run: false + is_kaggle_kernel: true + start_time: 1741501024 + t: + 1: + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 55 + - 71 + - 75 + - 95 + - 98 + - 105 + 3: + - 4 + - 13 + - 23 + - 42 + - 55 + 4: 3.12.5 + 5: 0.19.8 + 6: 4.49.0 + 8: + - 2 + - 5 + 13: linux-x86_64 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250309_001704-olsr0k56/files/media/table/metrics_0_6ef39efda9c5b8df00aa.table.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250309_001704-olsr0k56/files/media/table/metrics_0_6ef39efda9c5b8df00aa.table.json new file mode 100644 index 0000000000000000000000000000000000000000..77159ae5e473bf0346a99be068b6883fb748e509 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250309_001704-olsr0k56/files/media/table/metrics_0_6ef39efda9c5b8df00aa.table.json @@ -0,0 +1 @@ +{"columns": ["N", "correctness_acc", "correctness_auroc", "correctness_ece", "dataset", "split", "ts"], "data": [[18534, 0.7173842787742615, 0.7892882581075626, 0.06694770110802215, "offline:all_20k_uniform/choice/correctness", "train", 1809.453210788779], [2000, 0.921999990940094, 0.9786359035529648, 0.031201183140277818, "offline:all_20k_uniform/choice/correctness", "validation", 158.65735564380884]]} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250309_001704-olsr0k56/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250309_001704-olsr0k56/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..2792b13f75343aad22e7768c077396e1fe7de81d --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/correctness/wandb/offline-run-20250309_001704-olsr0k56/files/output.log @@ -0,0 +1,19 @@ +Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [00:12<00:00, 4.09s/it] + 0%| | 0/1 [00:00 + fire.Fire(main) + File "/net/scratch2/zlyu/conda/envs/312/lib/python3.12/site-packages/fire/core.py", line 143, in Fire + component_trace = _Fire(component, args, parsed_flag_args, context, name) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/net/scratch2/zlyu/conda/envs/312/lib/python3.12/site-packages/fire/core.py", line 477, in _Fire + component, remaining_args = _CallAndUpdateTrace( + ^^^^^^^^^^^^^^^^^^^^ + File "/net/scratch2/zlyu/conda/envs/312/lib/python3.12/site-packages/fire/core.py", line 693, in _CallAndUpdateTrace + component = fn(*varargs, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/llm/logging.py", line 228, in _wrapped_entrypoint + return f(**kwargs) + ^^^^^^^^^^^ + File "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", line 215, in main + wandb.save(f"{log_dir}/*", base_path=log_dir) + File "/net/scratch2/zlyu/conda/envs/312/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 401, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/net/scratch2/zlyu/conda/envs/312/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/net/scratch2/zlyu/conda/envs/312/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 1975, in save + return self._save( + ^^^^^^^^^^^ + File "/net/scratch2/zlyu/conda/envs/312/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 1997, in _save + raise ValueError("Glob may not start with '*' relative to the base path") +ValueError: Glob may not start with '*' relative to the base path +[2025-03-08 21:49:07,870] (get_model:INFO) Loaded "llama2:13b-chat". +[2025-03-08 21:49:09,011] (get_peft_model_from_checkpoint:INFO) Loaded PEFT adapter 'query' checkpoint from 'cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000' +output_size: 10 +[2025-03-08 21:49:09,012] (main:INFO) output_size: 10 +[2025-03-08 21:49:09,020] (get_model:INFO) Loaded "mlp_probability". +[2025-03-08 21:49:09,594] (get_classifier_head:INFO) Loaded classifier model checkpoint from 'cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000'. +[2025-03-08 21:49:09,679] (get_dataset:INFO) root: /net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_datasets/llama2:13b-chat +[2025-03-08 21:49:09,679] (get_offline:INFO) root: /net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_datasets/llama2:13b-chat/offline/all_20k_uniform/choice/probability +[2025-03-08 21:49:09,885] (get_offline:INFO) root: /net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_datasets/llama2:13b-chat/offline/all_20k_uniform/choice/probability +[2025-03-08 21:49:09,885] (get_offline:INFO) data_files: {'train': ['/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_datasets/llama2:13b-chat/offline/all_20k_uniform/choice/probability/train/0.csv'], 'validation': ['/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_datasets/llama2:13b-chat/offline/all_20k_uniform/choice/probability/validation/0.csv']} +[2025-03-08 21:49:12,699] (get_dataset:INFO) Loaded "offline:all_20k_uniform/choice/probability"; train (N = 18534) / validation (N = 2000) +[2025-03-08 21:50:56,232] (save_metrics_data:INFO) Metrics data saved to "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/validation/classifier_data.bin". \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_214844-tmkoufpd/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_214844-tmkoufpd/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_214844-tmkoufpd/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_214844-tmkoufpd/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_214844-tmkoufpd/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..efb409898957d57fbac85e45595f549cc1d5f0c8 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_214844-tmkoufpd/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T03:48:46.463034Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128023867392" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_214844-tmkoufpd/files/wandb-summary.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_214844-tmkoufpd/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..222a4231c6f0f09ebfa7aedd958e73d0018ac1f1 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_214844-tmkoufpd/files/wandb-summary.json @@ -0,0 +1 @@ +{"_step": 0, "metrics": {"artifact_path": "wandb-client-artifact://djqxd40trc5vs9tc00455benkl6mrjlzfp61m6bqxt6h4oup4dizkfs7cfz3vcin6f3abcn2ecx2q9ywu8ppgnm9034et7n5mucggtghcujzou8yvxe4p9ywvkcylnp7/metrics.table.json", "_latest_artifact_path": "wandb-client-artifact://ok8sf3ggw2f57m4cixs8xvgit1021fltb09mb53ea9u8g4ugdgzc5gp5ue3xf86ffz3mfpau3agf9jqytcpn85raj0mzk84h4dipt160xdgfs9j22lnh98pq0omt9e50:latest/metrics.table.json", "path": "media/table/metrics_0_dd4a98cd474835970ee2.table.json", "ncols": 8, "nrows": 1, "_type": "table-file", "sha256": "dd4a98cd474835970ee2ef4f2683fd1e6e78b0557cc69ed253a3fb3972ae827a", "size": 235}, "_timestamp": 1741492256.461733, "_runtime": 129.999268746} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_214844-tmkoufpd/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_214844-tmkoufpd/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..be4d56414589f9c6a444f0a175e24264cf9004de --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_214844-tmkoufpd/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-08T21:48:46.288888279-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmplt8pevh8/port-2630395.txt","pid":2630395,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T21:48:46.291781737-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2630395} +{"time":"2025-03-08T21:48:46.291791687-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":44575,"Zone":""}} +{"time":"2025-03-08T21:48:46.325193143-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:43704"} +{"time":"2025-03-08T21:48:46.464430124-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"tmkoufpd","id":"127.0.0.1:43704"} +{"time":"2025-03-08T21:48:48.541432589-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"tmkoufpd","id":"127.0.0.1:43704"} +{"time":"2025-03-08T21:50:57.70951984-06:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:43704"} +{"time":"2025-03-08T21:50:57.710448243-06:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-08T21:50:57.710448773-06:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:43704"} +{"time":"2025-03-08T21:50:57.710525812-06:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:43704"} +{"time":"2025-03-08T21:50:57.713233502-06:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:43704"} +{"time":"2025-03-08T21:50:57.713252272-06:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:43704"} +{"time":"2025-03-08T21:50:57.713262062-06:00","level":"INFO","msg":"server is closed"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_214844-tmkoufpd/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_214844-tmkoufpd/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..3f38598f7e8d93d8d5b07a505334b07f975076ec --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_214844-tmkoufpd/logs/debug-internal.log @@ -0,0 +1,14 @@ +{"time":"2025-03-08T21:48:48.218824962-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_214844-tmkoufpd/logs/debug-core.log"} +{"time":"2025-03-08T21:48:48.54133659-06:00","level":"INFO","msg":"created new stream","id":"tmkoufpd"} +{"time":"2025-03-08T21:48:48.541413419-06:00","level":"INFO","msg":"stream: started","id":"tmkoufpd"} +{"time":"2025-03-08T21:48:48.541439079-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"tmkoufpd"} +{"time":"2025-03-08T21:48:48.541495838-06:00","level":"INFO","msg":"handler: started","stream_id":"tmkoufpd"} +{"time":"2025-03-08T21:48:48.541573048-06:00","level":"INFO","msg":"sender: started","stream_id":"tmkoufpd"} +{"time":"2025-03-08T21:48:49.269312323-06:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-08T21:50:57.710112216-06:00","level":"INFO","msg":"stream: closing","id":"tmkoufpd"} +{"time":"2025-03-08T21:50:57.710432233-06:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-08T21:50:57.711550635-06:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-08T21:50:57.711824243-06:00","level":"INFO","msg":"handler: closed","stream_id":"tmkoufpd"} +{"time":"2025-03-08T21:50:57.711838052-06:00","level":"INFO","msg":"writer: Close: closed","stream_id":"tmkoufpd"} +{"time":"2025-03-08T21:50:57.711856252-06:00","level":"INFO","msg":"sender: closed","stream_id":"tmkoufpd"} +{"time":"2025-03-08T21:50:57.713083453-06:00","level":"INFO","msg":"stream: closed","id":"tmkoufpd"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_214844-tmkoufpd/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_214844-tmkoufpd/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_214844-tmkoufpd/run-tmkoufpd.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_214844-tmkoufpd/run-tmkoufpd.wandb new file mode 100644 index 0000000000000000000000000000000000000000..d56f6728c85d640419f9f98987d2eb2135881cca --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_214844-tmkoufpd/run-tmkoufpd.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e50ac774de3cd562d96b3fad692b4759ee8ab02c3ed06b7ad80e5dadd7879924 +size 443778 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_214844-tmkoufpd/run-tmkoufpd.wandb.synced b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_214844-tmkoufpd/run-tmkoufpd.wandb.synced new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_221649-yxmxbw23/files/media/table/metrics_0_03a1c455968f1fde02e0.table.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_221649-yxmxbw23/files/media/table/metrics_0_03a1c455968f1fde02e0.table.json new file mode 100644 index 0000000000000000000000000000000000000000..89848d8c7b7909dca5dbc740f78558fe2e718375 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_221649-yxmxbw23/files/media/table/metrics_0_03a1c455968f1fde02e0.table.json @@ -0,0 +1 @@ +{"columns": ["N", "unc_acc", "unc_auroc", "unc_ece", "acc", "dataset", "split", "ts"], "data": [[2000, 0.0, NaN, 0.5283251887857914, 0.09650000184774399, "offline:all_20k_uniform/choice/probability", "validation", 101.45142094884068]]} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_221649-yxmxbw23/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_221649-yxmxbw23/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_221649-yxmxbw23/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_221649-yxmxbw23/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_221649-yxmxbw23/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_221649-yxmxbw23/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_221649-yxmxbw23/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..cdde57aa5c8c8ab454f7a478852f28f145350895 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_221649-yxmxbw23/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T04:16:50.209437Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128034787328" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_221649-yxmxbw23/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_221649-yxmxbw23/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..d9340dac00b1219782ee789d8f02881c349e10aa --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_221649-yxmxbw23/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-08T22:16:50.0488449-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmpof3ovqf2/port-2665984.txt","pid":2665984,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T22:16:50.051793738-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2665984} +{"time":"2025-03-08T22:16:50.051789018-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":42217,"Zone":""}} +{"time":"2025-03-08T22:16:50.200890656-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:60234"} +{"time":"2025-03-08T22:16:50.214438004-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"yxmxbw23","id":"127.0.0.1:60234"} +{"time":"2025-03-08T22:16:50.37968176-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"yxmxbw23","id":"127.0.0.1:60234"} +{"time":"2025-03-08T22:18:50.581676473-06:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:60234"} +{"time":"2025-03-08T22:18:50.582274508-06:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:60234"} +{"time":"2025-03-08T22:18:50.582327198-06:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-08T22:18:50.582495377-06:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:60234"} +{"time":"2025-03-08T22:18:50.584978078-06:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:60234"} +{"time":"2025-03-08T22:18:50.585006228-06:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:60234"} +{"time":"2025-03-08T22:18:50.585020188-06:00","level":"INFO","msg":"server is closed"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_221649-yxmxbw23/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_221649-yxmxbw23/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c6ebae1e314a4a01115dfb00a37d4f7acbe06be7 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_221649-yxmxbw23/logs/debug-internal.log @@ -0,0 +1,14 @@ +{"time":"2025-03-08T22:16:50.242435704-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_221649-yxmxbw23/logs/debug-core.log"} +{"time":"2025-03-08T22:16:50.379594971-06:00","level":"INFO","msg":"created new stream","id":"yxmxbw23"} +{"time":"2025-03-08T22:16:50.37966685-06:00","level":"INFO","msg":"stream: started","id":"yxmxbw23"} +{"time":"2025-03-08T22:16:50.37969159-06:00","level":"INFO","msg":"handler: started","stream_id":"yxmxbw23"} +{"time":"2025-03-08T22:16:50.3797466-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"yxmxbw23"} +{"time":"2025-03-08T22:16:50.379932168-06:00","level":"INFO","msg":"sender: started","stream_id":"yxmxbw23"} +{"time":"2025-03-08T22:16:50.387303593-06:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-08T22:18:50.582330908-06:00","level":"INFO","msg":"stream: closing","id":"yxmxbw23"} +{"time":"2025-03-08T22:18:50.582419687-06:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-08T22:18:50.58342404-06:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-08T22:18:50.583545549-06:00","level":"INFO","msg":"handler: closed","stream_id":"yxmxbw23"} +{"time":"2025-03-08T22:18:50.583565069-06:00","level":"INFO","msg":"writer: Close: closed","stream_id":"yxmxbw23"} +{"time":"2025-03-08T22:18:50.583608038-06:00","level":"INFO","msg":"sender: closed","stream_id":"yxmxbw23"} +{"time":"2025-03-08T22:18:50.584805549-06:00","level":"INFO","msg":"stream: closed","id":"yxmxbw23"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_221649-yxmxbw23/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_221649-yxmxbw23/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_221649-yxmxbw23/run-yxmxbw23.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_221649-yxmxbw23/run-yxmxbw23.wandb new file mode 100644 index 0000000000000000000000000000000000000000..9c5f603bb60fca51fdb3f7eaeb734c18f9668c57 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_221649-yxmxbw23/run-yxmxbw23.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c53e50a5b23438219818cf5ba1e43537441bbf5fbe8143f2eac476ee55febef +size 388049 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222015-e0wgt1xk/files/config.yaml b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222015-e0wgt1xk/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4dc33b4d8cc7368b5694cdb58775ecb8ae101fc --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222015-e0wgt1xk/files/config.yaml @@ -0,0 +1,40 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + python_version: 3.12.5 + cli_version: 0.19.8 + framework: huggingface + huggingface_version: 4.49.0 + is_jupyter_run: false + is_kaggle_kernel: true + start_time: 1741494016 + t: + 1: + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 55 + - 71 + - 75 + - 95 + - 98 + - 105 + 3: + - 4 + - 13 + - 23 + - 42 + - 55 + 4: 3.12.5 + 5: 0.19.8 + 6: 4.49.0 + 8: + - 2 + - 5 + 13: linux-x86_64 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222015-e0wgt1xk/files/media/table/metrics_0_1a74e1a792703be6625b.table.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222015-e0wgt1xk/files/media/table/metrics_0_1a74e1a792703be6625b.table.json new file mode 100644 index 0000000000000000000000000000000000000000..a395dcf7255699d7e5fb17259814e7544bb4234a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222015-e0wgt1xk/files/media/table/metrics_0_1a74e1a792703be6625b.table.json @@ -0,0 +1 @@ +{"columns": ["N", "unc_acc", "unc_auroc", "unc_ece", "acc", "dataset", "split", "ts"], "data": [[2000, 0.0, NaN, 0.5283251887857914, 0.09650000184774399, "offline:all_20k_uniform/choice/probability", "validation", 101.216357502155]]} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222015-e0wgt1xk/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222015-e0wgt1xk/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222015-e0wgt1xk/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222015-e0wgt1xk/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222015-e0wgt1xk/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222015-e0wgt1xk/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222015-e0wgt1xk/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..6aaec4b3af9662eb27cb83c509eeb844799fc920 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222015-e0wgt1xk/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T04:20:16.103032Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128036311040" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222015-e0wgt1xk/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222015-e0wgt1xk/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..d45ab83e5ca3dc1c1e12184d01fa78bcada75701 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222015-e0wgt1xk/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-08T22:20:15.870324024-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmpkfacglfd/port-2672689.txt","pid":2672689,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T22:20:15.873412281-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2672689} +{"time":"2025-03-08T22:20:15.873425771-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":35741,"Zone":""}} +{"time":"2025-03-08T22:20:16.004330164-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:36364"} +{"time":"2025-03-08T22:20:16.105379502-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"e0wgt1xk","id":"127.0.0.1:36364"} +{"time":"2025-03-08T22:20:16.313986549-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"e0wgt1xk","id":"127.0.0.1:36364"} +{"time":"2025-03-08T22:22:17.572295093-06:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:36364"} +{"time":"2025-03-08T22:22:17.587800876-06:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:36364"} +{"time":"2025-03-08T22:22:17.587831246-06:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-08T22:22:17.587882586-06:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:36364"} +{"time":"2025-03-08T22:22:17.63900876-06:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:36364"} +{"time":"2025-03-08T22:22:17.6390241-06:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:36364"} +{"time":"2025-03-08T22:22:17.63903334-06:00","level":"INFO","msg":"server is closed"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222015-e0wgt1xk/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222015-e0wgt1xk/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..881b417262a07f04b016627d5a79b2240e05ec96 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222015-e0wgt1xk/logs/debug-internal.log @@ -0,0 +1,14 @@ +{"time":"2025-03-08T22:20:16.178651869-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222015-e0wgt1xk/logs/debug-core.log"} +{"time":"2025-03-08T22:20:16.313941679-06:00","level":"INFO","msg":"created new stream","id":"e0wgt1xk"} +{"time":"2025-03-08T22:20:16.313977609-06:00","level":"INFO","msg":"stream: started","id":"e0wgt1xk"} +{"time":"2025-03-08T22:20:16.314003449-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"e0wgt1xk"} +{"time":"2025-03-08T22:20:16.314053748-06:00","level":"INFO","msg":"sender: started","stream_id":"e0wgt1xk"} +{"time":"2025-03-08T22:20:16.314030729-06:00","level":"INFO","msg":"handler: started","stream_id":"e0wgt1xk"} +{"time":"2025-03-08T22:20:16.322910571-06:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-08T22:22:17.587421539-06:00","level":"INFO","msg":"stream: closing","id":"e0wgt1xk"} +{"time":"2025-03-08T22:22:17.587800406-06:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-08T22:22:17.60318262-06:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-08T22:22:17.603253639-06:00","level":"INFO","msg":"handler: closed","stream_id":"e0wgt1xk"} +{"time":"2025-03-08T22:22:17.603264139-06:00","level":"INFO","msg":"writer: Close: closed","stream_id":"e0wgt1xk"} +{"time":"2025-03-08T22:22:17.603294939-06:00","level":"INFO","msg":"sender: closed","stream_id":"e0wgt1xk"} +{"time":"2025-03-08T22:22:17.638878641-06:00","level":"INFO","msg":"stream: closed","id":"e0wgt1xk"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222015-e0wgt1xk/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222015-e0wgt1xk/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222015-e0wgt1xk/run-e0wgt1xk.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222015-e0wgt1xk/run-e0wgt1xk.wandb new file mode 100644 index 0000000000000000000000000000000000000000..2d942edf717f205a31ebe519113c84bc5ee14def --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222015-e0wgt1xk/run-e0wgt1xk.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7726699e7036b5dd009d705427ccc27f6671c8363d8e8bba4b8215fabb1dfffd +size 340537 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/files/config.yaml b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2762072642d18c73cf506a153b942622c1f77eb0 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/files/config.yaml @@ -0,0 +1,40 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + python_version: 3.12.5 + cli_version: 0.19.8 + framework: huggingface + huggingface_version: 4.49.0 + is_jupyter_run: false + is_kaggle_kernel: true + start_time: 1741494299 + t: + 1: + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 55 + - 71 + - 75 + - 95 + - 98 + - 105 + 3: + - 4 + - 13 + - 23 + - 42 + - 55 + 4: 3.12.5 + 5: 0.19.8 + 6: 4.49.0 + 8: + - 2 + - 5 + 13: linux-x86_64 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/files/media/table/metrics_0_407e5d8efa569a269e4f.table.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/files/media/table/metrics_0_407e5d8efa569a269e4f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..cf88940e202140790ec435bdc6d85b5af2febc65 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/files/media/table/metrics_0_407e5d8efa569a269e4f.table.json @@ -0,0 +1 @@ +{"columns": ["N", "unc_acc", "unc_auroc", "unc_ece", "acc", "dataset", "split", "ts"], "data": [[2000, 0.0, NaN, 0.5283251887857914, 0.09650000184774399, "offline:all_20k_uniform/choice/probability", "validation", 104.83903151471168]]} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..7eb6e8fe2cc8e0f0ad50a9bb53457815d91309f6 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/files/output.log @@ -0,0 +1,46 @@ +Loading checkpoint shards: 100%|██████████████████████████████████████████████████████████████████████████████████████| 3/3 [00:12<00:00, 4.01s/it] +100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████| 2000/2000 [01:44<00:00, 19.08it/s] +/net/scratch2/zlyu/conda/envs/312/lib/python3.12/site-packages/sklearn/metrics/_ranking.py:375: UndefinedMetricWarning: Only one class is present in y_true. ROC AUC score is not defined in that case. + warnings.warn( +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [01:45<00:00, 105.74s/it] +Traceback (most recent call last): + File "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", line 221, in + fire.Fire(main) + File "/net/scratch2/zlyu/conda/envs/312/lib/python3.12/site-packages/fire/core.py", line 143, in Fire + component_trace = _Fire(component, args, parsed_flag_args, context, name) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/net/scratch2/zlyu/conda/envs/312/lib/python3.12/site-packages/fire/core.py", line 477, in _Fire + component, remaining_args = _CallAndUpdateTrace( + ^^^^^^^^^^^^^^^^^^^^ + File "/net/scratch2/zlyu/conda/envs/312/lib/python3.12/site-packages/fire/core.py", line 693, in _CallAndUpdateTrace + component = fn(*varargs, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/llm/logging.py", line 228, in _wrapped_entrypoint + return f(**kwargs) + ^^^^^^^^^^^ + File "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", line 215, in main + wandb.save(f"{log_dir}/*", base_path=log_dir) + File "/net/scratch2/zlyu/conda/envs/312/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 401, in wrapper_fn + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/net/scratch2/zlyu/conda/envs/312/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 391, in wrapper + return func(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/net/scratch2/zlyu/conda/envs/312/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 1975, in save + return self._save( + ^^^^^^^^^^^ + File "/net/scratch2/zlyu/conda/envs/312/lib/python3.12/site-packages/wandb/sdk/wandb_run.py", line 1997, in _save + raise ValueError("Glob may not start with '*' relative to the base path") +ValueError: Glob may not start with '*' relative to the base path +[2025-03-08 22:25:14,165] (get_model:INFO) Loaded "llama2:13b-chat". +[2025-03-08 22:25:14,569] (get_peft_model_from_checkpoint:INFO) Loaded PEFT adapter 'query' checkpoint from 'cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000' +output_size: 10 +[2025-03-08 22:25:14,570] (main:INFO) output_size: 10 +[2025-03-08 22:25:14,579] (get_model:INFO) Loaded "mlp_probability". +[2025-03-08 22:25:14,591] (get_classifier_head:INFO) Loaded classifier model checkpoint from 'cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000'. +[2025-03-08 22:25:14,598] (get_dataset:INFO) root: /net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_datasets/llama2:13b-chat +[2025-03-08 22:25:14,599] (get_offline:INFO) root: /net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_datasets/llama2:13b-chat/offline/all_20k_uniform/choice/probability +[2025-03-08 22:25:14,600] (get_offline:INFO) root: /net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_datasets/llama2:13b-chat/offline/all_20k_uniform/choice/probability +[2025-03-08 22:25:14,600] (get_offline:INFO) data_files: {'train': ['/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_datasets/llama2:13b-chat/offline/all_20k_uniform/choice/probability/train/0.csv'], 'validation': ['/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_datasets/llama2:13b-chat/offline/all_20k_uniform/choice/probability/validation/0.csv']} +[2025-03-08 22:25:14,890] (get_dataset:INFO) Loaded "offline:all_20k_uniform/choice/probability"; train (N = 18534) / validation (N = 2000) +[2025-03-08 22:26:59,730] (save_metrics_data:INFO) Metrics data saved to "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/validation/classifier_data.bin". \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..7c43c64c9ff052aeb34ce35baf45e09d630b929c --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T04:24:59.960Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128038662144" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/files/wandb-summary.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..181454efc1a5850f4c7b78141d7120662fcfc7cf --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/files/wandb-summary.json @@ -0,0 +1 @@ +{"metrics": {"path": "media/table/metrics_0_407e5d8efa569a269e4f.table.json", "ncols": 8, "nrows": 1, "_type": "table-file", "sha256": "407e5d8efa569a269e4f8847be9ac6fa5aa00f7525d011fb4b90bbdf22d79a06", "size": 235, "artifact_path": "wandb-client-artifact://vy1u5yauj95ukjwrey2nslx8fnn56xlsmp8k8lv7elqtpf18ybswgjchw6i8lh1gnja8bok36tknehoynfoh98jc0fb9afyavy0vpolazd5ar7v6cno3h9kdwynd3gfm/metrics.table.json", "_latest_artifact_path": "wandb-client-artifact://omb2dgxytotfj95xmzi9bkrxtragqq0h9ci2p66rlq1bwwuoytbv2n8k7o9icohyuf6e2ru8kqiy0cq16lk3eb0lglwemypsescrmqnppcf8rwj44tfbq1t97s91rblw:latest/metrics.table.json"}, "_timestamp": 1741494420.1027892, "_runtime": 120.143483696, "_step": 0} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..7bf5014373b01fb619836366fbcdd3bb9e4de19c --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-08T22:24:59.819018409-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmp_zkkgckq/port-2681766.txt","pid":2681766,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T22:24:59.820742716-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2681766} +{"time":"2025-03-08T22:24:59.820729866-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":32841,"Zone":""}} +{"time":"2025-03-08T22:24:59.948594421-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:59944"} +{"time":"2025-03-08T22:24:59.962315627-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"49mgyijh","id":"127.0.0.1:59944"} +{"time":"2025-03-08T22:25:00.181092644-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"49mgyijh","id":"127.0.0.1:59944"} +{"time":"2025-03-08T22:27:00.833080832-06:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:59944"} +{"time":"2025-03-08T22:27:00.833481359-06:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:59944"} +{"time":"2025-03-08T22:27:00.833528549-06:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-08T22:27:00.833715547-06:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:59944"} +{"time":"2025-03-08T22:27:00.835475534-06:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:59944"} +{"time":"2025-03-08T22:27:00.835496634-06:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:59944"} +{"time":"2025-03-08T22:27:00.835512154-06:00","level":"INFO","msg":"server is closed"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f518cc0946016550d2d59d403a7a7c32a5dc6fa9 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/logs/debug-internal.log @@ -0,0 +1,14 @@ +{"time":"2025-03-08T22:25:00.048471986-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/logs/debug-core.log"} +{"time":"2025-03-08T22:25:00.181052584-06:00","level":"INFO","msg":"created new stream","id":"49mgyijh"} +{"time":"2025-03-08T22:25:00.181084084-06:00","level":"INFO","msg":"stream: started","id":"49mgyijh"} +{"time":"2025-03-08T22:25:00.181127374-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"49mgyijh"} +{"time":"2025-03-08T22:25:00.181140104-06:00","level":"INFO","msg":"sender: started","stream_id":"49mgyijh"} +{"time":"2025-03-08T22:25:00.181163604-06:00","level":"INFO","msg":"handler: started","stream_id":"49mgyijh"} +{"time":"2025-03-08T22:25:00.187341027-06:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-08T22:27:00.833502939-06:00","level":"INFO","msg":"stream: closing","id":"49mgyijh"} +{"time":"2025-03-08T22:27:00.833659688-06:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-08T22:27:00.834485841-06:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-08T22:27:00.834592831-06:00","level":"INFO","msg":"handler: closed","stream_id":"49mgyijh"} +{"time":"2025-03-08T22:27:00.8346122-06:00","level":"INFO","msg":"writer: Close: closed","stream_id":"49mgyijh"} +{"time":"2025-03-08T22:27:00.83463864-06:00","level":"INFO","msg":"sender: closed","stream_id":"49mgyijh"} +{"time":"2025-03-08T22:27:00.835340065-06:00","level":"INFO","msg":"stream: closed","id":"49mgyijh"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/run-49mgyijh.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/run-49mgyijh.wandb new file mode 100644 index 0000000000000000000000000000000000000000..774428d2f9794a6ee42e4ab469a56b542c60ba2b --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/run-49mgyijh.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56813165867a9d849ab2edf528e8b19afe98d052e847015e000f2cb3f96ddcc7 +size 465188 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/run-49mgyijh.wandb.synced b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_222459-49mgyijh/run-49mgyijh.wandb.synced new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223355-mbn4f8iq/files/media/table/metrics_0_24e33a6bdbdc6b6e5dc4.table.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223355-mbn4f8iq/files/media/table/metrics_0_24e33a6bdbdc6b6e5dc4.table.json new file mode 100644 index 0000000000000000000000000000000000000000..188b07929825bba073561699474411481b71b996 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223355-mbn4f8iq/files/media/table/metrics_0_24e33a6bdbdc6b6e5dc4.table.json @@ -0,0 +1 @@ +{"columns": ["N", "unc_acc", "unc_auroc", "unc_ece", "acc", "dataset", "split", "ts"], "data": [[10, 0.0, NaN, 0.5878906279802323, 0.10000000149011612, "offline:all_20k_uniform/choice/probability", "validation", 1.1121722972020507]]} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223355-mbn4f8iq/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223355-mbn4f8iq/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223355-mbn4f8iq/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223355-mbn4f8iq/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223355-mbn4f8iq/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223355-mbn4f8iq/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223355-mbn4f8iq/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..3a5aa72b927164a8883d5078a5c77829b83989e6 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223355-mbn4f8iq/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T04:33:55.435172Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128042426368" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223355-mbn4f8iq/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223355-mbn4f8iq/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..09e46e49086eba0477ccfe3c2cf31df4e9483104 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223355-mbn4f8iq/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-08T22:33:55.273684719-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmp9y2hb0oy/port-2695166.txt","pid":2695166,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T22:33:55.275839851-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2695166} +{"time":"2025-03-08T22:33:55.275843541-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":43943,"Zone":""}} +{"time":"2025-03-08T22:33:55.423063709-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:47298"} +{"time":"2025-03-08T22:33:55.437495178-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"mbn4f8iq","id":"127.0.0.1:47298"} +{"time":"2025-03-08T22:33:55.732928337-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"mbn4f8iq","id":"127.0.0.1:47298"} +{"time":"2025-03-08T22:34:12.784845056-06:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:47298"} +{"time":"2025-03-08T22:34:12.785248093-06:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:47298"} +{"time":"2025-03-08T22:34:12.785300403-06:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-08T22:34:12.785410072-06:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:47298"} +{"time":"2025-03-08T22:34:12.787416365-06:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:47298"} +{"time":"2025-03-08T22:34:12.787437084-06:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:47298"} +{"time":"2025-03-08T22:34:12.787451274-06:00","level":"INFO","msg":"server is closed"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223355-mbn4f8iq/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223355-mbn4f8iq/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..9b60cea93d8fe4b42dd54d558eb9a943496c9d2f --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223355-mbn4f8iq/logs/debug-internal.log @@ -0,0 +1,14 @@ +{"time":"2025-03-08T22:33:55.592830618-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223355-mbn4f8iq/logs/debug-core.log"} +{"time":"2025-03-08T22:33:55.732886338-06:00","level":"INFO","msg":"created new stream","id":"mbn4f8iq"} +{"time":"2025-03-08T22:33:55.732920077-06:00","level":"INFO","msg":"stream: started","id":"mbn4f8iq"} +{"time":"2025-03-08T22:33:55.732942327-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"mbn4f8iq"} +{"time":"2025-03-08T22:33:55.732961567-06:00","level":"INFO","msg":"handler: started","stream_id":"mbn4f8iq"} +{"time":"2025-03-08T22:33:55.733237045-06:00","level":"INFO","msg":"sender: started","stream_id":"mbn4f8iq"} +{"time":"2025-03-08T22:33:55.765717081-06:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-08T22:34:12.785292183-06:00","level":"INFO","msg":"stream: closing","id":"mbn4f8iq"} +{"time":"2025-03-08T22:34:12.785366172-06:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-08T22:34:12.786250105-06:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-08T22:34:12.786360194-06:00","level":"INFO","msg":"handler: closed","stream_id":"mbn4f8iq"} +{"time":"2025-03-08T22:34:12.786377473-06:00","level":"INFO","msg":"writer: Close: closed","stream_id":"mbn4f8iq"} +{"time":"2025-03-08T22:34:12.786401943-06:00","level":"INFO","msg":"sender: closed","stream_id":"mbn4f8iq"} +{"time":"2025-03-08T22:34:12.787286096-06:00","level":"INFO","msg":"stream: closed","id":"mbn4f8iq"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223355-mbn4f8iq/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223355-mbn4f8iq/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223355-mbn4f8iq/run-mbn4f8iq.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223355-mbn4f8iq/run-mbn4f8iq.wandb new file mode 100644 index 0000000000000000000000000000000000000000..8b224fc87754bd4a5c71827764292428b5cc4255 Binary files /dev/null and b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223355-mbn4f8iq/run-mbn4f8iq.wandb differ diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223438-v76qzg0z/files/media/table/metrics_0_2189f7867e385afcbe97.table.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223438-v76qzg0z/files/media/table/metrics_0_2189f7867e385afcbe97.table.json new file mode 100644 index 0000000000000000000000000000000000000000..16c0f909609d0f1625ec050486db0688d643bd6a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223438-v76qzg0z/files/media/table/metrics_0_2189f7867e385afcbe97.table.json @@ -0,0 +1 @@ +{"columns": ["N", "unc_acc", "unc_auroc", "unc_ece", "acc", "dataset", "split", "ts"], "data": [[10, 0.0, NaN, 0.5878906279802323, 0.10000000149011612, "offline:all_20k_uniform/choice/probability", "validation", 1.088429800234735]]} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223438-v76qzg0z/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223438-v76qzg0z/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223438-v76qzg0z/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223438-v76qzg0z/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223438-v76qzg0z/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223438-v76qzg0z/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223438-v76qzg0z/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..dd9a11bf0506dd3748a11543852200a3cbf2ee45 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223438-v76qzg0z/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T04:34:39.063451Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128042774528" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223438-v76qzg0z/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223438-v76qzg0z/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..8cee31c63944709a82d0f03736cfcb68ec91afe4 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223438-v76qzg0z/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-08T22:34:38.945192546-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmp5dvx99ud/port-2696484.txt","pid":2696484,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T22:34:38.947493976-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2696484} +{"time":"2025-03-08T22:34:38.947486006-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":43853,"Zone":""}} +{"time":"2025-03-08T22:34:39.050620583-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:58176"} +{"time":"2025-03-08T22:34:39.064995422-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"v76qzg0z","id":"127.0.0.1:58176"} +{"time":"2025-03-08T22:34:39.280809585-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"v76qzg0z","id":"127.0.0.1:58176"} +{"time":"2025-03-08T22:34:55.309021846-06:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:58176"} +{"time":"2025-03-08T22:34:55.309684411-06:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-08T22:34:55.309663691-06:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:58176"} +{"time":"2025-03-08T22:34:55.309836559-06:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:58176"} +{"time":"2025-03-08T22:34:55.311720383-06:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:58176"} +{"time":"2025-03-08T22:34:55.311738653-06:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:58176"} +{"time":"2025-03-08T22:34:55.311751993-06:00","level":"INFO","msg":"server is closed"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223438-v76qzg0z/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223438-v76qzg0z/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..1d039ea7c4d497fbd3c7d169b5f2b20ee971647c --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223438-v76qzg0z/logs/debug-internal.log @@ -0,0 +1,14 @@ +{"time":"2025-03-08T22:34:39.141315843-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223438-v76qzg0z/logs/debug-core.log"} +{"time":"2025-03-08T22:34:39.280753295-06:00","level":"INFO","msg":"created new stream","id":"v76qzg0z"} +{"time":"2025-03-08T22:34:39.280796575-06:00","level":"INFO","msg":"stream: started","id":"v76qzg0z"} +{"time":"2025-03-08T22:34:39.280821485-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"v76qzg0z"} +{"time":"2025-03-08T22:34:39.280853344-06:00","level":"INFO","msg":"sender: started","stream_id":"v76qzg0z"} +{"time":"2025-03-08T22:34:39.280916544-06:00","level":"INFO","msg":"handler: started","stream_id":"v76qzg0z"} +{"time":"2025-03-08T22:34:39.289485702-06:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-08T22:34:55.309703041-06:00","level":"INFO","msg":"stream: closing","id":"v76qzg0z"} +{"time":"2025-03-08T22:34:55.30977634-06:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-08T22:34:55.310711052-06:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-08T22:34:55.310806741-06:00","level":"INFO","msg":"handler: closed","stream_id":"v76qzg0z"} +{"time":"2025-03-08T22:34:55.310822031-06:00","level":"INFO","msg":"writer: Close: closed","stream_id":"v76qzg0z"} +{"time":"2025-03-08T22:34:55.31096127-06:00","level":"INFO","msg":"sender: closed","stream_id":"v76qzg0z"} +{"time":"2025-03-08T22:34:55.311606684-06:00","level":"INFO","msg":"stream: closed","id":"v76qzg0z"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223438-v76qzg0z/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223438-v76qzg0z/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223438-v76qzg0z/run-v76qzg0z.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223438-v76qzg0z/run-v76qzg0z.wandb new file mode 100644 index 0000000000000000000000000000000000000000..435e63454ccf3cfc680be165ceb03bf2b1c46c7f Binary files /dev/null and b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_223438-v76qzg0z/run-v76qzg0z.wandb differ diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224727-210oh42o/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224727-210oh42o/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224727-210oh42o/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224727-210oh42o/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224727-210oh42o/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224727-210oh42o/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224727-210oh42o/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b8cc7c2b9f89a7ccd40d64963ee1d290bb7f8a9b --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224727-210oh42o/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T04:47:28.161177Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128048594944" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224727-210oh42o/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224727-210oh42o/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..049a9fea32cf5c2bac8b69c6b29d7d48587fdb15 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224727-210oh42o/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-08T22:47:28.01417119-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmpn277tzml/port-2714946.txt","pid":2714946,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T22:47:28.016963718-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2714946} +{"time":"2025-03-08T22:47:28.016958038-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":42819,"Zone":""}} +{"time":"2025-03-08T22:47:28.135742387-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:46664"} +{"time":"2025-03-08T22:47:28.16448213-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"210oh42o","id":"127.0.0.1:46664"} +{"time":"2025-03-08T22:47:28.346906856-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"210oh42o","id":"127.0.0.1:46664"} +{"time":"2025-03-08T22:47:44.848537362-06:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:46664"} +{"time":"2025-03-08T22:47:44.849048438-06:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:46664"} +{"time":"2025-03-08T22:47:44.849077057-06:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-08T22:47:44.849121847-06:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:46664"} +{"time":"2025-03-08T22:47:44.850823354-06:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:46664"} +{"time":"2025-03-08T22:47:44.850831523-06:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:46664"} +{"time":"2025-03-08T22:47:44.850837503-06:00","level":"INFO","msg":"server is closed"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224727-210oh42o/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224727-210oh42o/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..dcfcc9dce0d526fe299d8c89523309ad3b870ac1 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224727-210oh42o/logs/debug-internal.log @@ -0,0 +1,14 @@ +{"time":"2025-03-08T22:47:28.229075569-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224727-210oh42o/logs/debug-core.log"} +{"time":"2025-03-08T22:47:28.346874317-06:00","level":"INFO","msg":"created new stream","id":"210oh42o"} +{"time":"2025-03-08T22:47:28.346900526-06:00","level":"INFO","msg":"stream: started","id":"210oh42o"} +{"time":"2025-03-08T22:47:28.346910406-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"210oh42o"} +{"time":"2025-03-08T22:47:28.346934096-06:00","level":"INFO","msg":"handler: started","stream_id":"210oh42o"} +{"time":"2025-03-08T22:47:28.346974296-06:00","level":"INFO","msg":"sender: started","stream_id":"210oh42o"} +{"time":"2025-03-08T22:47:28.354876873-06:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-08T22:47:44.849072387-06:00","level":"INFO","msg":"stream: closing","id":"210oh42o"} +{"time":"2025-03-08T22:47:44.849142387-06:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-08T22:47:44.849819811-06:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-08T22:47:44.849880701-06:00","level":"INFO","msg":"handler: closed","stream_id":"210oh42o"} +{"time":"2025-03-08T22:47:44.849889041-06:00","level":"INFO","msg":"writer: Close: closed","stream_id":"210oh42o"} +{"time":"2025-03-08T22:47:44.849895831-06:00","level":"INFO","msg":"sender: closed","stream_id":"210oh42o"} +{"time":"2025-03-08T22:47:44.850762004-06:00","level":"INFO","msg":"stream: closed","id":"210oh42o"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224727-210oh42o/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224727-210oh42o/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224727-210oh42o/run-210oh42o.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224727-210oh42o/run-210oh42o.wandb new file mode 100644 index 0000000000000000000000000000000000000000..af1f713f63d4cf28979db78708b687ec4d230114 Binary files /dev/null and b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224727-210oh42o/run-210oh42o.wandb differ diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224907-vpywl79o/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224907-vpywl79o/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224907-vpywl79o/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224907-vpywl79o/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224907-vpywl79o/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224907-vpywl79o/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224907-vpywl79o/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c06bd42b8a9e6d8302be2ce113d4c4df460f700c --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224907-vpywl79o/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T04:49:08.314216Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128049340416" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224907-vpywl79o/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224907-vpywl79o/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..f7d00f211b7d4df9f280e12fd8ec5121c43b356d --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224907-vpywl79o/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-08T22:49:07.861197938-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmp42olu4q5/port-2717645.txt","pid":2717645,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T22:49:07.8634395-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":38343,"Zone":""}} +{"time":"2025-03-08T22:49:07.863568189-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2717645} +{"time":"2025-03-08T22:49:07.999208689-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:37196"} +{"time":"2025-03-08T22:49:08.317379699-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"vpywl79o","id":"127.0.0.1:37196"} +{"time":"2025-03-08T22:49:08.54792949-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"vpywl79o","id":"127.0.0.1:37196"} +{"time":"2025-03-08T22:49:25.578343536-06:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:37196"} +{"time":"2025-03-08T22:49:25.578954162-06:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:37196"} +{"time":"2025-03-08T22:49:25.579005721-06:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-08T22:49:25.57909874-06:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:37196"} +{"time":"2025-03-08T22:49:25.58171452-06:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:37196"} +{"time":"2025-03-08T22:49:25.581736919-06:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:37196"} +{"time":"2025-03-08T22:49:25.581750379-06:00","level":"INFO","msg":"server is closed"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224907-vpywl79o/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224907-vpywl79o/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7e222ca2c47fd2cb098572e3e6031f3690203687 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224907-vpywl79o/logs/debug-internal.log @@ -0,0 +1,14 @@ +{"time":"2025-03-08T22:49:08.406078609-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224907-vpywl79o/logs/debug-core.log"} +{"time":"2025-03-08T22:49:08.54784698-06:00","level":"INFO","msg":"created new stream","id":"vpywl79o"} +{"time":"2025-03-08T22:49:08.5479119-06:00","level":"INFO","msg":"stream: started","id":"vpywl79o"} +{"time":"2025-03-08T22:49:08.547994719-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"vpywl79o"} +{"time":"2025-03-08T22:49:08.548019259-06:00","level":"INFO","msg":"handler: started","stream_id":"vpywl79o"} +{"time":"2025-03-08T22:49:08.548093588-06:00","level":"INFO","msg":"sender: started","stream_id":"vpywl79o"} +{"time":"2025-03-08T22:49:08.558847413-06:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-08T22:49:25.578988431-06:00","level":"INFO","msg":"stream: closing","id":"vpywl79o"} +{"time":"2025-03-08T22:49:25.57912795-06:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-08T22:49:25.580309261-06:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-08T22:49:25.58040898-06:00","level":"INFO","msg":"handler: closed","stream_id":"vpywl79o"} +{"time":"2025-03-08T22:49:25.580429-06:00","level":"INFO","msg":"writer: Close: closed","stream_id":"vpywl79o"} +{"time":"2025-03-08T22:49:25.58046845-06:00","level":"INFO","msg":"sender: closed","stream_id":"vpywl79o"} +{"time":"2025-03-08T22:49:25.581556161-06:00","level":"INFO","msg":"stream: closed","id":"vpywl79o"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224907-vpywl79o/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224907-vpywl79o/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224907-vpywl79o/run-vpywl79o.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224907-vpywl79o/run-vpywl79o.wandb new file mode 100644 index 0000000000000000000000000000000000000000..1ce5504ec8e4e5f3e123b9732df15989af46b7a1 Binary files /dev/null and b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_224907-vpywl79o/run-vpywl79o.wandb differ diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225035-g7kkfhfq/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225035-g7kkfhfq/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225035-g7kkfhfq/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225035-g7kkfhfq/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225035-g7kkfhfq/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225035-g7kkfhfq/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225035-g7kkfhfq/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..18c12f2d4c0b2367ed43f8608e1195ee573fb667 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225035-g7kkfhfq/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T04:50:36.074525Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128050028544" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225035-g7kkfhfq/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225035-g7kkfhfq/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..2cabb750de9c15ebbf14cab396026d2888fbc82b --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225035-g7kkfhfq/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-08T22:50:35.874083069-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmpd8t2dehp/port-2719767.txt","pid":2719767,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T22:50:35.876079963-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":37049,"Zone":""}} +{"time":"2025-03-08T22:50:35.876301012-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2719767} +{"time":"2025-03-08T22:50:36.045310331-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:33076"} +{"time":"2025-03-08T22:50:36.075685162-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"g7kkfhfq","id":"127.0.0.1:33076"} +{"time":"2025-03-08T22:50:36.458476888-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"g7kkfhfq","id":"127.0.0.1:33076"} +{"time":"2025-03-08T22:50:53.645679999-06:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:33076"} +{"time":"2025-03-08T22:50:53.646181275-06:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-08T22:50:53.646141805-06:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:33076"} +{"time":"2025-03-08T22:50:53.646367533-06:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:33076"} +{"time":"2025-03-08T22:50:53.648723345-06:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:33076"} +{"time":"2025-03-08T22:50:53.648753424-06:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:33076"} +{"time":"2025-03-08T22:50:53.648781174-06:00","level":"INFO","msg":"server is closed"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225035-g7kkfhfq/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225035-g7kkfhfq/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..489418a1c4d531636adac2a068097d75635e5b91 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225035-g7kkfhfq/logs/debug-internal.log @@ -0,0 +1,14 @@ +{"time":"2025-03-08T22:50:36.309256342-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225035-g7kkfhfq/logs/debug-core.log"} +{"time":"2025-03-08T22:50:36.458401548-06:00","level":"INFO","msg":"created new stream","id":"g7kkfhfq"} +{"time":"2025-03-08T22:50:36.458465578-06:00","level":"INFO","msg":"stream: started","id":"g7kkfhfq"} +{"time":"2025-03-08T22:50:36.458688366-06:00","level":"INFO","msg":"handler: started","stream_id":"g7kkfhfq"} +{"time":"2025-03-08T22:50:36.458728996-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"g7kkfhfq"} +{"time":"2025-03-08T22:50:36.458916354-06:00","level":"INFO","msg":"sender: started","stream_id":"g7kkfhfq"} +{"time":"2025-03-08T22:50:36.533199069-06:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-08T22:50:53.646191945-06:00","level":"INFO","msg":"stream: closing","id":"g7kkfhfq"} +{"time":"2025-03-08T22:50:53.646370623-06:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-08T22:50:53.647400055-06:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-08T22:50:53.647537774-06:00","level":"INFO","msg":"handler: closed","stream_id":"g7kkfhfq"} +{"time":"2025-03-08T22:50:53.647561334-06:00","level":"INFO","msg":"writer: Close: closed","stream_id":"g7kkfhfq"} +{"time":"2025-03-08T22:50:53.647590384-06:00","level":"INFO","msg":"sender: closed","stream_id":"g7kkfhfq"} +{"time":"2025-03-08T22:50:53.648581456-06:00","level":"INFO","msg":"stream: closed","id":"g7kkfhfq"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225035-g7kkfhfq/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225035-g7kkfhfq/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225035-g7kkfhfq/run-g7kkfhfq.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225035-g7kkfhfq/run-g7kkfhfq.wandb new file mode 100644 index 0000000000000000000000000000000000000000..277aa2599ba86fedc65389d221074eb05b1c3be9 Binary files /dev/null and b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225035-g7kkfhfq/run-g7kkfhfq.wandb differ diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225249-pcok7zgm/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225249-pcok7zgm/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225249-pcok7zgm/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225249-pcok7zgm/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225249-pcok7zgm/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225249-pcok7zgm/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225249-pcok7zgm/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..debe918a3d7bca3aa472c7fd24be19f0882a4e58 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225249-pcok7zgm/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T04:52:49.708953Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128050880512" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225249-pcok7zgm/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225249-pcok7zgm/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..3f1a4a32a6aea22404c2b333aedcc373c294ab03 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225249-pcok7zgm/logs/debug-core.log @@ -0,0 +1,7 @@ +{"time":"2025-03-08T22:52:49.529482605-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmpojktwypc/port-2722700.txt","pid":2722700,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T22:52:49.53142823-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2722700} +{"time":"2025-03-08T22:52:49.53138418-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":41607,"Zone":""}} +{"time":"2025-03-08T22:52:49.688644965-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:37116"} +{"time":"2025-03-08T22:52:49.710186566-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"pcok7zgm","id":"127.0.0.1:37116"} +{"time":"2025-03-08T22:52:49.860294317-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"pcok7zgm","id":"127.0.0.1:37116"} +{"time":"2025-03-08T22:53:31.185464997-06:00","level":"INFO","msg":"Parent process exited, terminating service process."} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225249-pcok7zgm/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225249-pcok7zgm/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..5f721b51e96d8d2ae65ee6fd87002c250e2a0d92 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225249-pcok7zgm/logs/debug-internal.log @@ -0,0 +1,7 @@ +{"time":"2025-03-08T22:52:49.743031038-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225249-pcok7zgm/logs/debug-core.log"} +{"time":"2025-03-08T22:52:49.860250987-06:00","level":"INFO","msg":"created new stream","id":"pcok7zgm"} +{"time":"2025-03-08T22:52:49.860287327-06:00","level":"INFO","msg":"stream: started","id":"pcok7zgm"} +{"time":"2025-03-08T22:52:49.860355676-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"pcok7zgm"} +{"time":"2025-03-08T22:52:49.860389636-06:00","level":"INFO","msg":"handler: started","stream_id":"pcok7zgm"} +{"time":"2025-03-08T22:52:49.860382176-06:00","level":"INFO","msg":"sender: started","stream_id":"pcok7zgm"} +{"time":"2025-03-08T22:52:49.869187377-06:00","level":"INFO","msg":"Starting system monitor"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225249-pcok7zgm/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225249-pcok7zgm/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225249-pcok7zgm/run-pcok7zgm.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225249-pcok7zgm/run-pcok7zgm.wandb new file mode 100644 index 0000000000000000000000000000000000000000..17fd60ebaf0cb75ece08631f0713f0986dc73ef4 Binary files /dev/null and b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225249-pcok7zgm/run-pcok7zgm.wandb differ diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225341-dx7z19v2/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225341-dx7z19v2/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225341-dx7z19v2/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225341-dx7z19v2/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225341-dx7z19v2/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225341-dx7z19v2/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225341-dx7z19v2/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..67297586e78d1eb5396d20d47a7beb077b7d6df1 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225341-dx7z19v2/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T04:53:42.144768Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128051384320" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225341-dx7z19v2/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225341-dx7z19v2/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..35d34cdd6d4519de29224e912b4532aed8174029 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225341-dx7z19v2/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-08T22:53:41.937261769-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmpau73yfui/port-2724730.txt","pid":2724730,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T22:53:41.940016737-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2724730} +{"time":"2025-03-08T22:53:41.939947698-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":42591,"Zone":""}} +{"time":"2025-03-08T22:53:42.133305301-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:38896"} +{"time":"2025-03-08T22:53:42.146005131-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"dx7z19v2","id":"127.0.0.1:38896"} +{"time":"2025-03-08T22:53:42.329259424-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"dx7z19v2","id":"127.0.0.1:38896"} +{"time":"2025-03-08T22:54:00.35850174-06:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:38896"} +{"time":"2025-03-08T22:54:00.360577193-06:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-08T22:54:00.360548153-06:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:38896"} +{"time":"2025-03-08T22:54:00.360784142-06:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:38896"} +{"time":"2025-03-08T22:54:00.365922671-06:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:38896"} +{"time":"2025-03-08T22:54:00.365965701-06:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:38896"} +{"time":"2025-03-08T22:54:00.365991211-06:00","level":"INFO","msg":"server is closed"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225341-dx7z19v2/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225341-dx7z19v2/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a35c13fd0602218b088376314c5491a3a6408cb9 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225341-dx7z19v2/logs/debug-internal.log @@ -0,0 +1,14 @@ +{"time":"2025-03-08T22:53:42.190831959-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225341-dx7z19v2/logs/debug-core.log"} +{"time":"2025-03-08T22:53:42.329211344-06:00","level":"INFO","msg":"created new stream","id":"dx7z19v2"} +{"time":"2025-03-08T22:53:42.329249824-06:00","level":"INFO","msg":"stream: started","id":"dx7z19v2"} +{"time":"2025-03-08T22:53:42.329298123-06:00","level":"INFO","msg":"handler: started","stream_id":"dx7z19v2"} +{"time":"2025-03-08T22:53:42.329325273-06:00","level":"INFO","msg":"sender: started","stream_id":"dx7z19v2"} +{"time":"2025-03-08T22:53:42.329288593-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"dx7z19v2"} +{"time":"2025-03-08T22:53:42.33734687-06:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-08T22:54:00.360612973-06:00","level":"INFO","msg":"stream: closing","id":"dx7z19v2"} +{"time":"2025-03-08T22:54:00.360710262-06:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-08T22:54:00.362111561-06:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-08T22:54:00.36229331-06:00","level":"INFO","msg":"handler: closed","stream_id":"dx7z19v2"} +{"time":"2025-03-08T22:54:00.36231671-06:00","level":"INFO","msg":"writer: Close: closed","stream_id":"dx7z19v2"} +{"time":"2025-03-08T22:54:00.36235168-06:00","level":"INFO","msg":"sender: closed","stream_id":"dx7z19v2"} +{"time":"2025-03-08T22:54:00.365756123-06:00","level":"INFO","msg":"stream: closed","id":"dx7z19v2"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225341-dx7z19v2/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225341-dx7z19v2/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225341-dx7z19v2/run-dx7z19v2.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225341-dx7z19v2/run-dx7z19v2.wandb new file mode 100644 index 0000000000000000000000000000000000000000..1868f627ef196d9efaca618217162865d5b40807 Binary files /dev/null and b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225341-dx7z19v2/run-dx7z19v2.wandb differ diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225545-2at7gj38/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225545-2at7gj38/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225545-2at7gj38/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225545-2at7gj38/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225545-2at7gj38/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225545-2at7gj38/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225545-2at7gj38/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..5d79f08db99b79aafc029c872ee1386082a78b9a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225545-2at7gj38/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T04:55:46.350037Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128052289536" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225545-2at7gj38/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225545-2at7gj38/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..d76c874ba7ce0e99b435fa0af5618e4b5a5a10f7 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225545-2at7gj38/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-08T22:55:46.196690021-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmp7ybaxwai/port-2727968.txt","pid":2727968,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T22:55:46.199383569-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2727968} +{"time":"2025-03-08T22:55:46.199392819-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":43863,"Zone":""}} +{"time":"2025-03-08T22:55:46.324070663-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:38836"} +{"time":"2025-03-08T22:55:46.351564408-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"2at7gj38","id":"127.0.0.1:38836"} +{"time":"2025-03-08T22:55:46.639999818-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"2at7gj38","id":"127.0.0.1:38836"} +{"time":"2025-03-08T22:56:03.668879483-06:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:38836"} +{"time":"2025-03-08T22:56:03.669094231-06:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:38836"} +{"time":"2025-03-08T22:56:03.669116401-06:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-08T22:56:03.66917388-06:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:38836"} +{"time":"2025-03-08T22:56:03.670392671-06:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:38836"} +{"time":"2025-03-08T22:56:03.670410011-06:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:38836"} +{"time":"2025-03-08T22:56:03.670418691-06:00","level":"INFO","msg":"server is closed"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225545-2at7gj38/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225545-2at7gj38/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..680c806bbf1788fe5e28c912ff5a5e922ffc2e57 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225545-2at7gj38/logs/debug-internal.log @@ -0,0 +1,14 @@ +{"time":"2025-03-08T22:55:46.5057357-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225545-2at7gj38/logs/debug-core.log"} +{"time":"2025-03-08T22:55:46.639940979-06:00","level":"INFO","msg":"created new stream","id":"2at7gj38"} +{"time":"2025-03-08T22:55:46.639987648-06:00","level":"INFO","msg":"stream: started","id":"2at7gj38"} +{"time":"2025-03-08T22:55:46.640007748-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"2at7gj38"} +{"time":"2025-03-08T22:55:46.640052778-06:00","level":"INFO","msg":"handler: started","stream_id":"2at7gj38"} +{"time":"2025-03-08T22:55:46.640083128-06:00","level":"INFO","msg":"sender: started","stream_id":"2at7gj38"} +{"time":"2025-03-08T22:55:46.651448979-06:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-08T22:56:03.66914548-06:00","level":"INFO","msg":"stream: closing","id":"2at7gj38"} +{"time":"2025-03-08T22:56:03.66918457-06:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-08T22:56:03.669833525-06:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-08T22:56:03.669886445-06:00","level":"INFO","msg":"handler: closed","stream_id":"2at7gj38"} +{"time":"2025-03-08T22:56:03.669895985-06:00","level":"INFO","msg":"writer: Close: closed","stream_id":"2at7gj38"} +{"time":"2025-03-08T22:56:03.669903785-06:00","level":"INFO","msg":"sender: closed","stream_id":"2at7gj38"} +{"time":"2025-03-08T22:56:03.670176252-06:00","level":"INFO","msg":"stream: closed","id":"2at7gj38"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225545-2at7gj38/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225545-2at7gj38/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225545-2at7gj38/run-2at7gj38.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225545-2at7gj38/run-2at7gj38.wandb new file mode 100644 index 0000000000000000000000000000000000000000..619647d2bc55a50c4dbde7fb147f8cbc45edef71 Binary files /dev/null and b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225545-2at7gj38/run-2at7gj38.wandb differ diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225856-q7z0lptj/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225856-q7z0lptj/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225856-q7z0lptj/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225856-q7z0lptj/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225856-q7z0lptj/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225856-q7z0lptj/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225856-q7z0lptj/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..1acccbdadf939d77a2e9b7f7ed22ec754ab80107 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225856-q7z0lptj/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T04:58:57.543603Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128053547008" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225856-q7z0lptj/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225856-q7z0lptj/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..2fce507b9bd98a999af839006b5bd0a5cd3247ff --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225856-q7z0lptj/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-08T22:58:56.987955253-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmp7jx3flef/port-2731818.txt","pid":2731818,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T22:58:56.990833661-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2731818} +{"time":"2025-03-08T22:58:56.990811351-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":34085,"Zone":""}} +{"time":"2025-03-08T22:58:57.148593857-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:35866"} +{"time":"2025-03-08T22:58:57.54624991-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"q7z0lptj","id":"127.0.0.1:35866"} +{"time":"2025-03-08T22:58:57.720626946-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"q7z0lptj","id":"127.0.0.1:35866"} +{"time":"2025-03-08T22:59:13.935771308-06:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:35866"} +{"time":"2025-03-08T22:59:13.936520532-06:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:35866"} +{"time":"2025-03-08T22:59:13.936574592-06:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-08T22:59:13.936727051-06:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:35866"} +{"time":"2025-03-08T22:59:13.941485404-06:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:35866"} +{"time":"2025-03-08T22:59:13.941505464-06:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:35866"} +{"time":"2025-03-08T22:59:13.941516814-06:00","level":"INFO","msg":"server is closed"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225856-q7z0lptj/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225856-q7z0lptj/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..78743090dc1008511a8ba1162d084ce8b2d4381a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225856-q7z0lptj/logs/debug-internal.log @@ -0,0 +1,14 @@ +{"time":"2025-03-08T22:58:57.587349478-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225856-q7z0lptj/logs/debug-core.log"} +{"time":"2025-03-08T22:58:57.720585107-06:00","level":"INFO","msg":"created new stream","id":"q7z0lptj"} +{"time":"2025-03-08T22:58:57.720620666-06:00","level":"INFO","msg":"stream: started","id":"q7z0lptj"} +{"time":"2025-03-08T22:58:57.720679226-06:00","level":"INFO","msg":"handler: started","stream_id":"q7z0lptj"} +{"time":"2025-03-08T22:58:57.720676566-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"q7z0lptj"} +{"time":"2025-03-08T22:58:57.720732876-06:00","level":"INFO","msg":"sender: started","stream_id":"q7z0lptj"} +{"time":"2025-03-08T22:58:57.731272053-06:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-08T22:59:13.936586492-06:00","level":"INFO","msg":"stream: closing","id":"q7z0lptj"} +{"time":"2025-03-08T22:59:13.93675886-06:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-08T22:59:13.937984601-06:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-08T22:59:13.93811167-06:00","level":"INFO","msg":"handler: closed","stream_id":"q7z0lptj"} +{"time":"2025-03-08T22:59:13.93812985-06:00","level":"INFO","msg":"writer: Close: closed","stream_id":"q7z0lptj"} +{"time":"2025-03-08T22:59:13.93815457-06:00","level":"INFO","msg":"sender: closed","stream_id":"q7z0lptj"} +{"time":"2025-03-08T22:59:13.941295185-06:00","level":"INFO","msg":"stream: closed","id":"q7z0lptj"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225856-q7z0lptj/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225856-q7z0lptj/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225856-q7z0lptj/run-q7z0lptj.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225856-q7z0lptj/run-q7z0lptj.wandb new file mode 100644 index 0000000000000000000000000000000000000000..c497c1b47a40e73563408ceebc7754e4e0cb0b5e Binary files /dev/null and b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225856-q7z0lptj/run-q7z0lptj.wandb differ diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225941-g161p945/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225941-g161p945/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225941-g161p945/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225941-g161p945/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225941-g161p945/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225941-g161p945/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225941-g161p945/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f128ce710348b4a5147741184efa5f38cee88dfb --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225941-g161p945/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T04:59:41.419168Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128053936128" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225941-g161p945/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225941-g161p945/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..a5815f87a54271b742319eacc716f2e202c0541f --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225941-g161p945/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-08T22:59:41.317897438-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmptldnb7ks/port-2733269.txt","pid":2733269,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T22:59:41.319282127-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2733269} +{"time":"2025-03-08T22:59:41.319278607-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":38045,"Zone":""}} +{"time":"2025-03-08T22:59:41.411901794-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:37764"} +{"time":"2025-03-08T22:59:41.420295238-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"g161p945","id":"127.0.0.1:37764"} +{"time":"2025-03-08T22:59:41.5429271-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"g161p945","id":"127.0.0.1:37764"} +{"time":"2025-03-08T22:59:59.67422444-06:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:37764"} +{"time":"2025-03-08T22:59:59.674657216-06:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:37764"} +{"time":"2025-03-08T22:59:59.674672846-06:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-08T22:59:59.674736426-06:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:37764"} +{"time":"2025-03-08T22:59:59.676633391-06:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:37764"} +{"time":"2025-03-08T22:59:59.676643151-06:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:37764"} +{"time":"2025-03-08T22:59:59.6766482-06:00","level":"INFO","msg":"server is closed"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225941-g161p945/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225941-g161p945/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d81a62fabfeca833e67907c7f919dfb3b11e62c2 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225941-g161p945/logs/debug-internal.log @@ -0,0 +1,14 @@ +{"time":"2025-03-08T22:59:41.431999517-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225941-g161p945/logs/debug-core.log"} +{"time":"2025-03-08T22:59:41.54290251-06:00","level":"INFO","msg":"created new stream","id":"g161p945"} +{"time":"2025-03-08T22:59:41.54292188-06:00","level":"INFO","msg":"stream: started","id":"g161p945"} +{"time":"2025-03-08T22:59:41.5429458-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"g161p945"} +{"time":"2025-03-08T22:59:41.54296733-06:00","level":"INFO","msg":"handler: started","stream_id":"g161p945"} +{"time":"2025-03-08T22:59:41.54296584-06:00","level":"INFO","msg":"sender: started","stream_id":"g161p945"} +{"time":"2025-03-08T22:59:41.54921673-06:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-08T22:59:59.674699896-06:00","level":"INFO","msg":"stream: closing","id":"g161p945"} +{"time":"2025-03-08T22:59:59.674743415-06:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-08T22:59:59.67541859-06:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-08T22:59:59.675472-06:00","level":"INFO","msg":"handler: closed","stream_id":"g161p945"} +{"time":"2025-03-08T22:59:59.67548035-06:00","level":"INFO","msg":"writer: Close: closed","stream_id":"g161p945"} +{"time":"2025-03-08T22:59:59.675537759-06:00","level":"INFO","msg":"sender: closed","stream_id":"g161p945"} +{"time":"2025-03-08T22:59:59.676576811-06:00","level":"INFO","msg":"stream: closed","id":"g161p945"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225941-g161p945/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225941-g161p945/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225941-g161p945/run-g161p945.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225941-g161p945/run-g161p945.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e7e83a19c025e9408c73321d6f5af5745932c568 Binary files /dev/null and b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_225941-g161p945/run-g161p945.wandb differ diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230040-3zskcy79/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230040-3zskcy79/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230040-3zskcy79/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230040-3zskcy79/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230040-3zskcy79/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230040-3zskcy79/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230040-3zskcy79/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b111d0ace363e7fa7b3c8c39502a9b93662f2dec --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230040-3zskcy79/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T05:00:41.311821Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128054476800" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230040-3zskcy79/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230040-3zskcy79/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..2a353c7be43f4bdb4130a6f03873ae53300e5222 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230040-3zskcy79/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-08T23:00:41.127633997-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmpak8bbwta/port-2734976.txt","pid":2734976,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T23:00:41.130688863-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2734976} +{"time":"2025-03-08T23:00:41.130696083-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":33787,"Zone":""}} +{"time":"2025-03-08T23:00:41.297667579-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:44282"} +{"time":"2025-03-08T23:00:41.313555145-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"3zskcy79","id":"127.0.0.1:44282"} +{"time":"2025-03-08T23:00:41.498209652-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"3zskcy79","id":"127.0.0.1:44282"} +{"time":"2025-03-08T23:00:57.604291758-06:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:44282"} +{"time":"2025-03-08T23:00:57.604540756-06:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:44282"} +{"time":"2025-03-08T23:00:57.604551186-06:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-08T23:00:57.604627695-06:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:44282"} +{"time":"2025-03-08T23:00:57.605925205-06:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:44282"} +{"time":"2025-03-08T23:00:57.605940555-06:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:44282"} +{"time":"2025-03-08T23:00:57.605950685-06:00","level":"INFO","msg":"server is closed"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230040-3zskcy79/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230040-3zskcy79/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..231ac1fd7137af2981cf5e6c48aba37dbe22ffa2 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230040-3zskcy79/logs/debug-internal.log @@ -0,0 +1,14 @@ +{"time":"2025-03-08T23:00:41.383523238-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230040-3zskcy79/logs/debug-core.log"} +{"time":"2025-03-08T23:00:41.498162973-06:00","level":"INFO","msg":"created new stream","id":"3zskcy79"} +{"time":"2025-03-08T23:00:41.498200143-06:00","level":"INFO","msg":"stream: started","id":"3zskcy79"} +{"time":"2025-03-08T23:00:41.498226232-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"3zskcy79"} +{"time":"2025-03-08T23:00:41.498232982-06:00","level":"INFO","msg":"handler: started","stream_id":"3zskcy79"} +{"time":"2025-03-08T23:00:41.498253382-06:00","level":"INFO","msg":"sender: started","stream_id":"3zskcy79"} +{"time":"2025-03-08T23:00:41.505770214-06:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-08T23:00:57.604592485-06:00","level":"INFO","msg":"stream: closing","id":"3zskcy79"} +{"time":"2025-03-08T23:00:57.604633295-06:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-08T23:00:57.60527842-06:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-08T23:00:57.60533628-06:00","level":"INFO","msg":"handler: closed","stream_id":"3zskcy79"} +{"time":"2025-03-08T23:00:57.605346009-06:00","level":"INFO","msg":"writer: Close: closed","stream_id":"3zskcy79"} +{"time":"2025-03-08T23:00:57.605354299-06:00","level":"INFO","msg":"sender: closed","stream_id":"3zskcy79"} +{"time":"2025-03-08T23:00:57.605842326-06:00","level":"INFO","msg":"stream: closed","id":"3zskcy79"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230040-3zskcy79/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230040-3zskcy79/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230040-3zskcy79/run-3zskcy79.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230040-3zskcy79/run-3zskcy79.wandb new file mode 100644 index 0000000000000000000000000000000000000000..2b1e8d4f7c6c3ac42d621571c60854165f275cc9 Binary files /dev/null and b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230040-3zskcy79/run-3zskcy79.wandb differ diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230231-dmvk44te/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230231-dmvk44te/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230231-dmvk44te/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230231-dmvk44te/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230231-dmvk44te/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230231-dmvk44te/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230231-dmvk44te/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..77c6fc5e20585f818e3b4dfd8243d8250875caf6 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230231-dmvk44te/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T05:02:31.473647Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128055173120" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230231-dmvk44te/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230231-dmvk44te/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..460a4627992d3d0fed2cc77da9ce70332362bf51 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230231-dmvk44te/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-08T23:02:31.262677027-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmpd6tt5acv/port-2737557.txt","pid":2737557,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T23:02:31.26486743-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2737557} +{"time":"2025-03-08T23:02:31.26486207-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":40773,"Zone":""}} +{"time":"2025-03-08T23:02:31.424446525-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:53198"} +{"time":"2025-03-08T23:02:31.476797966-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"dmvk44te","id":"127.0.0.1:53198"} +{"time":"2025-03-08T23:02:31.644858924-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"dmvk44te","id":"127.0.0.1:53198"} +{"time":"2025-03-08T23:02:48.670248932-06:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:53198"} +{"time":"2025-03-08T23:02:48.670854038-06:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:53198"} +{"time":"2025-03-08T23:02:48.670917387-06:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-08T23:02:48.671012456-06:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:53198"} +{"time":"2025-03-08T23:02:48.673555936-06:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:53198"} +{"time":"2025-03-08T23:02:48.673580056-06:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:53198"} +{"time":"2025-03-08T23:02:48.673596186-06:00","level":"INFO","msg":"server is closed"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230231-dmvk44te/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230231-dmvk44te/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..acaf93bdde00295a3be5b1c1de86db6eb2718f59 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230231-dmvk44te/logs/debug-internal.log @@ -0,0 +1,14 @@ +{"time":"2025-03-08T23:02:31.512304849-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230231-dmvk44te/logs/debug-core.log"} +{"time":"2025-03-08T23:02:31.644824675-06:00","level":"INFO","msg":"created new stream","id":"dmvk44te"} +{"time":"2025-03-08T23:02:31.644851414-06:00","level":"INFO","msg":"stream: started","id":"dmvk44te"} +{"time":"2025-03-08T23:02:31.644872394-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"dmvk44te"} +{"time":"2025-03-08T23:02:31.644910264-06:00","level":"INFO","msg":"handler: started","stream_id":"dmvk44te"} +{"time":"2025-03-08T23:02:31.644995313-06:00","level":"INFO","msg":"sender: started","stream_id":"dmvk44te"} +{"time":"2025-03-08T23:02:31.65191406-06:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-08T23:02:48.670870987-06:00","level":"INFO","msg":"stream: closing","id":"dmvk44te"} +{"time":"2025-03-08T23:02:48.671020096-06:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-08T23:02:48.672159727-06:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-08T23:02:48.672279097-06:00","level":"INFO","msg":"handler: closed","stream_id":"dmvk44te"} +{"time":"2025-03-08T23:02:48.672298846-06:00","level":"INFO","msg":"writer: Close: closed","stream_id":"dmvk44te"} +{"time":"2025-03-08T23:02:48.672303696-06:00","level":"INFO","msg":"sender: closed","stream_id":"dmvk44te"} +{"time":"2025-03-08T23:02:48.673401968-06:00","level":"INFO","msg":"stream: closed","id":"dmvk44te"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230231-dmvk44te/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230231-dmvk44te/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230231-dmvk44te/run-dmvk44te.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230231-dmvk44te/run-dmvk44te.wandb new file mode 100644 index 0000000000000000000000000000000000000000..db40870937b036d2a935340d0f03faf0850e9fd2 Binary files /dev/null and b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230231-dmvk44te/run-dmvk44te.wandb differ diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230812-1lcjnxe1/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230812-1lcjnxe1/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230812-1lcjnxe1/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230812-1lcjnxe1/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230812-1lcjnxe1/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230812-1lcjnxe1/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230812-1lcjnxe1/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b2fe83a391084a9fb8f8576b698ecd8e60143a92 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230812-1lcjnxe1/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T05:08:12.957356Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128057298944" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230812-1lcjnxe1/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230812-1lcjnxe1/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..8cea3f2ed6e481b8e8e2d989e6b84b1798096d14 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230812-1lcjnxe1/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-08T23:08:12.762977469-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmpqki24h5j/port-2744171.txt","pid":2744171,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T23:08:12.765952006-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2744171} +{"time":"2025-03-08T23:08:12.765942136-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":35973,"Zone":""}} +{"time":"2025-03-08T23:08:12.935565924-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:34682"} +{"time":"2025-03-08T23:08:12.959870155-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"1lcjnxe1","id":"127.0.0.1:34682"} +{"time":"2025-03-08T23:08:13.260070935-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"1lcjnxe1","id":"127.0.0.1:34682"} +{"time":"2025-03-08T23:08:29.398421321-06:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:34682"} +{"time":"2025-03-08T23:08:29.401501447-06:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:34682"} +{"time":"2025-03-08T23:08:29.401546997-06:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-08T23:08:29.401674146-06:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:34682"} +{"time":"2025-03-08T23:08:29.409005228-06:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:34682"} +{"time":"2025-03-08T23:08:29.409014608-06:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:34682"} +{"time":"2025-03-08T23:08:29.409022398-06:00","level":"INFO","msg":"server is closed"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230812-1lcjnxe1/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230812-1lcjnxe1/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..598e5f77dce2b870cced116cedea9ececa634de1 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230812-1lcjnxe1/logs/debug-internal.log @@ -0,0 +1,14 @@ +{"time":"2025-03-08T23:08:13.124049006-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230812-1lcjnxe1/logs/debug-core.log"} +{"time":"2025-03-08T23:08:13.260018986-06:00","level":"INFO","msg":"created new stream","id":"1lcjnxe1"} +{"time":"2025-03-08T23:08:13.260060625-06:00","level":"INFO","msg":"stream: started","id":"1lcjnxe1"} +{"time":"2025-03-08T23:08:13.260130835-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"1lcjnxe1"} +{"time":"2025-03-08T23:08:13.260148975-06:00","level":"INFO","msg":"handler: started","stream_id":"1lcjnxe1"} +{"time":"2025-03-08T23:08:13.260241804-06:00","level":"INFO","msg":"sender: started","stream_id":"1lcjnxe1"} +{"time":"2025-03-08T23:08:13.273410571-06:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-08T23:08:29.401571847-06:00","level":"INFO","msg":"stream: closing","id":"1lcjnxe1"} +{"time":"2025-03-08T23:08:29.401687226-06:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-08T23:08:29.403874429-06:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-08T23:08:29.403980718-06:00","level":"INFO","msg":"handler: closed","stream_id":"1lcjnxe1"} +{"time":"2025-03-08T23:08:29.403995038-06:00","level":"INFO","msg":"writer: Close: closed","stream_id":"1lcjnxe1"} +{"time":"2025-03-08T23:08:29.404037167-06:00","level":"INFO","msg":"sender: closed","stream_id":"1lcjnxe1"} +{"time":"2025-03-08T23:08:29.408924049-06:00","level":"INFO","msg":"stream: closed","id":"1lcjnxe1"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230812-1lcjnxe1/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230812-1lcjnxe1/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230812-1lcjnxe1/run-1lcjnxe1.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230812-1lcjnxe1/run-1lcjnxe1.wandb new file mode 100644 index 0000000000000000000000000000000000000000..a6cfd819a33048c0bc844ca5384130cd9e605e86 Binary files /dev/null and b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230812-1lcjnxe1/run-1lcjnxe1.wandb differ diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230927-emw0owd9/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230927-emw0owd9/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230927-emw0owd9/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230927-emw0owd9/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230927-emw0owd9/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230927-emw0owd9/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230927-emw0owd9/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..8ec9f23eb15445b3fa4b43e610d54811cad7845c --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230927-emw0owd9/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T05:09:28.393536Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128057868288" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230927-emw0owd9/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230927-emw0owd9/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..f04b4a61873fd760034672f33ae2356ef485a985 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230927-emw0owd9/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-08T23:09:27.678795831-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmp5ikj4adl/port-2746099.txt","pid":2746099,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T23:09:27.681880677-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2746099} +{"time":"2025-03-08T23:09:27.681868417-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":36071,"Zone":""}} +{"time":"2025-03-08T23:09:27.799952728-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:52546"} +{"time":"2025-03-08T23:09:28.396330791-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"emw0owd9","id":"127.0.0.1:52546"} +{"time":"2025-03-08T23:09:28.623763909-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"emw0owd9","id":"127.0.0.1:52546"} +{"time":"2025-03-08T23:09:44.656925069-06:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:52546"} +{"time":"2025-03-08T23:09:44.657278166-06:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-08T23:09:44.657256047-06:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:52546"} +{"time":"2025-03-08T23:09:44.657370346-06:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:52546"} +{"time":"2025-03-08T23:09:44.658878404-06:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:52546"} +{"time":"2025-03-08T23:09:44.658890184-06:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:52546"} +{"time":"2025-03-08T23:09:44.658897374-06:00","level":"INFO","msg":"server is closed"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230927-emw0owd9/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230927-emw0owd9/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..29e3e6e7d65e5ed8c139ce0748e35251b6b6b5bb --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230927-emw0owd9/logs/debug-internal.log @@ -0,0 +1,14 @@ +{"time":"2025-03-08T23:09:28.509240791-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230927-emw0owd9/logs/debug-core.log"} +{"time":"2025-03-08T23:09:28.623726469-06:00","level":"INFO","msg":"created new stream","id":"emw0owd9"} +{"time":"2025-03-08T23:09:28.623756039-06:00","level":"INFO","msg":"stream: started","id":"emw0owd9"} +{"time":"2025-03-08T23:09:28.623786149-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"emw0owd9"} +{"time":"2025-03-08T23:09:28.623819589-06:00","level":"INFO","msg":"handler: started","stream_id":"emw0owd9"} +{"time":"2025-03-08T23:09:28.623867448-06:00","level":"INFO","msg":"sender: started","stream_id":"emw0owd9"} +{"time":"2025-03-08T23:09:28.630116699-06:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-08T23:09:44.657303796-06:00","level":"INFO","msg":"stream: closing","id":"emw0owd9"} +{"time":"2025-03-08T23:09:44.657385276-06:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-08T23:09:44.65806042-06:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-08T23:09:44.65816433-06:00","level":"INFO","msg":"handler: closed","stream_id":"emw0owd9"} +{"time":"2025-03-08T23:09:44.658179509-06:00","level":"INFO","msg":"writer: Close: closed","stream_id":"emw0owd9"} +{"time":"2025-03-08T23:09:44.658196279-06:00","level":"INFO","msg":"sender: closed","stream_id":"emw0owd9"} +{"time":"2025-03-08T23:09:44.658778515-06:00","level":"INFO","msg":"stream: closed","id":"emw0owd9"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230927-emw0owd9/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230927-emw0owd9/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230927-emw0owd9/run-emw0owd9.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230927-emw0owd9/run-emw0owd9.wandb new file mode 100644 index 0000000000000000000000000000000000000000..b024da215444379f350fb9e98ae848b03d2256ab Binary files /dev/null and b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_230927-emw0owd9/run-emw0owd9.wandb differ diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231032-1lwxoay7/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231032-1lwxoay7/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231032-1lwxoay7/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231032-1lwxoay7/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231032-1lwxoay7/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231032-1lwxoay7/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231032-1lwxoay7/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..523e762a92063a3b9733d61079605e066214717f --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231032-1lwxoay7/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T05:10:32.793644Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128058331136" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231032-1lwxoay7/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231032-1lwxoay7/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..39bd6dd7b7d6518043367b2fbdaddab86aa36078 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231032-1lwxoay7/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-08T23:10:32.623405964-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmpeoocvo1o/port-2747773.txt","pid":2747773,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T23:10:32.625635957-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2747773} +{"time":"2025-03-08T23:10:32.625623007-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":38533,"Zone":""}} +{"time":"2025-03-08T23:10:32.783354908-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:37598"} +{"time":"2025-03-08T23:10:32.794977488-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"1lwxoay7","id":"127.0.0.1:37598"} +{"time":"2025-03-08T23:10:32.960596607-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"1lwxoay7","id":"127.0.0.1:37598"} +{"time":"2025-03-08T23:10:50.283883005-06:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:37598"} +{"time":"2025-03-08T23:10:50.28457055-06:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:37598"} +{"time":"2025-03-08T23:10:50.284632599-06:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-08T23:10:50.284705119-06:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:37598"} +{"time":"2025-03-08T23:10:50.287609706-06:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:37598"} +{"time":"2025-03-08T23:10:50.287628906-06:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:37598"} +{"time":"2025-03-08T23:10:50.287644176-06:00","level":"INFO","msg":"server is closed"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231032-1lwxoay7/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231032-1lwxoay7/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..1fc0d0e27faba4b3c969a50fbe1bc01bbdd73fb3 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231032-1lwxoay7/logs/debug-internal.log @@ -0,0 +1,14 @@ +{"time":"2025-03-08T23:10:32.82799985-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231032-1lwxoay7/logs/debug-core.log"} +{"time":"2025-03-08T23:10:32.960562757-06:00","level":"INFO","msg":"created new stream","id":"1lwxoay7"} +{"time":"2025-03-08T23:10:32.960590327-06:00","level":"INFO","msg":"stream: started","id":"1lwxoay7"} +{"time":"2025-03-08T23:10:32.960617597-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"1lwxoay7"} +{"time":"2025-03-08T23:10:32.960625247-06:00","level":"INFO","msg":"handler: started","stream_id":"1lwxoay7"} +{"time":"2025-03-08T23:10:32.960641147-06:00","level":"INFO","msg":"sender: started","stream_id":"1lwxoay7"} +{"time":"2025-03-08T23:10:32.966072925-06:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-08T23:10:50.284634329-06:00","level":"INFO","msg":"stream: closing","id":"1lwxoay7"} +{"time":"2025-03-08T23:10:50.284743108-06:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-08T23:10:50.285484112-06:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-08T23:10:50.285550762-06:00","level":"INFO","msg":"handler: closed","stream_id":"1lwxoay7"} +{"time":"2025-03-08T23:10:50.285560132-06:00","level":"INFO","msg":"writer: Close: closed","stream_id":"1lwxoay7"} +{"time":"2025-03-08T23:10:50.285569762-06:00","level":"INFO","msg":"sender: closed","stream_id":"1lwxoay7"} +{"time":"2025-03-08T23:10:50.287469127-06:00","level":"INFO","msg":"stream: closed","id":"1lwxoay7"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231032-1lwxoay7/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231032-1lwxoay7/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231032-1lwxoay7/run-1lwxoay7.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231032-1lwxoay7/run-1lwxoay7.wandb new file mode 100644 index 0000000000000000000000000000000000000000..1c2f20dd0680238623c1a902d074294316d1111f Binary files /dev/null and b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231032-1lwxoay7/run-1lwxoay7.wandb differ diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231237-q24cuc1j/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231237-q24cuc1j/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231237-q24cuc1j/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231237-q24cuc1j/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231237-q24cuc1j/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231237-q24cuc1j/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231237-q24cuc1j/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..4436ff388da3702727997480a86f0569f0c24554 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231237-q24cuc1j/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T05:12:38.596827Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128059252736" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231237-q24cuc1j/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231237-q24cuc1j/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..b8eab0802fd212894e8afb0f82337ba41c85069f --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231237-q24cuc1j/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-08T23:12:38.193916912-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmpnslqensz/port-2750759.txt","pid":2750759,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T23:12:38.197137167-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2750759} +{"time":"2025-03-08T23:12:38.197129097-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":36639,"Zone":""}} +{"time":"2025-03-08T23:12:38.290150223-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:58014"} +{"time":"2025-03-08T23:12:38.5998949-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"q24cuc1j","id":"127.0.0.1:58014"} +{"time":"2025-03-08T23:12:38.921608665-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"q24cuc1j","id":"127.0.0.1:58014"} +{"time":"2025-03-08T23:12:59.162690142-06:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:58014"} +{"time":"2025-03-08T23:12:59.212572423-06:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:58014"} +{"time":"2025-03-08T23:12:59.212662643-06:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:58014"} +{"time":"2025-03-08T23:12:59.212686172-06:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-08T23:12:59.311301045-06:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:58014"} +{"time":"2025-03-08T23:12:59.311316125-06:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:58014"} +{"time":"2025-03-08T23:12:59.311323095-06:00","level":"INFO","msg":"server is closed"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231237-q24cuc1j/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231237-q24cuc1j/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..e00f3dd1fd9f236d0a8ad4d054e59088598cf750 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231237-q24cuc1j/logs/debug-internal.log @@ -0,0 +1,14 @@ +{"time":"2025-03-08T23:12:38.777453457-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231237-q24cuc1j/logs/debug-core.log"} +{"time":"2025-03-08T23:12:38.921564085-06:00","level":"INFO","msg":"created new stream","id":"q24cuc1j"} +{"time":"2025-03-08T23:12:38.921600915-06:00","level":"INFO","msg":"stream: started","id":"q24cuc1j"} +{"time":"2025-03-08T23:12:38.921619285-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"q24cuc1j"} +{"time":"2025-03-08T23:12:38.921646784-06:00","level":"INFO","msg":"sender: started","stream_id":"q24cuc1j"} +{"time":"2025-03-08T23:12:38.921659834-06:00","level":"INFO","msg":"handler: started","stream_id":"q24cuc1j"} +{"time":"2025-03-08T23:12:38.936421899-06:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-08T23:12:59.212809091-06:00","level":"INFO","msg":"stream: closing","id":"q24cuc1j"} +{"time":"2025-03-08T23:12:59.21293607-06:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-08T23:12:59.235254997-06:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-08T23:12:59.235353256-06:00","level":"INFO","msg":"handler: closed","stream_id":"q24cuc1j"} +{"time":"2025-03-08T23:12:59.235365606-06:00","level":"INFO","msg":"writer: Close: closed","stream_id":"q24cuc1j"} +{"time":"2025-03-08T23:12:59.235396206-06:00","level":"INFO","msg":"sender: closed","stream_id":"q24cuc1j"} +{"time":"2025-03-08T23:12:59.311188926-06:00","level":"INFO","msg":"stream: closed","id":"q24cuc1j"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231237-q24cuc1j/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231237-q24cuc1j/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231237-q24cuc1j/run-q24cuc1j.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231237-q24cuc1j/run-q24cuc1j.wandb new file mode 100644 index 0000000000000000000000000000000000000000..63cf91ae0c8659bcc956110e85b7cf7d18a9bc5b Binary files /dev/null and b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_231237-q24cuc1j/run-q24cuc1j.wandb differ diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232036-kfyi93x7/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232036-kfyi93x7/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232036-kfyi93x7/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232036-kfyi93x7/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232036-kfyi93x7/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232036-kfyi93x7/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232036-kfyi93x7/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..2219fc6718fbd2e8d285e881aa0849001fca4f5a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232036-kfyi93x7/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T05:20:36.607833Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128062255104" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232036-kfyi93x7/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232036-kfyi93x7/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..346663e67a1b4566e4c6ba7bd022365d40966dab --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232036-kfyi93x7/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-08T23:20:36.464628707-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmpm6fsven7/port-2761346.txt","pid":2761346,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T23:20:36.467322296-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2761346} +{"time":"2025-03-08T23:20:36.467287856-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":43855,"Zone":""}} +{"time":"2025-03-08T23:20:36.571127257-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:59000"} +{"time":"2025-03-08T23:20:36.609286651-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"kfyi93x7","id":"127.0.0.1:59000"} +{"time":"2025-03-08T23:20:36.875710477-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"kfyi93x7","id":"127.0.0.1:59000"} +{"time":"2025-03-08T23:20:53.910225096-06:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:59000"} +{"time":"2025-03-08T23:20:53.910756801-06:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:59000"} +{"time":"2025-03-08T23:20:53.910804531-06:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-08T23:20:53.91088892-06:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:59000"} +{"time":"2025-03-08T23:20:53.913211752-06:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:59000"} +{"time":"2025-03-08T23:20:53.913221812-06:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:59000"} +{"time":"2025-03-08T23:20:53.913228122-06:00","level":"INFO","msg":"server is closed"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232036-kfyi93x7/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232036-kfyi93x7/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6bcc4eb4e33c25b6a944dad2f3acee5dfbc5f6fb --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232036-kfyi93x7/logs/debug-internal.log @@ -0,0 +1,14 @@ +{"time":"2025-03-08T23:20:36.744832625-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232036-kfyi93x7/logs/debug-core.log"} +{"time":"2025-03-08T23:20:36.875680287-06:00","level":"INFO","msg":"created new stream","id":"kfyi93x7"} +{"time":"2025-03-08T23:20:36.875702907-06:00","level":"INFO","msg":"stream: started","id":"kfyi93x7"} +{"time":"2025-03-08T23:20:36.875715707-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"kfyi93x7"} +{"time":"2025-03-08T23:20:36.875726077-06:00","level":"INFO","msg":"handler: started","stream_id":"kfyi93x7"} +{"time":"2025-03-08T23:20:36.875784116-06:00","level":"INFO","msg":"sender: started","stream_id":"kfyi93x7"} +{"time":"2025-03-08T23:20:36.881347413-06:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-08T23:20:53.910842471-06:00","level":"INFO","msg":"stream: closing","id":"kfyi93x7"} +{"time":"2025-03-08T23:20:53.91095212-06:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-08T23:20:53.911883193-06:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-08T23:20:53.911953782-06:00","level":"INFO","msg":"handler: closed","stream_id":"kfyi93x7"} +{"time":"2025-03-08T23:20:53.912005352-06:00","level":"INFO","msg":"writer: Close: closed","stream_id":"kfyi93x7"} +{"time":"2025-03-08T23:20:53.912009622-06:00","level":"INFO","msg":"sender: closed","stream_id":"kfyi93x7"} +{"time":"2025-03-08T23:20:53.913124333-06:00","level":"INFO","msg":"stream: closed","id":"kfyi93x7"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232036-kfyi93x7/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232036-kfyi93x7/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232036-kfyi93x7/run-kfyi93x7.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232036-kfyi93x7/run-kfyi93x7.wandb new file mode 100644 index 0000000000000000000000000000000000000000..1304115b27a4063be6b9647bf95e5beb16ac34bb Binary files /dev/null and b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232036-kfyi93x7/run-kfyi93x7.wandb differ diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232712-px06whmc/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232712-px06whmc/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232712-px06whmc/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232712-px06whmc/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232712-px06whmc/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232712-px06whmc/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232712-px06whmc/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..131d826684fa1549797214712b99a1210a9dba5e --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232712-px06whmc/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T05:27:13.900446Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128064847872" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232712-px06whmc/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232712-px06whmc/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..ba8c94d6e4a74767dfec075d2579f3ec94a6aaa1 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232712-px06whmc/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-08T23:27:13.709916053-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmppf5bqzs_/port-2769305.txt","pid":2769305,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T23:27:13.713073128-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2769305} +{"time":"2025-03-08T23:27:13.713050369-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":41383,"Zone":""}} +{"time":"2025-03-08T23:27:13.857608364-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:56150"} +{"time":"2025-03-08T23:27:13.901830619-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"px06whmc","id":"127.0.0.1:56150"} +{"time":"2025-03-08T23:27:15.515947368-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"px06whmc","id":"127.0.0.1:56150"} +{"time":"2025-03-08T23:30:11.005446173-06:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:56150"} +{"time":"2025-03-08T23:30:11.006067848-06:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:56150"} +{"time":"2025-03-08T23:30:11.006106798-06:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-08T23:30:11.006223457-06:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:56150"} +{"time":"2025-03-08T23:30:11.009166594-06:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:56150"} +{"time":"2025-03-08T23:30:11.009200264-06:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:56150"} +{"time":"2025-03-08T23:30:11.009222844-06:00","level":"INFO","msg":"server is closed"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232712-px06whmc/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232712-px06whmc/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6a16ac1d1fe9ca92c3560bcc91ecac04a1eb9e27 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232712-px06whmc/logs/debug-internal.log @@ -0,0 +1,14 @@ +{"time":"2025-03-08T23:27:15.300041738-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232712-px06whmc/logs/debug-core.log"} +{"time":"2025-03-08T23:27:15.515848648-06:00","level":"INFO","msg":"created new stream","id":"px06whmc"} +{"time":"2025-03-08T23:27:15.515932508-06:00","level":"INFO","msg":"stream: started","id":"px06whmc"} +{"time":"2025-03-08T23:27:15.515973657-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"px06whmc"} +{"time":"2025-03-08T23:27:15.516054027-06:00","level":"INFO","msg":"sender: started","stream_id":"px06whmc"} +{"time":"2025-03-08T23:27:15.516123466-06:00","level":"INFO","msg":"handler: started","stream_id":"px06whmc"} +{"time":"2025-03-08T23:27:15.61459225-06:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-08T23:30:11.006114538-06:00","level":"INFO","msg":"stream: closing","id":"px06whmc"} +{"time":"2025-03-08T23:30:11.006289817-06:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-08T23:30:11.007585937-06:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-08T23:30:11.007772675-06:00","level":"INFO","msg":"handler: closed","stream_id":"px06whmc"} +{"time":"2025-03-08T23:30:11.007801535-06:00","level":"INFO","msg":"sender: closed","stream_id":"px06whmc"} +{"time":"2025-03-08T23:30:11.007792025-06:00","level":"INFO","msg":"writer: Close: closed","stream_id":"px06whmc"} +{"time":"2025-03-08T23:30:11.009005746-06:00","level":"INFO","msg":"stream: closed","id":"px06whmc"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232712-px06whmc/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232712-px06whmc/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232712-px06whmc/run-px06whmc.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232712-px06whmc/run-px06whmc.wandb new file mode 100644 index 0000000000000000000000000000000000000000..867fe6df89cefc43ebc599540e99c2ce2cc7ef74 Binary files /dev/null and b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_232712-px06whmc/run-px06whmc.wandb differ diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233024-lb5k8yk9/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233024-lb5k8yk9/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233024-lb5k8yk9/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233024-lb5k8yk9/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233024-lb5k8yk9/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233024-lb5k8yk9/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233024-lb5k8yk9/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..1c8230b9b4e452c7236226ba1f0edd6120b4f930 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233024-lb5k8yk9/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T05:30:25.154701Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128066154496" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233024-lb5k8yk9/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233024-lb5k8yk9/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..f0d5e4a485b0cb7c61dd24e3c49e74dc322a95e8 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233024-lb5k8yk9/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-08T23:30:24.859648585-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmp9u1frcma/port-2774024.txt","pid":2774024,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T23:30:24.861885868-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2774024} +{"time":"2025-03-08T23:30:24.861877588-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":35321,"Zone":""}} +{"time":"2025-03-08T23:30:24.946090302-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:54522"} +{"time":"2025-03-08T23:30:25.155999429-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"lb5k8yk9","id":"127.0.0.1:54522"} +{"time":"2025-03-08T23:30:25.361979956-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"lb5k8yk9","id":"127.0.0.1:54522"} +{"time":"2025-03-08T23:30:44.536068777-06:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:54522"} +{"time":"2025-03-08T23:30:44.536539393-06:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:54522"} +{"time":"2025-03-08T23:30:44.536562803-06:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-08T23:30:44.536616603-06:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:54522"} +{"time":"2025-03-08T23:30:44.53827125-06:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:54522"} +{"time":"2025-03-08T23:30:44.53828289-06:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:54522"} +{"time":"2025-03-08T23:30:44.53828873-06:00","level":"INFO","msg":"server is closed"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233024-lb5k8yk9/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233024-lb5k8yk9/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f535b4bda007fe63953654290c605ed826b14f45 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233024-lb5k8yk9/logs/debug-internal.log @@ -0,0 +1,14 @@ +{"time":"2025-03-08T23:30:25.24840508-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233024-lb5k8yk9/logs/debug-core.log"} +{"time":"2025-03-08T23:30:25.361955706-06:00","level":"INFO","msg":"created new stream","id":"lb5k8yk9"} +{"time":"2025-03-08T23:30:25.361974806-06:00","level":"INFO","msg":"stream: started","id":"lb5k8yk9"} +{"time":"2025-03-08T23:30:25.361998316-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"lb5k8yk9"} +{"time":"2025-03-08T23:30:25.362012346-06:00","level":"INFO","msg":"sender: started","stream_id":"lb5k8yk9"} +{"time":"2025-03-08T23:30:25.362016196-06:00","level":"INFO","msg":"handler: started","stream_id":"lb5k8yk9"} +{"time":"2025-03-08T23:30:25.368944202-06:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-08T23:30:44.536564743-06:00","level":"INFO","msg":"stream: closing","id":"lb5k8yk9"} +{"time":"2025-03-08T23:30:44.536623833-06:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-08T23:30:44.537275378-06:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-08T23:30:44.537337247-06:00","level":"INFO","msg":"handler: closed","stream_id":"lb5k8yk9"} +{"time":"2025-03-08T23:30:44.537344887-06:00","level":"INFO","msg":"writer: Close: closed","stream_id":"lb5k8yk9"} +{"time":"2025-03-08T23:30:44.537356747-06:00","level":"INFO","msg":"sender: closed","stream_id":"lb5k8yk9"} +{"time":"2025-03-08T23:30:44.538193011-06:00","level":"INFO","msg":"stream: closed","id":"lb5k8yk9"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233024-lb5k8yk9/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233024-lb5k8yk9/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233024-lb5k8yk9/run-lb5k8yk9.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233024-lb5k8yk9/run-lb5k8yk9.wandb new file mode 100644 index 0000000000000000000000000000000000000000..32aa7e10015673430cfdd0db8f389da942bcd914 Binary files /dev/null and b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233024-lb5k8yk9/run-lb5k8yk9.wandb differ diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233220-mwtwevur/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233220-mwtwevur/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233220-mwtwevur/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233220-mwtwevur/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233220-mwtwevur/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233220-mwtwevur/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233220-mwtwevur/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..77694477532559637baf41ed79631cdf36600702 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233220-mwtwevur/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T05:32:20.653088Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128067149824" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233220-mwtwevur/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233220-mwtwevur/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..6d9af2822ebdd40885088b7a6cb86149e2effa3a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233220-mwtwevur/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-08T23:32:20.445379097-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmp3ifhd9e6/port-2777210.txt","pid":2777210,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T23:32:20.447999757-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2777210} +{"time":"2025-03-08T23:32:20.447996647-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":35423,"Zone":""}} +{"time":"2025-03-08T23:32:20.624689222-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:58224"} +{"time":"2025-03-08T23:32:20.654144623-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"mwtwevur","id":"127.0.0.1:58224"} +{"time":"2025-03-08T23:32:20.793162801-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"mwtwevur","id":"127.0.0.1:58224"} +{"time":"2025-03-08T23:32:38.065860152-06:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:58224"} +{"time":"2025-03-08T23:32:38.066404778-06:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-08T23:32:38.066382978-06:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:58224"} +{"time":"2025-03-08T23:32:38.066563367-06:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:58224"} +{"time":"2025-03-08T23:32:38.068389562-06:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:58224"} +{"time":"2025-03-08T23:32:38.068400782-06:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:58224"} +{"time":"2025-03-08T23:32:38.068407852-06:00","level":"INFO","msg":"server is closed"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233220-mwtwevur/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233220-mwtwevur/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..1b0a8400c24b74279991c2f452b248c7f0259d08 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233220-mwtwevur/logs/debug-internal.log @@ -0,0 +1,14 @@ +{"time":"2025-03-08T23:32:20.666254929-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233220-mwtwevur/logs/debug-core.log"} +{"time":"2025-03-08T23:32:20.793132391-06:00","level":"INFO","msg":"created new stream","id":"mwtwevur"} +{"time":"2025-03-08T23:32:20.793155961-06:00","level":"INFO","msg":"stream: started","id":"mwtwevur"} +{"time":"2025-03-08T23:32:20.793190651-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"mwtwevur"} +{"time":"2025-03-08T23:32:20.793195541-06:00","level":"INFO","msg":"handler: started","stream_id":"mwtwevur"} +{"time":"2025-03-08T23:32:20.793217581-06:00","level":"INFO","msg":"sender: started","stream_id":"mwtwevur"} +{"time":"2025-03-08T23:32:20.799002235-06:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-08T23:32:38.066427978-06:00","level":"INFO","msg":"stream: closing","id":"mwtwevur"} +{"time":"2025-03-08T23:32:38.066587227-06:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-08T23:32:38.0674812-06:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-08T23:32:38.067549069-06:00","level":"INFO","msg":"handler: closed","stream_id":"mwtwevur"} +{"time":"2025-03-08T23:32:38.067560349-06:00","level":"INFO","msg":"writer: Close: closed","stream_id":"mwtwevur"} +{"time":"2025-03-08T23:32:38.067575009-06:00","level":"INFO","msg":"sender: closed","stream_id":"mwtwevur"} +{"time":"2025-03-08T23:32:38.068312673-06:00","level":"INFO","msg":"stream: closed","id":"mwtwevur"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233220-mwtwevur/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233220-mwtwevur/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233220-mwtwevur/run-mwtwevur.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233220-mwtwevur/run-mwtwevur.wandb new file mode 100644 index 0000000000000000000000000000000000000000..c007b792ac8ce2928a438e892f3a1f45ba4df109 Binary files /dev/null and b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233220-mwtwevur/run-mwtwevur.wandb differ diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/files/config.yaml b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3988f54331e7f5df0f35811bf81f5a6c70739177 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/files/config.yaml @@ -0,0 +1,40 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + python_version: 3.12.5 + cli_version: 0.19.8 + framework: huggingface + huggingface_version: 4.49.0 + is_jupyter_run: false + is_kaggle_kernel: true + start_time: 1741498443 + t: + 1: + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 55 + - 71 + - 75 + - 95 + - 98 + - 105 + 3: + - 4 + - 13 + - 23 + - 42 + - 55 + 4: 3.12.5 + 5: 0.19.8 + 6: 4.49.0 + 8: + - 2 + - 5 + 13: linux-x86_64 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/files/media/table/metrics_0_b10ef52130d62e780af7.table.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/files/media/table/metrics_0_b10ef52130d62e780af7.table.json new file mode 100644 index 0000000000000000000000000000000000000000..3ea41bbd3bae41a635ed5d502059c5e10075bfdd --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/files/media/table/metrics_0_b10ef52130d62e780af7.table.json @@ -0,0 +1 @@ +{"columns": ["probability_acc", "probability_macro-f1", "probability_confusion-matrix", "dataset", "split", "ts"], "data": [[0.4375, 0.4976190476190476, [[0, 1, 0, 0, 0, 0], [0, 1, 3, 0, 0, 0], [0, 1, 1, 2, 0, 0], [0, 0, 2, 2, 0, 0], [0, 0, 0, 0, 2, 0], [0, 0, 0, 0, 0, 1]], "offline:all_20k_uniform/choice/probability", "validation", 1.4197452310472727]]} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..4e8cc7e302443c4b96a31f9bbc488a327ca7657f --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/files/output.log @@ -0,0 +1,63 @@ +Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [00:11<00:00, 3.98s/it] +100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 16/16 [00:01<00:00, 11.69it/s] +100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:02<00:00, 2.14s/it] +[2025-03-08 23:34:18,040] (get_model:INFO) Loaded "llama2:13b-chat". +[2025-03-08 23:34:18,369] (get_peft_model_from_checkpoint:INFO) Loaded PEFT adapter 'query' checkpoint from 'cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000' +output_size: 10 +[2025-03-08 23:34:18,370] (main:INFO) output_size: 10 +[2025-03-08 23:34:18,377] (get_model:INFO) Loaded "mlp_probability". +[2025-03-08 23:34:18,384] (get_classifier_head:INFO) Loaded classifier model checkpoint from 'cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000'. +[2025-03-08 23:34:18,391] (get_dataset:INFO) root: /net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_datasets/llama2:13b-chat +[2025-03-08 23:34:18,392] (get_offline:INFO) root: /net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_datasets/llama2:13b-chat/offline/all_20k_uniform/choice/probability +[2025-03-08 23:34:18,393] (get_offline:INFO) root: /net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_datasets/llama2:13b-chat/offline/all_20k_uniform/choice/probability +[2025-03-08 23:34:18,394] (get_offline:INFO) data_files: {'train': ['/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_datasets/llama2:13b-chat/offline/all_20k_uniform/choice/probability/train/0.csv'], 'validation': ['/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_datasets/llama2:13b-chat/offline/all_20k_uniform/choice/probability/validation/0.csv']} +[2025-03-08 23:34:18,644] (get_dataset:INFO) Loaded "offline:all_20k_uniform/choice/probability"; train (N = 18534) / validation (N = 2000) +{'context': ["Premise:\nRugby player lifted to attempt a score against a red team defending.\n\nHypothesis:\nchess match turns violent\n\nChoices:\na: Yes\nb: It's impossible to say\nc: No"], 'target': ['c'], 'target_prompt': ['\nAnswer:'], 'output': [''], 'dataset_name': [''], 'probability_label': tensor([4], device='cuda:0')} +torch.Size([1]) +tensor([4], device='cuda:0') +{'context': ["Premise:\nPeople are throwing tomatoes at each other.\n\nHypothesis:\nThe people are having a food fight.\n\nChoices:\na: Yes\nb: It's impossible to say\nc: No"], 'target': ['a'], 'target_prompt': ['\nAnswer:'], 'output': ['a'], 'dataset_name': [''], 'probability_label': tensor([9], device='cuda:0')} +torch.Size([1]) +tensor([9], device='cuda:0') +{'context': ["Premise:\nA small ice cream stand with two people standing near it.\n\nHypothesis:\nTwo people selling ice cream from a car.\n\nChoices:\na: Yes\nb: It's impossible to say\nc: No"], 'target': ['c'], 'target_prompt': ['\nAnswer:'], 'output': [''], 'dataset_name': [''], 'probability_label': tensor([6], device='cuda:0')} +torch.Size([1]) +tensor([6], device='cuda:0') +{'context': ["Premise:\nAn old ban with a black hat and tie walks down a city street.\n\nHypothesis:\nThe cat has no owner.\n\nChoices:\na: Yes\nb: It's impossible to say\nc: No"], 'target': ['c'], 'target_prompt': ['\nAnswer:'], 'output': [''], 'dataset_name': [''], 'probability_label': tensor([5], device='cuda:0')} +torch.Size([1]) +tensor([5], device='cuda:0') +{'context': ["Premise:\nA boy rides on a camel in a crowded area while talking on his cellphone.\n\nHypothesis:\nA boy is riding on an elephant through the streets.\n\nChoices:\na: Yes\nb: It's impossible to say\nc: No"], 'target': ['c'], 'target_prompt': ['\nAnswer:'], 'output': [''], 'dataset_name': [''], 'probability_label': tensor([8], device='cuda:0')} +torch.Size([1]) +tensor([8], device='cuda:0') +{'context': ["Premise:\nTwo Asian children, a boy and a girl, the girl looking squarely at the camera and the boy making a face.\n\nHypothesis:\nthe children are taking model pictures\n\nChoices:\na: Yes\nb: It's impossible to say\nc: No"], 'target': ['b'], 'target_prompt': ['\nAnswer:'], 'output': [''], 'dataset_name': [''], 'probability_label': tensor([5], device='cuda:0')} +torch.Size([1]) +tensor([5], device='cuda:0') +{'context': ["Premise:\nTwo toddler girls appear to enjoy basking on a pile of leaves as they smile for the camera.\n\nHypothesis:\nTwo toddlers play on a pile of leaves.\n\nChoices:\na: Yes\nb: It's impossible to say\nc: No"], 'target': ['a'], 'target_prompt': ['\nAnswer:'], 'output': [''], 'dataset_name': [''], 'probability_label': tensor([5], device='cuda:0')} +torch.Size([1]) +tensor([5], device='cuda:0') +{'context': ["Premise:\nTwo guys and a woman standing and talking in a playground.\n\nHypothesis:\nThree adults are standing and thinking about space.\n\nChoices:\na: Yes\nb: It's impossible to say\nc: No"], 'target': ['b'], 'target_prompt': ['\nAnswer:'], 'output': [''], 'dataset_name': [''], 'probability_label': tensor([8], device='cuda:0')} +torch.Size([1]) +tensor([8], device='cuda:0') +{'context': ["Premise:\nTwo men play basketball.\n\nHypothesis:\nTwo men are competing for the last spot on the basketball team.\n\nChoices:\na: Yes\nb: It's impossible to say\nc: No"], 'target': ['b'], 'target_prompt': ['\nAnswer:'], 'output': [''], 'dataset_name': [''], 'probability_label': tensor([6], device='cuda:0')} +torch.Size([1]) +tensor([6], device='cuda:0') +{'context': ["Premise:\nTwo people are walking down an isolated road.\n\nHypothesis:\nTwo humans are walking toward their house.\n\nChoices:\na: Yes\nb: It's impossible to say\nc: No"], 'target': ['b'], 'target_prompt': ['\nAnswer:'], 'output': [''], 'dataset_name': [''], 'probability_label': tensor([7], device='cuda:0')} +torch.Size([1]) +tensor([7], device='cuda:0') +{'context': ["Premise:\nPeople are throwing tomatoes at each other.\n\nHypothesis:\nThe men are covered in tomatoes.\n\nChoices:\na: Yes\nb: It's impossible to say\nc: No"], 'target': ['b'], 'target_prompt': ['\nAnswer:'], 'output': [''], 'dataset_name': [''], 'probability_label': tensor([6], device='cuda:0')} +torch.Size([1]) +tensor([6], device='cuda:0') +{'context': ["Premise:\nOutdoors in front of a crowd, a man plays an instrument by blowing into pipes he holds up to his face.\n\nHypothesis:\nA person playing an instrument outside.\n\nChoices:\na: Yes\nb: It's impossible to say\nc: No"], 'target': ['a'], 'target_prompt': ['\nAnswer:'], 'output': [''], 'dataset_name': [''], 'probability_label': tensor([7], device='cuda:0')} +torch.Size([1]) +tensor([7], device='cuda:0') +{'context': ["Premise:\nA shirtless man wearing a black hat and a scarf works on an anvil.\n\nHypothesis:\nA man works with an anvil.\n\nChoices:\na: Yes\nb: It's impossible to say\nc: No"], 'target': ['a'], 'target_prompt': ['\nAnswer:'], 'output': [''], 'dataset_name': [''], 'probability_label': tensor([5], device='cuda:0')} +torch.Size([1]) +tensor([5], device='cuda:0') +{'context': ["Premise:\nBalding man in gray sweater plays with his dog.\n\nHypothesis:\nA man puts the sweater he knitted on a dog.\n\nChoices:\na: Yes\nb: It's impossible to say\nc: No"], 'target': ['c'], 'target_prompt': ['\nAnswer:'], 'output': [''], 'dataset_name': [''], 'probability_label': tensor([7], device='cuda:0')} +torch.Size([1]) +tensor([7], device='cuda:0') +{'context': ["Premise:\nA barefooted adolescent boy in a yellow shirt reaching the top of a small skateboarding ramp.\n\nHypothesis:\na boy is barefooted\n\nChoices:\na: Yes\nb: It's impossible to say\nc: No"], 'target': ['a'], 'target_prompt': ['\nAnswer:'], 'output': [''], 'dataset_name': [''], 'probability_label': tensor([7], device='cuda:0')} +torch.Size([1]) +tensor([7], device='cuda:0') +{'context': ["Premise:\nA pregnant lady singing on stage while holding a flag behind her.\n\nHypothesis:\nA woman is sick in bed.\n\nChoices:\na: Yes\nb: It's impossible to say\nc: No"], 'target': ['c'], 'target_prompt': ['\nAnswer:'], 'output': [''], 'dataset_name': [''], 'probability_label': tensor([6], device='cuda:0')} +torch.Size([1]) +tensor([6], device='cuda:0') +[2025-03-08 23:34:20,065] (save_metrics_data:INFO) Metrics data saved to "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/validation/classifier_data.bin". \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..7e9186b27ec8d2c80776d06043d9df60eb6031c1 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T05:34:03.441721Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128067973120" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/files/wandb-summary.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6ea5bc25ae5f5acb5133c3176b3a83975a403e3a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/files/wandb-summary.json @@ -0,0 +1 @@ +{"metrics": {"sha256": "b10ef52130d62e780af7d345b29e481758013398adc21a8f985e28b1fb62525b", "size": 356, "artifact_path": "wandb-client-artifact://tqokt9oe64qw856te70lqdopd30cwbgf3bgrrxxxk89hrc5jon9uqx45zut9f8svrid1mbk8i2t08i99ba3r3d3ehi0a1r6nhdvece18ub3k3rcor65s98l79apau4j8/metrics.table.json", "_latest_artifact_path": "wandb-client-artifact://lx1jxb4nyugw6z9izy6eei1ca9p36sbp37eakfxtg4p7wvca7km9zds7q728tte9dbc1zqxo9ojnkh6z07r6vkzo7w7cfclm97r7amnqr74p73zlihrakocm49co65qk:latest/metrics.table.json", "path": "media/table/metrics_0_b10ef52130d62e780af7.table.json", "ncols": 6, "nrows": 1, "_type": "table-file"}, "_timestamp": 1741498460.2923396, "_runtime": 16.851047473, "_step": 0} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..35faa3b57684853770aeb4093bd30e6a87bf1d82 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-08T23:34:03.148082834-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmpdm7f5ue3/port-2779973.txt","pid":2779973,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T23:34:03.150279197-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2779973} +{"time":"2025-03-08T23:34:03.150258717-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":38939,"Zone":""}} +{"time":"2025-03-08T23:34:03.335069728-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:40600"} +{"time":"2025-03-08T23:34:03.44536388-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"2f7ksbmf","id":"127.0.0.1:40600"} +{"time":"2025-03-08T23:34:03.594177702-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"2f7ksbmf","id":"127.0.0.1:40600"} +{"time":"2025-03-08T23:34:20.626080309-06:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:40600"} +{"time":"2025-03-08T23:34:20.626535906-06:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:40600"} +{"time":"2025-03-08T23:34:20.626574895-06:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-08T23:34:20.626688304-06:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:40600"} +{"time":"2025-03-08T23:34:20.628398241-06:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:40600"} +{"time":"2025-03-08T23:34:20.628421571-06:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:40600"} +{"time":"2025-03-08T23:34:20.628434681-06:00","level":"INFO","msg":"server is closed"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..f209b4a6e2311750e2771da0ab64170481883e66 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/logs/debug-internal.log @@ -0,0 +1,14 @@ +{"time":"2025-03-08T23:34:03.470695843-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/logs/debug-core.log"} +{"time":"2025-03-08T23:34:03.593816615-06:00","level":"INFO","msg":"created new stream","id":"2f7ksbmf"} +{"time":"2025-03-08T23:34:03.594160552-06:00","level":"INFO","msg":"stream: started","id":"2f7ksbmf"} +{"time":"2025-03-08T23:34:03.594226072-06:00","level":"INFO","msg":"handler: started","stream_id":"2f7ksbmf"} +{"time":"2025-03-08T23:34:03.594277891-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"2f7ksbmf"} +{"time":"2025-03-08T23:34:03.594322921-06:00","level":"INFO","msg":"sender: started","stream_id":"2f7ksbmf"} +{"time":"2025-03-08T23:34:03.607077682-06:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-08T23:34:20.626607475-06:00","level":"INFO","msg":"stream: closing","id":"2f7ksbmf"} +{"time":"2025-03-08T23:34:20.626698874-06:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-08T23:34:20.627407639-06:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-08T23:34:20.627503148-06:00","level":"INFO","msg":"handler: closed","stream_id":"2f7ksbmf"} +{"time":"2025-03-08T23:34:20.627516578-06:00","level":"INFO","msg":"writer: Close: closed","stream_id":"2f7ksbmf"} +{"time":"2025-03-08T23:34:20.627577048-06:00","level":"INFO","msg":"sender: closed","stream_id":"2f7ksbmf"} +{"time":"2025-03-08T23:34:20.628278102-06:00","level":"INFO","msg":"stream: closed","id":"2f7ksbmf"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/run-2f7ksbmf.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/run-2f7ksbmf.wandb new file mode 100644 index 0000000000000000000000000000000000000000..8ebeb718b9fe0e6622744788370fb952452e011b Binary files /dev/null and b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/run-2f7ksbmf.wandb differ diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/run-2f7ksbmf.wandb.synced b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233403-2f7ksbmf/run-2f7ksbmf.wandb.synced new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233926-lvxg4rg8/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233926-lvxg4rg8/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233926-lvxg4rg8/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233926-lvxg4rg8/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233926-lvxg4rg8/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233926-lvxg4rg8/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233926-lvxg4rg8/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c40b1556c5ab70eec5d228e2ed8b88ebfb88335b --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233926-lvxg4rg8/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T05:39:26.950810Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128070430720" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233926-lvxg4rg8/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233926-lvxg4rg8/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..0f32cb69f0f571c2dea7d78383c7520c6f04a34a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233926-lvxg4rg8/logs/debug-core.log @@ -0,0 +1,7 @@ +{"time":"2025-03-08T23:39:26.782050641-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmp9i4qafr8/port-2787947.txt","pid":2787947,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T23:39:26.785331044-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2787947} +{"time":"2025-03-08T23:39:26.785291074-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":39955,"Zone":""}} +{"time":"2025-03-08T23:39:26.937499025-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:35300"} +{"time":"2025-03-08T23:39:26.952103207-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"lvxg4rg8","id":"127.0.0.1:35300"} +{"time":"2025-03-08T23:39:27.116391093-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"lvxg4rg8","id":"127.0.0.1:35300"} +{"time":"2025-03-08T23:39:57.4054147-06:00","level":"INFO","msg":"Parent process exited, terminating service process."} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233926-lvxg4rg8/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233926-lvxg4rg8/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c99bd776280284b8ce233d0797386505a43f4102 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233926-lvxg4rg8/logs/debug-internal.log @@ -0,0 +1,7 @@ +{"time":"2025-03-08T23:39:26.977365601-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233926-lvxg4rg8/logs/debug-core.log"} +{"time":"2025-03-08T23:39:27.116329593-06:00","level":"INFO","msg":"created new stream","id":"lvxg4rg8"} +{"time":"2025-03-08T23:39:27.116377523-06:00","level":"INFO","msg":"stream: started","id":"lvxg4rg8"} +{"time":"2025-03-08T23:39:27.116403513-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"lvxg4rg8"} +{"time":"2025-03-08T23:39:27.116442583-06:00","level":"INFO","msg":"sender: started","stream_id":"lvxg4rg8"} +{"time":"2025-03-08T23:39:27.116500052-06:00","level":"INFO","msg":"handler: started","stream_id":"lvxg4rg8"} +{"time":"2025-03-08T23:39:27.131318943-06:00","level":"INFO","msg":"Starting system monitor"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233926-lvxg4rg8/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233926-lvxg4rg8/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233926-lvxg4rg8/run-lvxg4rg8.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233926-lvxg4rg8/run-lvxg4rg8.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e4d6faf18d7df7e2e723d88e16fd422193c20578 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_233926-lvxg4rg8/run-lvxg4rg8.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29049e6e91b59ab0eacd36579d4364e4640576d75bc77c0151fffde32658ddb4 +size 327680 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234002-arah433m/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234002-arah433m/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234002-arah433m/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234002-arah433m/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234002-arah433m/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234002-arah433m/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234002-arah433m/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..34b8b50cd03f55e2c567dcf65bd6daf26d133d28 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234002-arah433m/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T05:40:02.451486Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128070905856" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234002-arah433m/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234002-arah433m/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..74b7d2da0b74f4901962937d888d480e8659ca74 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234002-arah433m/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-08T23:40:02.290942866-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmpm6mnr7_9/port-2789508.txt","pid":2789508,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T23:40:02.293917419-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2789508} +{"time":"2025-03-08T23:40:02.293913289-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":35027,"Zone":""}} +{"time":"2025-03-08T23:40:02.444695869-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:51276"} +{"time":"2025-03-08T23:40:02.454687313-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"arah433m","id":"127.0.0.1:51276"} +{"time":"2025-03-08T23:40:02.595942464-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"arah433m","id":"127.0.0.1:51276"} +{"time":"2025-03-08T23:40:25.091133685-06:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:51276"} +{"time":"2025-03-08T23:40:25.091587552-06:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:51276"} +{"time":"2025-03-08T23:40:25.091609442-06:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-08T23:40:25.091683122-06:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:51276"} +{"time":"2025-03-08T23:40:25.093612151-06:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:51276"} +{"time":"2025-03-08T23:40:25.093625751-06:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:51276"} +{"time":"2025-03-08T23:40:25.093633371-06:00","level":"INFO","msg":"server is closed"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234002-arah433m/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234002-arah433m/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b407edcd01f6f874ed8584c6d8134594b7bbcebc --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234002-arah433m/logs/debug-internal.log @@ -0,0 +1,14 @@ +{"time":"2025-03-08T23:40:02.467899361-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234002-arah433m/logs/debug-core.log"} +{"time":"2025-03-08T23:40:02.595901334-06:00","level":"INFO","msg":"created new stream","id":"arah433m"} +{"time":"2025-03-08T23:40:02.595933794-06:00","level":"INFO","msg":"stream: started","id":"arah433m"} +{"time":"2025-03-08T23:40:02.595957414-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"arah433m"} +{"time":"2025-03-08T23:40:02.595965584-06:00","level":"INFO","msg":"handler: started","stream_id":"arah433m"} +{"time":"2025-03-08T23:40:02.596003953-06:00","level":"INFO","msg":"sender: started","stream_id":"arah433m"} +{"time":"2025-03-08T23:40:02.603000985-06:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-08T23:40:25.091655182-06:00","level":"INFO","msg":"stream: closing","id":"arah433m"} +{"time":"2025-03-08T23:40:25.091725311-06:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-08T23:40:25.092693936-06:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-08T23:40:25.092782265-06:00","level":"INFO","msg":"handler: closed","stream_id":"arah433m"} +{"time":"2025-03-08T23:40:25.092800385-06:00","level":"INFO","msg":"writer: Close: closed","stream_id":"arah433m"} +{"time":"2025-03-08T23:40:25.092823945-06:00","level":"INFO","msg":"sender: closed","stream_id":"arah433m"} +{"time":"2025-03-08T23:40:25.093531281-06:00","level":"INFO","msg":"stream: closed","id":"arah433m"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234002-arah433m/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234002-arah433m/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234002-arah433m/run-arah433m.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234002-arah433m/run-arah433m.wandb new file mode 100644 index 0000000000000000000000000000000000000000..4f8da3f2e988fba0d923ea1bb4ae73d273dfdbca Binary files /dev/null and b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234002-arah433m/run-arah433m.wandb differ diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234031-htonl59j/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234031-htonl59j/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234031-htonl59j/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234031-htonl59j/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234031-htonl59j/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234031-htonl59j/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234031-htonl59j/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..434824e4fc0b55d1dc41a806fde648ffde340612 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234031-htonl59j/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T05:40:31.416033Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128071352320" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234031-htonl59j/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234031-htonl59j/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..8d42fe482b701828d24312d8f931f2a49323caa6 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234031-htonl59j/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-08T23:40:31.234101515-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmp8mzvepdl/port-2790690.txt","pid":2790690,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T23:40:31.236135403-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2790690} +{"time":"2025-03-08T23:40:31.236129243-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":41045,"Zone":""}} +{"time":"2025-03-08T23:40:31.406593416-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:60794"} +{"time":"2025-03-08T23:40:31.417851402-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"htonl59j","id":"127.0.0.1:60794"} +{"time":"2025-03-08T23:40:31.56956271-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"htonl59j","id":"127.0.0.1:60794"} +{"time":"2025-03-08T23:41:39.825002067-06:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:60794"} +{"time":"2025-03-08T23:41:39.825602243-06:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:60794"} +{"time":"2025-03-08T23:41:39.825689513-06:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:60794"} +{"time":"2025-03-08T23:41:39.825712493-06:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-08T23:41:39.828472377-06:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:60794"} +{"time":"2025-03-08T23:41:39.828505237-06:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:60794"} +{"time":"2025-03-08T23:41:39.828525287-06:00","level":"INFO","msg":"server is closed"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234031-htonl59j/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234031-htonl59j/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..60efba845662de7a9951802abbcb99dedd86d8d5 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234031-htonl59j/logs/debug-internal.log @@ -0,0 +1,14 @@ +{"time":"2025-03-08T23:40:31.437447453-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234031-htonl59j/logs/debug-core.log"} +{"time":"2025-03-08T23:40:31.569506651-06:00","level":"INFO","msg":"created new stream","id":"htonl59j"} +{"time":"2025-03-08T23:40:31.56955304-06:00","level":"INFO","msg":"stream: started","id":"htonl59j"} +{"time":"2025-03-08T23:40:31.56957675-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"htonl59j"} +{"time":"2025-03-08T23:40:31.56970056-06:00","level":"INFO","msg":"sender: started","stream_id":"htonl59j"} +{"time":"2025-03-08T23:40:31.569856239-06:00","level":"INFO","msg":"handler: started","stream_id":"htonl59j"} +{"time":"2025-03-08T23:40:31.577437746-06:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-08T23:41:39.825563724-06:00","level":"INFO","msg":"stream: closing","id":"htonl59j"} +{"time":"2025-03-08T23:41:39.825725883-06:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-08T23:41:39.826764137-06:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-08T23:41:39.826826806-06:00","level":"INFO","msg":"handler: closed","stream_id":"htonl59j"} +{"time":"2025-03-08T23:41:39.826838136-06:00","level":"INFO","msg":"writer: Close: closed","stream_id":"htonl59j"} +{"time":"2025-03-08T23:41:39.826851246-06:00","level":"INFO","msg":"sender: closed","stream_id":"htonl59j"} +{"time":"2025-03-08T23:41:39.828272538-06:00","level":"INFO","msg":"stream: closed","id":"htonl59j"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234031-htonl59j/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234031-htonl59j/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234031-htonl59j/run-htonl59j.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234031-htonl59j/run-htonl59j.wandb new file mode 100644 index 0000000000000000000000000000000000000000..29baa7affc0defaa31d91846d182adfec66169da --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234031-htonl59j/run-htonl59j.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adba9a0c0aeeb38d94e0cc08b0999882df16409a8fe743ae947ba229a1ec7581 +size 143289 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234033-nuxm0lk5/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234033-nuxm0lk5/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234033-nuxm0lk5/files/requirements.txt b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234033-nuxm0lk5/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..31f4ada245ad36ee2a33aa004076b2234bbc044a --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234033-nuxm0lk5/files/requirements.txt @@ -0,0 +1,527 @@ +userpath==1.9.2 +pipx==1.7.1 +argcomplete==3.5.2 +kaleido==0.2.1 +litellm==1.42.5 +nvidia-nvtx-cu12==12.4.127 +typing-inspect==0.9.0 +nnsight==0.3.7.dev14+g7d9bdaf +python-dotenv==1.0.1 +python-dotenv==0.21.0 +traitlets==5.14.3 +colorama==0.4.6 +llvmlite==0.43.0 +zstandard==0.23.0 +zstandard==0.22.0 +regex==2024.11.6 +zipp==3.20.0 +zipp==3.21.0 +eindex-callum==0.1.2 +nvidia-cudnn-cu12==9.1.0.70 +sse-starlette==1.8.2 +babe==0.0.7 +plotly==5.24.1 +kiwisolver==1.4.8 +boostedblob==0.15.4 +filelock==3.16.1 +filelock==3.15.4 +ruff==0.6.8 +mpmath==1.3.0 +uri-template==1.3.0 +automated-interpretability==0.0.6 +scipy==1.14.1 +webencodings==0.5.1 +gdown==5.2.0 +executing==2.0.1 +python-dateutil==2.9.0.post0 +python-dateutil==2.9.0 +blessed==1.20.0 +nvidia-curand-cu12==10.3.5.147 +tqdm==4.67.0 +tqdm==4.66.5 +qtconsole==5.5.2 +toolwrapper==2.1.0 +ninja==1.11.1.1 +uc-micro-py==1.0.3 +uvloop==0.20.0 +line_profiler==4.1.3 +absl-py==2.1.0 +types-PyYAML==6.0.12.20240917 +yarl==1.9.4 +pandas==2.2.3 +nvidia-cublas-cu12==12.4.5.8 +importlib_resources==6.4.4 +importlib_resources==6.4.0 +iniconfig==2.0.0 +mypy-extensions==1.0.0 +notebook_shim==0.2.4 +gprof2dot==2024.6.6 +fqdn==1.5.1 +delphi==0.2.0 +simple-parsing==0.1.6 +multiprocess==0.70.16 +pybind11==2.13.5 +jupyter_core==5.7.2 +GitPython==3.1.43 +sentry-sdk==2.18.0 +eval_type_backport==0.2.2 +bitsandbytes==0.44.1 +Jinja2==3.1.4 +beartype==0.14.1 +jiwer==3.0.4 +pdm==2.22.3 +blinker==1.9.0 +huggingface-hub==0.26.2 +mergekit==0.0.4.4 +chardet==5.2.0 +chardet==4.0.0 +torch==2.5.1 +itsdangerous==2.2.0 +llm-calibration==0.1.0 +rouge_score==0.1.2 +torchaudio==2.5.1 +i2==0.1.28 +python-slugify==8.0.4 +python-socketio==5.11.3 +pip==24.2 +altair==5.4.1 +jsonpatch==1.33 +text-unidecode==1.3 +shtab==1.7.1 +tensorboard==2.18.0 +dnspython==2.7.0 +linkify-it-py==2.0.3 +doit==0.36.0 +websockets==11.0.3 +nvidia-nvjitlink-cu12==12.4.127 +hydra-colorlog==1.2.0 +thefuzz==0.22.1 +pandocfilters==1.5.1 +tqdm-multiprocess==0.0.11 +mistune==3.0.2 +virtualenv==20.28.0 +tiktoken==0.7.0 +intel-cmplr-lic-rt==2025.0.4 +pure_eval==0.2.3 +dpctl==0.18.3 +lightning==2.4.0 +nvidia-cusparse-cu12==12.3.1.170 +python-multipart==0.0.12 +grpcio==1.66.2 +arrow==1.3.0 +marshmallow==3.22.0 +xformers==0.0.28.post3 +dataclasses-json==0.6.7 +pydub==0.25.1 +word2number==1.1 +markdown-it-py==2.2.0 +gguf==0.10.0 +docker-pycreds==0.4.0 +comm==0.2.2 +langchain-text-splitters==0.3.0 +jupyter-console==6.6.3 +transformers==4.49.0 +fastjsonschema==2.20.0 +ptyprocess==0.7.0 +hf_transfer==0.1.8 +jupyter_client==8.6.2 +pfzy==0.3.4 +installer==0.7.0 +s3transfer==0.10.2 +tcolorpy==0.1.6 +gitdb==4.0.11 +requests==2.32.3 +platformdirs==4.2.2 +platformdirs==3.10.0 +boto3==1.35.6 +six==1.17.0 +anyio==4.8.0 +anyio==4.6.2 +wandb==0.19.8 +dol==0.2.65 +torchmetrics==1.4.1 +python-engineio==4.9.1 +gql==3.5.0 +wcwidth==0.2.13 +lightning-utilities==0.11.6 +transformers-stream-generator==0.0.5 +jmespath==1.0.1 +better-abc==0.0.3 +typeguard==4.4.1 +pydantic==2.10.6 +Pygments==2.18.0 +Pygments==2.15.1 +pyarrow-hotfix==0.6 +tensorboard-data-server==0.7.2 +nltk==3.9.1 +openai==1.54.4 +cycler==0.12.1 +sae_auto_interp==0.1.0 +ipykernel==6.29.5 +python-json-logger==2.0.7 +fancy-einsum==0.0.3 +pyairports==2.1.1 +ray==2.34.0 +pycparser==2.22 +pycparser==2.21 +textgrad==0.1.5 +spacecutter==0.2.1 +nvidia-nccl-cu12==2.21.5 +sentencepiece==0.2.0 +pytorch-lightning==2.4.0 +analytics-python==1.2.9 +charset-normalizer==3.4.0 +widgetsnbextension==4.0.13 +evaluate==0.4.2 +jupyter-lsp==2.2.5 +docopt==0.6.2 +scikit-dimension==0.3.4 +serpent==1.41 +rich==13.9.4 +safetensors==0.4.5 +sae_bench_template==0.1.0 +einops==0.8.0 +pycryptodomex==3.20.0 +tcmlib==1.2.0 +graze==0.1.24 +pydantic-settings==2.8.1 +pytorch-lightning-bolts==0.3.2.post1 +deepspeed==0.15.3 +cloudpickle==3.0.0 +prometheus_client==0.20.0 +optimum==1.23.2 +tornado==6.4.1 +nbconvert==7.16.4 +autograd==1.7.0 +mosestokenizer==1.2.1 +pexpect==4.9.0 +decorator==5.1.1 +together==1.4.1 +ctransformers==0.2.27 +inquirerpy==0.3.4 +typepy==1.3.2 +jsonlines==4.0.0 +dash-table==5.0.0 +langchain-community==0.3.0 +torchvision==0.20.1 +aiohttp==3.10.5 +rfc3986-validator==0.1.1 +h11==0.14.0 +distro==1.9.0 +scikit-learn-intelex==2025.0.1 +datadreamer.dev==0.38.0 +ring==0.10.1 +gpustat==1.1.1 +pathos==0.3.3 +jupyter-events==0.10.0 +botocore==1.35.6 +jsonschema-specifications==2023.12.1 +pyzmq==26.0.0 +dep-logic==0.4.10 +umf==0.9.1 +sae-lens==5.2.1 +transformer-lens==0.0.0 +xxhash==3.5.0 +rfc3339-validator==0.1.4 +importlib_metadata==8.4.0 +importlib_metadata==8.5.0 +tenacity==8.5.0 +requests-toolbelt==1.0.0 +PyYAML==6.0.2 +config2py==0.1.36 +argparse==1.4.0 +semantic-version==2.10.0 +parso==0.8.4 +babel==2.16.0 +resolvelib==1.1.0 +tinycss2==1.3.0 +jupyterlab_widgets==3.0.13 +pycountry==24.6.1 +intel-cmplr-lib-rt==2025.0.4 +h2==4.0.0 +Pyro5==5.15 +faiss-cpu==1.9.0 +debugpy==1.8.5 +pymongo==4.10.1 +numpy==2.2.3 +aiohappyeyeballs==2.4.0 +lark==1.2.2 +langchain-cli==0.0.35 +Fraction==2.2.0 +lm_eval==0.4.5 +lxml==4.9.4 +narwhals==1.29.0 +umap-learn==0.5.7 +socksio==1.0.0 +graphviz==0.20.3 +nvidia-cuda-cupti-cu12==12.4.127 +langdetect==1.0.9 +pox==0.3.5 +setuptools==75.5.0 +setuptools==75.1.0 +wheel==0.43.0 +lightning-bolts==0.7.0 +webcolors==24.8.0 +watchfiles==0.23.0 +accelerate==0.34.2 +click==8.1.7 +sae==0.1.0 +setfit==1.1.0 +pyarrow==16.1.0 +termcolor==2.4.0 +omegaconf==2.4.0.dev3 +docstring_parser==0.16 +hpack==4.0.0 +jaxtyping==0.2.36 +annotated-types==0.7.0 +MarkupSafe==3.0.2 +sqlitedict==2.1.0 +sentence-transformers==3.0.1 +simple-websocket==1.0.0 +nvidia-cufft-cu12==11.2.1.3 +black==25.1.0 +wirerope==0.4.7 +msgpack==1.1.0 +Markdown==3.7 +uvicorn==0.32.0 +langchain-openai==0.2.0 +more-itertools==10.4.0 +hydra-core==1.4.0.dev1 +tzdata==2025.1 +shellingham==1.5.4 +shellingham==1.5.0 +diffusers==0.30.1 +smmap==5.0.1 +librosa==0.10.2.post1 +Werkzeug==3.0.4 +numexpr==2.10.1 +nvidia-cusolver-cu12==11.6.1.9 +aioprocessing==2.0.1 +beautifulsoup4==4.12.3 +typing_extensions==4.12.2 +orjson==3.10.7 +litgpt==0.5.2 +langchain-experimental==0.3.0 +setproctitle==1.3.3 +pytest==8.3.2 +neuron_explainer==0.0.6 +findpython==0.6.2 +jupyter==1.0.0 +immutabledict==4.2.0 +pathvalidate==3.2.1 +janus==1.0.0 +uctools==1.3.0 +Send2Trash==1.8.3 +py-cpuinfo==9.0.0 +llm-swarm==0.1.1 +websocket-client==1.8.0 +terminado==0.18.1 +pydantic_core==2.27.2 +multidict==6.0.5 +pbs-installer==2025.2.12 +vllm==0.6.4.post1 +intel-sycl-rt==2025.0.4 +natsort==8.4.0 +langchain-core==0.3.1 +mord==0.7 +hydra_zen==0.14.0rc1 +backoff==2.2.1 +statsmodels==0.14.2 +dill==0.3.8 +cached-property==1.5.2 +PySocks==1.7.1 +sacrebleu==2.4.3 +nvidia-cuda-nvrtc-cu12==12.4.127 +circuitsvis==1.43.3 +tyro==0.5.18 +partial-json-parser==0.2.1.1.post4 +nest-asyncio==1.6.0 +retrying==1.3.4 +msgspec==0.18.6 +aiofiles==23.2.1 +sortedcontainers==2.4.0 +SQLAlchemy==2.0.34 +fonttools==4.56.0 +intel-cmplr-lib-ur==2025.0.4 +mistral_common==1.5.1 +loguru==0.7.2 +langsmith==0.1.121 +tbb==2022.0.0 +dash==2.18.2 +Flask==3.0.3 +hydra-submitit-launcher==1.2.0 +tokenizers==0.21.0 +packaging==24.2 +pytest-profiling==1.7.0 +json5==0.9.25 +wsproto==1.2.0 +contourpy==1.3.1 +weave==0.50.14 +pillow==11.1.0 +torchtyping==0.1.5 +hjson==3.1.0 +emoji==2.12.1 +lazy_loader==0.4 +immutables==0.20 +prometheus-fastapi-instrumentator==7.0.0 +DataProperty==1.0.1 +sniffio==1.3.1 +sniffio==1.3.0 +typeshed_client==2.7.0 +jupyter_server_terminals==0.5.3 +pytablewriter==1.2.0 +compressed-tensors==0.8.0 +RapidFuzz==3.10.0 +jupyter_server==2.14.2 +sympy==1.13.1 +psutil==6.0.0 +schedulefree==1.4 +types-python-dateutil==2.9.0.20240821 +httpx==0.27.0 +neovim==0.3.1 +gradio_client==1.3.0 +ollama==0.4.2 +rpds-py==0.20.0 +referencing==0.35.1 +langchain==0.3.0 +colorlog==6.8.2 +unearth==0.17.2 +dash-html-components==2.0.0 +outlines==0.0.46 +tabledata==1.3.3 +joblib==1.4.2 +nvidia-cuda-runtime-cu12==12.4.127 +sae-vis==0.2.21 +protobuf==3.20.3 +nbclient==0.10.0 +numba==0.60.0 +humanfriendly==10.0 +diskcache==5.6.3 +urllib3==2.2.3 +soupsieve==2.6 +tldr==3.3.0 +httpcore==1.0.7 +httpcore==1.0.5 +collectibles==0.1.5 +py2store==0.1.20 +hyperframe==6.0.1 +graphql-core==3.2.3 +hydra-joblib-launcher==1.2.0 +plotly-express==0.4.1 +distlib==0.3.8 +ffmpy==0.4.0 +jupyterlab==4.2.5 +stack-data==0.6.3 +gradio==3.37.0 +pathspec==0.10.3 +kaggle==1.6.17 +seaborn==0.13.2 +submitit==1.5.1 +bidict==0.23.1 +networkx==3.4.2 +lm-format-enforcer==0.10.9 +fsspec==2024.6.1 +mdurl==0.1.2 +mdurl==0.1.0 +human-eval==1.0.3 +jsonschema==4.23.0 +litserve==0.2.4 +antlr4-python3-runtime==4.9.3 +scikit-learn==1.6.0 +sparsify==0.1.0 +mdit-py-plugins==0.3.3 +trl==0.12.1 +jsonargparse==4.32.1 +threadpoolctl==3.5.0 +async-lru==2.0.4 +interegular==0.3.3 +opencv-python-headless==4.10.0.84 +portalocker==2.10.1 +fire==0.6.0 +vec-inf==0.3.3 +prompt_toolkit==3.0.47 +pynvim==0.5.2 +jupyterlab_server==2.27.3 +datasets==3.3.2 +typer==0.12.5 +alpaca_eval==0.6.5 +pytz==2025.1 +QtPy==2.4.1 +uuid_utils==0.9.0 +pygraphviz==1.14 +asttokens==2.4.1 +litdata==0.2.17 +notebook==7.2.2 +httptools==0.6.1 +jupyterlab_pygments==0.3.0 +skorch==1.1.0 +pyproject_hooks==1.0.0 +defusedxml==0.7.1 +fastapi==0.115.5 +nbformat==5.10.4 +liger_kernel==0.3.1 +overrides==7.7.0 +pynndescent==0.5.13 +LLMtuner==0.1.0 +pooch==1.8.2 +nbstripout==0.8.1 +dictionary-learning==0.1.0 +Brotli==1.1.0 +langserve==0.3.0 +wandb-workspaces==0.1.8 +isoduration==20.11.0 +lm-saes==0.1.0 +aiosignal==1.3.1 +matplotlib==3.10.1 +daal==2025.0.1 +starlette==0.41.3 +jiter==0.5.0 +langchain-huggingface==0.1.0 +jsonpointer==3.0.0 +flash-attn==2.6.3 +coloredlogs==15.0.1 +dash-core-components==2.0.0 +peft==0.14.0 +audioread==3.0.1 +patsy==0.5.6 +argon2-cffi-bindings==21.2.0 +asyncio==3.4.3 +attrs==24.2.0 +gritql==0.1.5 +IProgress==0.4 +jedi==0.19.1 +argon2-cffi==23.1.0 +greenlet==3.0.3 +greenlet==3.0.1 +pickleshare==0.7.5 +tensorboardX==2.6.2.2 +sae-bench==0.3.2 +pluggy==1.5.0 +matplotlib-inline==0.1.7 +pyparsing==3.2.1 +mbstrdecoder==1.1.3 +soxr==0.5.0 +triton==3.1.0 +nvidia-ml-py==12.560.30 +idna==3.10 +cffi==1.17.0 +cffi==1.17.1 +ipython==8.26.0 +dictionary_training==0.1.0 +frozenlist==1.4.1 +hishel==0.1.1 +certifi==2025.1.31 +bleach==6.1.0 +ppft==1.7.6.9 +vllm-flash-attn==2.6.1 +openfile==0.0.7 +ipywidgets==8.1.5 +blobfile==2.1.1 +soundfile==0.12.1 +tabulate==0.9.0 +wonderwords==2.2.0 +import-deps==0.3.0 +tomli==2.0.1 +tomlkit==0.13.2 +lm-saes==0.1.0 +bigcode_eval==0.0.0 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234033-nuxm0lk5/files/wandb-metadata.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234033-nuxm0lk5/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..decac24c8d10375fcfab88ddc0a2060687e80a5c --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234033-nuxm0lk5/files/wandb-metadata.json @@ -0,0 +1,54 @@ +{ + "os": "Linux-5.15.0-126-generic-x86_64-with-glibc2.35", + "python": "CPython 3.12.5", + "startedAt": "2025-03-09T05:40:34.256854Z", + "args": [ + "--data_dir=cached_datasets/llama2:13b-chat", + "--dataset=offline:all_20k_uniform/choice/probability", + "--prompt-style=choice", + "--model-name=llama2:13b-chat", + "--query-peft-dir=cached_models/llama2:13b-chat/all_20k_uniform/choice/probability/checkpoint-5000", + "--log-dir=cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "--mode=class_choice", + "--with-classifier" + ], + "program": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/experiments/evaluate.py", + "codePath": "experiments/evaluate.py", + "git": { + "remote": "https://github.com/activatedgeek/calibration-tuning", + "commit": "02861e6a66aaebfaabfe8f1d77b34ec952886966" + }, + "root": "/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability", + "host": "j004-ds", + "executable": "/net/scratch2/zlyu/conda/envs/312/bin/python", + "codePathLocal": "experiments/evaluate.py", + "cpu_count": 32, + "cpu_count_logical": 32, + "gpu": "NVIDIA A100 80GB PCIe", + "gpu_count": 1, + "disk": { + "/": { + "total": "157934215168", + "used": "128071405568" + } + }, + "memory": { + "total": "1082060558336" + }, + "cpu": { + "count": 32, + "countLogical": 32 + }, + "gpu_nvidia": [ + { + "name": "NVIDIA A100 80GB PCIe", + "memoryTotal": "85899345920", + "cudaCores": 6912, + "architecture": "Ampere" + } + ], + "slurm": { + "job_id": "391220" + }, + "cudaVersion": "12.4" +} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234033-nuxm0lk5/logs/debug-core.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234033-nuxm0lk5/logs/debug-core.log new file mode 100644 index 0000000000000000000000000000000000000000..45dc5a603425d98ba7566595485350fdd16fcb1c --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234033-nuxm0lk5/logs/debug-core.log @@ -0,0 +1,13 @@ +{"time":"2025-03-08T23:40:34.06502519-06:00","level":"INFO","msg":"main: starting server","port-filename":"/net/scratch2/zlyu/tmp/tmpop4pukkd/port-2791119.txt","pid":2791119,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false} +{"time":"2025-03-08T23:40:34.067475506-06:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2791119} +{"time":"2025-03-08T23:40:34.067473416-06:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":35461,"Zone":""}} +{"time":"2025-03-08T23:40:34.245904142-06:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:45942"} +{"time":"2025-03-08T23:40:34.258276523-06:00","level":"INFO","msg":"handleInformInit: received","streamId":"nuxm0lk5","id":"127.0.0.1:45942"} +{"time":"2025-03-08T23:40:34.420154652-06:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"nuxm0lk5","id":"127.0.0.1:45942"} +{"time":"2025-03-09T00:13:59.829964196-06:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:45942"} +{"time":"2025-03-09T00:13:59.830539941-06:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:45942"} +{"time":"2025-03-09T00:13:59.830647791-06:00","level":"INFO","msg":"server is shutting down"} +{"time":"2025-03-09T00:13:59.8307377-06:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:45942"} +{"time":"2025-03-09T00:13:59.833941704-06:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:45942"} +{"time":"2025-03-09T00:13:59.833993854-06:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:45942"} +{"time":"2025-03-09T00:13:59.834017043-06:00","level":"INFO","msg":"server is closed"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234033-nuxm0lk5/logs/debug-internal.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234033-nuxm0lk5/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..2605e60de9182b6ced420b162ffd384dda359801 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234033-nuxm0lk5/logs/debug-internal.log @@ -0,0 +1,14 @@ +{"time":"2025-03-08T23:40:34.304105455-06:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/net/scratch/zlyu/labs/llm_uncertainty/libs/calibration-tuning/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234033-nuxm0lk5/logs/debug-core.log"} +{"time":"2025-03-08T23:40:34.420077732-06:00","level":"INFO","msg":"created new stream","id":"nuxm0lk5"} +{"time":"2025-03-08T23:40:34.420145702-06:00","level":"INFO","msg":"stream: started","id":"nuxm0lk5"} +{"time":"2025-03-08T23:40:34.420246411-06:00","level":"INFO","msg":"handler: started","stream_id":"nuxm0lk5"} +{"time":"2025-03-08T23:40:34.420284291-06:00","level":"INFO","msg":"writer: Do: started","stream_id":"nuxm0lk5"} +{"time":"2025-03-08T23:40:34.420872158-06:00","level":"INFO","msg":"sender: started","stream_id":"nuxm0lk5"} +{"time":"2025-03-08T23:40:34.431652687-06:00","level":"INFO","msg":"Starting system monitor"} +{"time":"2025-03-09T00:13:59.830618761-06:00","level":"INFO","msg":"stream: closing","id":"nuxm0lk5"} +{"time":"2025-03-09T00:13:59.830827269-06:00","level":"INFO","msg":"Stopping system monitor"} +{"time":"2025-03-09T00:13:59.832395757-06:00","level":"INFO","msg":"Stopped system monitor"} +{"time":"2025-03-09T00:13:59.832611785-06:00","level":"INFO","msg":"handler: closed","stream_id":"nuxm0lk5"} +{"time":"2025-03-09T00:13:59.832635385-06:00","level":"INFO","msg":"writer: Close: closed","stream_id":"nuxm0lk5"} +{"time":"2025-03-09T00:13:59.832668354-06:00","level":"INFO","msg":"sender: closed","stream_id":"nuxm0lk5"} +{"time":"2025-03-09T00:13:59.833707636-06:00","level":"INFO","msg":"stream: closed","id":"nuxm0lk5"} diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234033-nuxm0lk5/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234033-nuxm0lk5/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234033-nuxm0lk5/run-nuxm0lk5.wandb b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234033-nuxm0lk5/run-nuxm0lk5.wandb new file mode 100644 index 0000000000000000000000000000000000000000..0d7ec5716e49860a2cfba782906ae3e30dcebbce --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234033-nuxm0lk5/run-nuxm0lk5.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82f4c23855ba8a5fa396e021b90fb663edfdb26fbbc7a9680a22e4b24edad169 +size 6916335 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234203-ga90r15d/logs/debug.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250308_234203-ga90r15d/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250309_001852-ojcexm6h/files/config.yaml b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250309_001852-ojcexm6h/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..afcb0c299a8aede4b166be231634ad6e15e1e609 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250309_001852-ojcexm6h/files/config.yaml @@ -0,0 +1,40 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + python_version: 3.12.5 + cli_version: 0.19.8 + framework: huggingface + huggingface_version: 4.49.0 + is_jupyter_run: false + is_kaggle_kernel: true + start_time: 1741501132 + t: + 1: + - 1 + - 5 + - 11 + - 41 + - 49 + - 51 + - 53 + - 55 + - 71 + - 75 + - 95 + - 98 + - 105 + 3: + - 4 + - 13 + - 23 + - 42 + - 55 + 4: 3.12.5 + 5: 0.19.8 + 6: 4.49.0 + 8: + - 2 + - 5 + 13: linux-x86_64 diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250309_001852-ojcexm6h/files/media/table/metrics_0_15e9c5d8dc7c9f5cd165.table.json b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250309_001852-ojcexm6h/files/media/table/metrics_0_15e9c5d8dc7c9f5cd165.table.json new file mode 100644 index 0000000000000000000000000000000000000000..4ca82890cab6cb23c9566d44ca001d799d5a28aa --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250309_001852-ojcexm6h/files/media/table/metrics_0_15e9c5d8dc7c9f5cd165.table.json @@ -0,0 +1 @@ +{"columns": ["probability_acc", "probability_macro-f1", "probability_confusion-matrix", "dataset", "split", "ts"], "data": [[0.7878493579367649, 0.5644259995927692, [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 1, 2, 0, 0, 0, 0, 0], [0, 0, 0, 37, 79, 20, 11, 2, 0, 0], [0, 0, 0, 1, 450, 209, 66, 13, 1, 1], [0, 0, 0, 4, 113, 1131, 427, 172, 40, 6], [0, 0, 0, 4, 24, 308, 1068, 517, 139, 12], [0, 0, 0, 0, 7, 77, 196, 1558, 481, 53], [0, 0, 0, 0, 0, 5, 45, 242, 2234, 374], [0, 0, 0, 0, 0, 1, 2, 13, 264, 8124]], "offline:all_20k_uniform/choice/probability", "train", 1886.1079187728465], [0.47, 0.44630341546069796, [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 5, 101, 37, 1, 0, 0], [0, 0, 0, 0, 2, 168, 264, 38, 0, 0], [0, 0, 0, 0, 0, 114, 181, 234, 3, 0], [0, 0, 0, 0, 0, 43, 36, 305, 105, 0], [0, 0, 0, 0, 0, 2, 8, 47, 261, 4], [0, 0, 0, 0, 0, 0, 0, 0, 21, 20]], "offline:all_20k_uniform/choice/probability", "validation", 102.36943727079779]]} \ No newline at end of file diff --git a/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250309_001852-ojcexm6h/files/output.log b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250309_001852-ojcexm6h/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..04f19db9beb1197b9e69176acdce3e5e3ba24418 --- /dev/null +++ b/cached_results/llama2:13b-chat/all_20k_uniform/choice/probability/wandb/offline-run-20250309_001852-ojcexm6h/files/output.log @@ -0,0 +1,18 @@ +Loading checkpoint shards: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 3/3 [00:15<00:00, 5.15s/it] + 0%| | 0/1 [00:00