new jr user study
Browse files- src/app.py +1 -1
- src/data.py +0 -1
- src/lsp_wrappers.py +1 -0
- src/ui/screens_model_comparison.py +2 -1
- src/upload.py +2 -1
- study_config.yaml +12 -8
src/app.py
CHANGED
|
@@ -47,7 +47,7 @@ def _init_submodule() -> None:
|
|
| 47 |
|
| 48 |
# GitHub serves a tarball of any branch/tag/SHA at this URL.
|
| 49 |
# Pinned to a specific commit SHA so future lsp changes don't break us.
|
| 50 |
-
branch = "
|
| 51 |
tarball_url = f"https://api.github.com/repos/batu-el/lsp/tarball/{branch}"
|
| 52 |
tmp_tar = Path("/tmp/lsp.tar.gz")
|
| 53 |
tmp_extract = Path("/tmp/lsp_extract")
|
|
|
|
| 47 |
|
| 48 |
# GitHub serves a tarball of any branch/tag/SHA at this URL.
|
| 49 |
# Pinned to a specific commit SHA so future lsp changes don't break us.
|
| 50 |
+
branch = "862a08160b07aed48238c7dadc191e138a00dc9a"
|
| 51 |
tarball_url = f"https://api.github.com/repos/batu-el/lsp/tarball/{branch}"
|
| 52 |
tmp_tar = Path("/tmp/lsp.tar.gz")
|
| 53 |
tmp_extract = Path("/tmp/lsp_extract")
|
src/data.py
CHANGED
|
@@ -727,7 +727,6 @@ def init_state(cfg: dict) -> dict:
|
|
| 727 |
"use_demographics": m.get("use_demographics", False),
|
| 728 |
"use_background": m.get("use_background", False),
|
| 729 |
"personalization": m.get("personalization"),
|
| 730 |
-
"detailed_instruction": m.get("detailed_instruction", True),
|
| 731 |
})
|
| 732 |
|
| 733 |
for i, slot in enumerate(slots):
|
|
|
|
| 727 |
"use_demographics": m.get("use_demographics", False),
|
| 728 |
"use_background": m.get("use_background", False),
|
| 729 |
"personalization": m.get("personalization"),
|
|
|
|
| 730 |
})
|
| 731 |
|
| 732 |
for i, slot in enumerate(slots):
|
src/lsp_wrappers.py
CHANGED
|
@@ -142,6 +142,7 @@ def build_seller_system_prompt_preference(
|
|
| 142 |
from prompts.seller_system.preference import get_seller_system_prompt
|
| 143 |
pv = cfg["prompt_variant"]
|
| 144 |
a, b = pair["product_a"], pair["product_b"]
|
|
|
|
| 145 |
result = get_seller_system_prompt(
|
| 146 |
personalization=pv["personalization"],
|
| 147 |
title=a.get("title"),
|
|
|
|
| 142 |
from prompts.seller_system.preference import get_seller_system_prompt
|
| 143 |
pv = cfg["prompt_variant"]
|
| 144 |
a, b = pair["product_a"], pair["product_b"]
|
| 145 |
+
# lsp API: only `personalization` is forwarded (no detailed_instruction / include_bio here).
|
| 146 |
result = get_seller_system_prompt(
|
| 147 |
personalization=pv["personalization"],
|
| 148 |
title=a.get("title"),
|
src/ui/screens_model_comparison.py
CHANGED
|
@@ -46,10 +46,11 @@ ETHICS_FINAL = (
|
|
| 46 |
def _prompt_variant_for_model(mconf: dict) -> dict:
|
| 47 |
use_d = mconf.get("use_demographics", False)
|
| 48 |
use_b = mconf.get("use_background", False)
|
|
|
|
|
|
|
| 49 |
return {
|
| 50 |
"personalization": mconf.get("personalization", bool(use_d or use_b)),
|
| 51 |
"include_bio": bool(mconf.get("use_background", use_b)),
|
| 52 |
-
"detailed_instruction": mconf.get("detailed_instruction", True),
|
| 53 |
}
|
| 54 |
|
| 55 |
|
|
|
|
| 46 |
def _prompt_variant_for_model(mconf: dict) -> dict:
|
| 47 |
use_d = mconf.get("use_demographics", False)
|
| 48 |
use_b = mconf.get("use_background", False)
|
| 49 |
+
# Only keys read by build_seller_system_prompt_preference β get_seller_system_prompt:
|
| 50 |
+
# `personalization`. (include_bio is unused there too; bio gated by use_background + format_demographics.)
|
| 51 |
return {
|
| 52 |
"personalization": mconf.get("personalization", bool(use_d or use_b)),
|
| 53 |
"include_bio": bool(mconf.get("use_background", use_b)),
|
|
|
|
| 54 |
}
|
| 55 |
|
| 56 |
|
src/upload.py
CHANGED
|
@@ -226,7 +226,8 @@ def _save_and_upload_csv(
|
|
| 226 |
"personalization",
|
| 227 |
mconf.get("use_demographics", False) or mconf.get("use_background", False),
|
| 228 |
),
|
| 229 |
-
|
|
|
|
| 230 |
}
|
| 231 |
rows.append([
|
| 232 |
submission_id,
|
|
|
|
| 226 |
"personalization",
|
| 227 |
mconf.get("use_demographics", False) or mconf.get("use_background", False),
|
| 228 |
),
|
| 229 |
+
# Not passed to lsp get_seller_system_prompt; optional YAML audit only.
|
| 230 |
+
"detailed_instruction": mconf.get("detailed_instruction", ""),
|
| 231 |
}
|
| 232 |
rows.append([
|
| 233 |
submission_id,
|
study_config.yaml
CHANGED
|
@@ -6,8 +6,8 @@
|
|
| 6 |
# "preference" : participants compare Product A vs Product B (7-pt preference scale)
|
| 7 |
# "likelihood" : participants evaluate a single product (7-pt likelihood-to-buy scale)
|
| 8 |
# "model_comparison" : one pair; same participant chats with multiple seller models
|
| 9 |
-
# (order randomized). Use pairs_per_user: 1
|
| 10 |
-
#
|
| 11 |
# study_type: preference
|
| 12 |
|
| 13 |
# Categories to include. Each entry needs a name and a count.
|
|
@@ -49,11 +49,11 @@ pair_selection_seed: 42 # Seed for reproducible 50-item pool selection p
|
|
| 49 |
# max_turns: 3 # Hard cap; input is disabled after this many exchanges
|
| 50 |
|
| 51 |
# Prolific
|
| 52 |
-
prolific_completion_code: "
|
| 53 |
prolific_study_id: "69fd27af45ed482b722bd5f1"
|
| 54 |
|
| 55 |
# HuggingFace dataset repo where results (JSON + CSV) are uploaded
|
| 56 |
-
output_dataset_repo: "ehejin/user_study-preference-
|
| 57 |
|
| 58 |
# ββ Example: model_comparison (uncomment and set study_type; comment out model_variants) ββ
|
| 59 |
#
|
|
@@ -71,11 +71,15 @@ comparison_models:
|
|
| 71 |
use_demographics: false
|
| 72 |
use_background: false
|
| 73 |
personalization: false
|
| 74 |
-
|
| 75 |
-
- name: finetuned_NP2
|
| 76 |
model_name: "meta-llama/Llama-3.1-8B-Instruct"
|
| 77 |
-
sampler_path: "tinker://
|
| 78 |
use_demographics: false
|
| 79 |
use_background: false
|
| 80 |
personalization: false
|
| 81 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6 |
# "preference" : participants compare Product A vs Product B (7-pt preference scale)
|
| 7 |
# "likelihood" : participants evaluate a single product (7-pt likelihood-to-buy scale)
|
| 8 |
# "model_comparison" : one pair; same participant chats with multiple seller models
|
| 9 |
+
# (order randomized). Use pairs_per_user: 1 and comparison_models: (name, model_name,
|
| 10 |
+
# sampler_path, use_demographics, use_background, personalization). Omit model_variants.
|
| 11 |
# study_type: preference
|
| 12 |
|
| 13 |
# Categories to include. Each entry needs a name and a count.
|
|
|
|
| 49 |
# max_turns: 3 # Hard cap; input is disabled after this many exchanges
|
| 50 |
|
| 51 |
# Prolific
|
| 52 |
+
prolific_completion_code: "CIE6CQV7"
|
| 53 |
prolific_study_id: "69fd27af45ed482b722bd5f1"
|
| 54 |
|
| 55 |
# HuggingFace dataset repo where results (JSON + CSV) are uploaded
|
| 56 |
+
output_dataset_repo: "ehejin/user_study-preference-personalized_0514_comparison_JR1_2"
|
| 57 |
|
| 58 |
# ββ Example: model_comparison (uncomment and set study_type; comment out model_variants) ββ
|
| 59 |
#
|
|
|
|
| 71 |
use_demographics: false
|
| 72 |
use_background: false
|
| 73 |
personalization: false
|
| 74 |
+
- name: finetuned_JR1
|
|
|
|
| 75 |
model_name: "meta-llama/Llama-3.1-8B-Instruct"
|
| 76 |
+
sampler_path: "tinker://12852bca-8673-59f1-9bf2-90438f1ceebe:train:0/weights/000030", "sampler_path": "tinker://12852bca-8673-59f1-9bf2-90438f1ceebe:train:0/sampler_weights/000030"
|
| 77 |
use_demographics: false
|
| 78 |
use_background: false
|
| 79 |
personalization: false
|
| 80 |
+
- name: finetuned_JR2
|
| 81 |
+
model_name: "meta-llama/Llama-3.1-8B-Instruct"
|
| 82 |
+
sampler_path: "tinker://5e6db03e-85d5-5d3c-95db-8c68e7718be1:train:0/sampler_weights/000120"
|
| 83 |
+
use_demographics: false
|
| 84 |
+
use_background: false
|
| 85 |
+
personalization: false
|