ehejin commited on
Commit
b58a0fc
Β·
1 Parent(s): 6415d03

new jr user study

Browse files
src/app.py CHANGED
@@ -47,7 +47,7 @@ def _init_submodule() -> None:
47
 
48
  # GitHub serves a tarball of any branch/tag/SHA at this URL.
49
  # Pinned to a specific commit SHA so future lsp changes don't break us.
50
- branch = "74582acd911f81309ba8b22cef9286c2887dda18"
51
  tarball_url = f"https://api.github.com/repos/batu-el/lsp/tarball/{branch}"
52
  tmp_tar = Path("/tmp/lsp.tar.gz")
53
  tmp_extract = Path("/tmp/lsp_extract")
 
47
 
48
  # GitHub serves a tarball of any branch/tag/SHA at this URL.
49
  # Pinned to a specific commit SHA so future lsp changes don't break us.
50
+ branch = "862a08160b07aed48238c7dadc191e138a00dc9a"
51
  tarball_url = f"https://api.github.com/repos/batu-el/lsp/tarball/{branch}"
52
  tmp_tar = Path("/tmp/lsp.tar.gz")
53
  tmp_extract = Path("/tmp/lsp_extract")
src/data.py CHANGED
@@ -727,7 +727,6 @@ def init_state(cfg: dict) -> dict:
727
  "use_demographics": m.get("use_demographics", False),
728
  "use_background": m.get("use_background", False),
729
  "personalization": m.get("personalization"),
730
- "detailed_instruction": m.get("detailed_instruction", True),
731
  })
732
 
733
  for i, slot in enumerate(slots):
 
727
  "use_demographics": m.get("use_demographics", False),
728
  "use_background": m.get("use_background", False),
729
  "personalization": m.get("personalization"),
 
730
  })
731
 
732
  for i, slot in enumerate(slots):
src/lsp_wrappers.py CHANGED
@@ -142,6 +142,7 @@ def build_seller_system_prompt_preference(
142
  from prompts.seller_system.preference import get_seller_system_prompt
143
  pv = cfg["prompt_variant"]
144
  a, b = pair["product_a"], pair["product_b"]
 
145
  result = get_seller_system_prompt(
146
  personalization=pv["personalization"],
147
  title=a.get("title"),
 
142
  from prompts.seller_system.preference import get_seller_system_prompt
143
  pv = cfg["prompt_variant"]
144
  a, b = pair["product_a"], pair["product_b"]
145
+ # lsp API: only `personalization` is forwarded (no detailed_instruction / include_bio here).
146
  result = get_seller_system_prompt(
147
  personalization=pv["personalization"],
148
  title=a.get("title"),
src/ui/screens_model_comparison.py CHANGED
@@ -46,10 +46,11 @@ ETHICS_FINAL = (
46
  def _prompt_variant_for_model(mconf: dict) -> dict:
47
  use_d = mconf.get("use_demographics", False)
48
  use_b = mconf.get("use_background", False)
 
 
49
  return {
50
  "personalization": mconf.get("personalization", bool(use_d or use_b)),
51
  "include_bio": bool(mconf.get("use_background", use_b)),
52
- "detailed_instruction": mconf.get("detailed_instruction", True),
53
  }
54
 
55
 
 
46
  def _prompt_variant_for_model(mconf: dict) -> dict:
47
  use_d = mconf.get("use_demographics", False)
48
  use_b = mconf.get("use_background", False)
49
+ # Only keys read by build_seller_system_prompt_preference β†’ get_seller_system_prompt:
50
+ # `personalization`. (include_bio is unused there too; bio gated by use_background + format_demographics.)
51
  return {
52
  "personalization": mconf.get("personalization", bool(use_d or use_b)),
53
  "include_bio": bool(mconf.get("use_background", use_b)),
 
54
  }
55
 
56
 
src/upload.py CHANGED
@@ -226,7 +226,8 @@ def _save_and_upload_csv(
226
  "personalization",
227
  mconf.get("use_demographics", False) or mconf.get("use_background", False),
228
  ),
229
- "detailed_instruction": mconf.get("detailed_instruction", True),
 
230
  }
231
  rows.append([
232
  submission_id,
 
226
  "personalization",
227
  mconf.get("use_demographics", False) or mconf.get("use_background", False),
228
  ),
229
+ # Not passed to lsp get_seller_system_prompt; optional YAML audit only.
230
+ "detailed_instruction": mconf.get("detailed_instruction", ""),
231
  }
232
  rows.append([
233
  submission_id,
study_config.yaml CHANGED
@@ -6,8 +6,8 @@
6
  # "preference" : participants compare Product A vs Product B (7-pt preference scale)
7
  # "likelihood" : participants evaluate a single product (7-pt likelihood-to-buy scale)
8
  # "model_comparison" : one pair; same participant chats with multiple seller models
9
- # (order randomized). Use pairs_per_user: 1, comparison_models: (see example at bottom),
10
- # and omit or ignore model_variants for that study_type.
11
  # study_type: preference
12
 
13
  # Categories to include. Each entry needs a name and a count.
@@ -49,11 +49,11 @@ pair_selection_seed: 42 # Seed for reproducible 50-item pool selection p
49
  # max_turns: 3 # Hard cap; input is disabled after this many exchanges
50
 
51
  # Prolific
52
- prolific_completion_code: "C18E7285"
53
  prolific_study_id: "69fd27af45ed482b722bd5f1"
54
 
55
  # HuggingFace dataset repo where results (JSON + CSV) are uploaded
56
- output_dataset_repo: "ehejin/user_study-preference-personalized_0505_comparison_NP2"
57
 
58
  # ── Example: model_comparison (uncomment and set study_type; comment out model_variants) ──
59
  #
@@ -71,11 +71,15 @@ comparison_models:
71
  use_demographics: false
72
  use_background: false
73
  personalization: false
74
- detailed_instruction: true
75
- - name: finetuned_NP2
76
  model_name: "meta-llama/Llama-3.1-8B-Instruct"
77
- sampler_path: "tinker://c9dec0ea-ba72-52f0-a311-2f9f327f0a12:train:0/sampler_weights/000125"
78
  use_demographics: false
79
  use_background: false
80
  personalization: false
81
- detailed_instruction: false
 
 
 
 
 
 
6
  # "preference" : participants compare Product A vs Product B (7-pt preference scale)
7
  # "likelihood" : participants evaluate a single product (7-pt likelihood-to-buy scale)
8
  # "model_comparison" : one pair; same participant chats with multiple seller models
9
+ # (order randomized). Use pairs_per_user: 1 and comparison_models: (name, model_name,
10
+ # sampler_path, use_demographics, use_background, personalization). Omit model_variants.
11
  # study_type: preference
12
 
13
  # Categories to include. Each entry needs a name and a count.
 
49
  # max_turns: 3 # Hard cap; input is disabled after this many exchanges
50
 
51
  # Prolific
52
+ prolific_completion_code: "CIE6CQV7"
53
  prolific_study_id: "69fd27af45ed482b722bd5f1"
54
 
55
  # HuggingFace dataset repo where results (JSON + CSV) are uploaded
56
+ output_dataset_repo: "ehejin/user_study-preference-personalized_0514_comparison_JR1_2"
57
 
58
  # ── Example: model_comparison (uncomment and set study_type; comment out model_variants) ──
59
  #
 
71
  use_demographics: false
72
  use_background: false
73
  personalization: false
74
+ - name: finetuned_JR1
 
75
  model_name: "meta-llama/Llama-3.1-8B-Instruct"
76
+ sampler_path: "tinker://12852bca-8673-59f1-9bf2-90438f1ceebe:train:0/weights/000030", "sampler_path": "tinker://12852bca-8673-59f1-9bf2-90438f1ceebe:train:0/sampler_weights/000030"
77
  use_demographics: false
78
  use_background: false
79
  personalization: false
80
+ - name: finetuned_JR2
81
+ model_name: "meta-llama/Llama-3.1-8B-Instruct"
82
+ sampler_path: "tinker://5e6db03e-85d5-5d3c-95db-8c68e7718be1:train:0/sampler_weights/000120"
83
+ use_demographics: false
84
+ use_background: false
85
+ personalization: false