File size: 3,079 Bytes
5ef3875 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | """Answers without an API: Python documentation lookups and checked function writing."""
from ycode.lm.assist import (
CHECKED_NOTE,
DOC_SOURCE_NOTE,
UNVERIFIED_NOTE,
Assistant,
defines,
docs_answer,
requested_function,
)
def test_docs_answers_for_builtins_stdlib_and_keywords():
assert "Return the number of items" in docs_answer("What does the len() function do?")
assert "json.loads(" in docs_answer("how do I use json.loads")
assert "os.path.join(" in docs_answer("Explain os.path.join")
assert "Yield tuples" in docs_answer("what does `zip` do?")
assert "(Python keyword)" in docs_answer("what is yield?")
both = docs_answer("what is the difference between a list and a tuple")
assert "mutable sequence" in both and "immutable sequence" in both
assert docs_answer("What does len() do?").endswith(DOC_SOURCE_NOTE)
def test_docs_only_for_documentation_questions():
assert docs_answer("Write a function that uses len()") is None # a coding request, not a docs one
assert docs_answer("hello there") is None
assert docs_answer("what does my_project.helpers do?") is None # never imports the user's code
assert docs_answer("what does _thread do?") is None
def test_requested_function_and_defines():
assert requested_function("Write a Python function is_even(n) that returns True if n is even.") == "is_even"
assert requested_function("Write a function called `slugify(text)`") == "slugify"
assert requested_function("How does sorting work?") is None
assert defines("def is_even(n):\n return n % 2 == 0\n", "is_even")
assert not defines("def other(n):\n return n\n", "is_even")
assert not defines("def is_even(n) return", "is_even")
class ScriptedLM:
def __init__(self, replies):
self.replies = list(replies)
self.temperatures = []
def chat(self, turns, *, temperature=0.7, **_kw):
self.temperatures.append(temperature)
return self.replies.pop(0)
def test_assistant_returns_first_candidate_defining_the_function():
lm = ScriptedLM(["```python\ndef wrong_name(s):\n return s\n```",
"```python\ndef reverse_string(s):\n return s[::-1]\n```"])
streamed = []
answer = Assistant(lm, candidates=4).answer(
[("user", "Write a function reverse_string(s) that returns s reversed.")], on_text=streamed.append)
assert "return s[::-1]" in answer and CHECKED_NOTE in answer
assert "".join(streamed) == answer
assert lm.temperatures == [0.2, 0.7] # nearly greedy first, then sampling
def test_assistant_flags_unverified_code():
lm = ScriptedLM(["def nope(x): pass"] * 3)
answer = Assistant(lm, candidates=3).answer([("user", "Write a function add(a, b).")])
assert answer.endswith(UNVERIFIED_NOTE) and len(lm.temperatures) == 3
def test_assistant_uses_docs_before_the_model():
lm = ScriptedLM([])
answer = Assistant(lm).answer([("user", "What does the len() function do?")])
assert "Return the number of items" in answer and lm.temperatures == []
|