| """Answers without an API: Python documentation lookups and checked function writing.""" |
|
|
| from ycode.lm.assist import ( |
| CHECKED_NOTE, |
| DOC_SOURCE_NOTE, |
| UNVERIFIED_NOTE, |
| Assistant, |
| defines, |
| docs_answer, |
| requested_function, |
| ) |
|
|
|
|
| def test_docs_answers_for_builtins_stdlib_and_keywords(): |
| assert "Return the number of items" in docs_answer("What does the len() function do?") |
| assert "json.loads(" in docs_answer("how do I use json.loads") |
| assert "os.path.join(" in docs_answer("Explain os.path.join") |
| assert "Yield tuples" in docs_answer("what does `zip` do?") |
| assert "(Python keyword)" in docs_answer("what is yield?") |
| both = docs_answer("what is the difference between a list and a tuple") |
| assert "mutable sequence" in both and "immutable sequence" in both |
| assert docs_answer("What does len() do?").endswith(DOC_SOURCE_NOTE) |
|
|
|
|
| def test_docs_only_for_documentation_questions(): |
| assert docs_answer("Write a function that uses len()") is None |
| assert docs_answer("hello there") is None |
| assert docs_answer("what does my_project.helpers do?") is None |
| assert docs_answer("what does _thread do?") is None |
|
|
|
|
| def test_requested_function_and_defines(): |
| assert requested_function("Write a Python function is_even(n) that returns True if n is even.") == "is_even" |
| assert requested_function("Write a function called `slugify(text)`") == "slugify" |
| assert requested_function("How does sorting work?") is None |
| assert defines("def is_even(n):\n return n % 2 == 0\n", "is_even") |
| assert not defines("def other(n):\n return n\n", "is_even") |
| assert not defines("def is_even(n) return", "is_even") |
|
|
|
|
| class ScriptedLM: |
| def __init__(self, replies): |
| self.replies = list(replies) |
| self.temperatures = [] |
|
|
| def chat(self, turns, *, temperature=0.7, **_kw): |
| self.temperatures.append(temperature) |
| return self.replies.pop(0) |
|
|
|
|
| def test_assistant_returns_first_candidate_defining_the_function(): |
| lm = ScriptedLM(["```python\ndef wrong_name(s):\n return s\n```", |
| "```python\ndef reverse_string(s):\n return s[::-1]\n```"]) |
| streamed = [] |
| answer = Assistant(lm, candidates=4).answer( |
| [("user", "Write a function reverse_string(s) that returns s reversed.")], on_text=streamed.append) |
| assert "return s[::-1]" in answer and CHECKED_NOTE in answer |
| assert "".join(streamed) == answer |
| assert lm.temperatures == [0.2, 0.7] |
|
|
|
|
| def test_assistant_flags_unverified_code(): |
| lm = ScriptedLM(["def nope(x): pass"] * 3) |
| answer = Assistant(lm, candidates=3).answer([("user", "Write a function add(a, b).")]) |
| assert answer.endswith(UNVERIFIED_NOTE) and len(lm.temperatures) == 3 |
|
|
|
|
| def test_assistant_uses_docs_before_the_model(): |
| lm = ScriptedLM([]) |
| answer = Assistant(lm).answer([("user", "What does the len() function do?")]) |
| assert "Return the number of items" in answer and lm.temperatures == [] |
|
|
