Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
.env
ExampleContents/
# Written from colleagues' documents, like ExampleContents itself.
corpus-specs/
.workbench/
__pycache__/
*.egg-info/
Expand Down
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,8 @@ prints, and the in2lambda function each stage calls.
What the checks find — a block of the source in no field, two fields from the same
lines, a gap in the numbering, a part nothing answers — returns to the model as a
fixing round, with in2lambda's draft commands as its tools: `mark ignore`,
`question add`, `part add`, `question solution`, `split block`, and `field replace` for
`question add`, `part add`, `question solution`, `part solution`, `split block`,
`field set` for a field that is empty or took the wrong lines, and `field replace` for
wording that no range of the source gives. The model writes a field by naming where its
text is in the source rather than by typing the text out. in2lambda records each
command in the draft's log with the layer of the field it wrote, so a reader can read
Expand Down
7 changes: 4 additions & 3 deletions docs/how-it-works.md
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ Each backend limits a call differently:
| --- | --- | --- |
| Spec | the spec system prompt, and the frozen source as `in2lambda.source.show` prints it | `in2lambda-spec.yaml`, and nothing else |
| Spec rewrite | the same, with the errors of the last report appended to the prompt | `in2lambda-spec.yaml`, and nothing else |
| Fixing round | the fixing system prompt, the frozen source, every finding of the report, and a reviewer's note where there is one | the six draft commands, and nothing else |
| Fixing round | the fixing system prompt, the frozen source, every finding of the report, and a reviewer's note where there is one | the eight draft commands, and nothing else |

The spec call has no tools. Its reply is the YAML of a spec, past a code fence where
the model wrote one. The agent refuses a reply that is not YAML, a reply that is not a
Expand All @@ -226,8 +226,9 @@ The spec rewrite is the same call with the report's errors in the prompt. It run
per run, before any fixing round, where the run reused a saved spec and the checks
fault the draft. It writes layer 1 fields, and it is not one of the `--rounds`.

The fixing round's tools are the six in2lambda draft commands: `mark ignore`,
`question add`, `part add`, `question solution`, `field replace` and `split block`.
The fixing round's tools are the eight in2lambda draft commands: `mark ignore`,
`question add`, `part add`, `question solution`, `part solution`, `field replace`,
`field set` and `split block`.
in2lambda writes the field, records the command in the draft's log as `in2lambda-agent`
and decides the layer. A command in2lambda refuses returns its refusal to the model as
the tool's result, and the round continues. A field's text is named by block id or line
Expand Down
51 changes: 50 additions & 1 deletion in2lambda_agent/fix.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,10 @@
part_add a lettered part of a question that is written already.
question_solution a question's worked solution, wherever it is written. It
answers every part of that question that has none of its own.
part_solution one part's own worked solution, from the lines under it on
the sheet. Use it where each part is answered where it
stands, and question_solution where one solution answers
the whole question.
split_block cuts a block in two at a line, so that each half can be
named: `b7` split at 14 becomes `b7a` and `b7b`. Use it when
one block holds two things — a question and its first part
Expand All @@ -90,6 +94,12 @@
place is at most {LITERAL_MAX} characters, as a literal is.
A field whose text the source does not hold is left as a
finding.
field_set quotes other lines into a field that is written already,
for a field that is empty or that took the wrong lines. It
writes the field again from the lines you name and drops
the ones it held, which are then in no field until you
answer for them. It has no literal: what it writes is in
the source.

Lines that are already in a field cannot be put in another one. A command naming \
them is refused, and so is one naming a block that is not there; either way you \
Expand All @@ -110,6 +120,11 @@
"description": "The question to add to, by the key of its text: q2.",
}

_PART = {
"type": "string",
"description": "The part to answer, by the key of its text: q1.p2.",
}

# Every command that fills a field takes one or the other of these, and in2lambda
# refuses both at once, so neither is required and the model is told which to use.
_WHERE = {
Expand Down Expand Up @@ -144,6 +159,15 @@
"not write one."
),
"split block": "Cut one block in two at a line, so each half can be named.",
"field set": (
"Quote other lines of the source into a field already written, for a "
"field that is empty or that took the wrong lines. The lines it held "
"are dropped, and are then in no field."
),
"part solution": (
"Give one part the worked solution written under it. Where one "
"solution answers the whole question, question solution writes it."
),
}

_PARAMETERS: dict[str, dict[str, Any]] = {
Expand Down Expand Up @@ -204,6 +228,31 @@
},
"required": ["block", "at"],
},
# No `literal`: this command quotes lines into a field and in2lambda takes
# nothing else, so the wording no range of the source holds is `field
# replace`'s to repair once the field has been set.
"field set": {
"type": "object",
"properties": {
"field": {
"type": "string",
"description": "The field to write again, by its key: q1.text.",
},
"text": {
"type": "string",
"description": (
"Where the field's text is in the frozen source: a block "
"id, b7, or lines, s13 or s13:14."
),
},
},
"required": ["field", "text"],
},
"part solution": {
"type": "object",
"properties": {"part": _PART, **_WHERE},
"required": ["part"],
},
}


Expand Down Expand Up @@ -412,7 +461,7 @@ def _writes_field(draft: Path, name: str, args: dict[str, Any]) -> Optional[str]

def _subject(call: ToolCall) -> str:
"""What one command was about, for the stage line: a block, question or field."""
for name in ("block", "question", "field", "text", "literal"):
for name in ("block", "question", "part", "field", "text", "literal"):
if name in call.arguments:
return str(call.arguments[name])
return ""
2 changes: 2 additions & 0 deletions in2lambda_agent/package.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@
"question solution",
"field replace",
"split block",
"field set",
"part solution",
)
"""The draft commands a report is fixed with, named as the log names them."""

Expand Down
4 changes: 2 additions & 2 deletions poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

76 changes: 76 additions & 0 deletions tests/test_fix.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ def test_the_tools_are_the_packages_draft_commands(draft):
"question_solution",
"field_replace",
"split_block",
"field_set",
"part_solution",
]


Expand All @@ -65,6 +67,7 @@ def test_the_tools_are_the_packages_draft_commands(draft):
"q1.text",
),
("split_block", {"block": "b7", "at": 14}, "b7a and b7b"),
("field_set", {"field": "q1.text", "text": "s22"}, "q1.text"),
],
)
def test_each_tool_runs_its_command_and_in2lambda_logs_it(
Expand All @@ -82,6 +85,72 @@ def test_each_tool_runs_its_command_and_in2lambda_logs_it(
}


def test_a_field_set_writes_a_written_field_from_the_lines_it_names(draft):
# The finding it answers: a field that is empty, or that took the wrong
# lines. What the field held before is dropped, so the lines it came from
# are in no field and the next round is told about them.
before = json.loads(draft.read_text())["fields"]["q1.text"]["ranges"]

result = run(draft, "field_set", {"field": "q1.text", "text": "s22"})
written = json.loads(draft.read_text())["fields"]["q1.text"]

assert result == "field set wrote q1.text"
assert before == [[5, 5]]
assert (written["ranges"], written["layer"], written["edited"]) == (
[[22, 22]],
3,
False,
)


def test_a_part_solution_answers_one_part_rather_than_the_question(draft):
# A sheet that writes a solution under each part: the part is added from
# the lines holding it, and then answered on its own.
run(draft, "part_add", {"question": "q1", "text": "b7"})

result = run(draft, "part_solution", {"part": "q1.p3", "text": "s22"})
written = json.loads(draft.read_text())["fields"]["q1.p3.solution"]

assert result == "part solution wrote q1.p3.solution"
assert (written["ranges"], written["layer"], written["edited"]) == (
[[22, 22]],
3,
False,
)
assert package.command_log(draft)[-1] == {
"command": "part solution",
"args": {"part": "q1.p3", "text": "s22"},
"by": package.BY,
}


def test_a_part_solution_typed_out_is_layer_4(draft):
run(draft, "part_add", {"question": "q1", "text": "b7"})

result = run(draft, "part_solution", {"part": "q1.p3", "literal": "Term by term."})
written = json.loads(draft.read_text())["fields"]["q1.p3.solution"]

assert result == "part solution wrote q1.p3.solution"
assert (written["layer"], written["edited"]) == (4, True)


def test_a_part_solution_longer_than_a_repair_is_refused_before_it_is_written(draft):
typed = "x" * (fix.LITERAL_MAX + 1)
run(draft, "part_add", {"question": "q1", "text": "b7"})

result = run(draft, "part_solution", {"part": "q1.p3", "literal": typed})

assert result.startswith("part solution was refused: ")
assert f"literal is {len(typed)} characters" in result
assert commands(draft) == ["spec run", "part add"]


def test_the_system_prompt_names_the_two_commands_that_write_a_written_field():
assert "part_solution one part's own worked solution" in fix.SYSTEM
assert "field_set quotes other lines into a field" in fix.SYSTEM
assert "It has no literal" in fix.SYSTEM


@pytest.mark.parametrize(
"arguments, layer, edited",
[
Expand Down Expand Up @@ -294,10 +363,13 @@ def test_every_tool_that_types_says_how_little_it_may_type(draft):
one for one in fix.tools(draft) if "literal" in one.parameters["properties"]
]

# `field_set` is not among them: in2lambda takes no literal for it, so a
# field it writes says what the source says and nothing else.
assert [one.name for one in typing] == [
"question_add",
"part_add",
"question_solution",
"part_solution",
]
assert all(
one.parameters["properties"]["literal"]["maxLength"] == fix.LITERAL_MAX
Expand Down Expand Up @@ -390,6 +462,10 @@ def test_the_halves_of_a_split_block_are_shown_to_the_next_round(draft):
],
"2 commands (mark ignore b9, question solution q2)",
),
(
[("part_solution", {"part": "q1.p3", "text": "s22"})],
"1 command (part solution q1.p3)",
),
],
)
def test_the_summary_names_what_each_command_was_about(calls, expected):
Expand Down
Loading