fix(advice): find the marker however the model formats it
The instruction asks for "ADVICE:", but the model writes "**ADVICE:**",
"## ADVICE:" or puts it in a code block. The extraction has to cope,
because leftover markdown in the advice is visible to the brewer.
The marker is now matched case-insensitively and with the decoration the
model puts around it, and the leftover formatting is stripped.
plain marker -> 'Warm it to 20C and wait.'
bold marker -> 'Warm it to 20C and wait.'
in a code block -> 'Warm it to 20C and wait.'
marker mid-text -> 'Warm it to 20C and wait.'
markdown heading -> 'Warm it to 20C and wait.'
lowercase marker -> 'Warm it to 20C and wait.'
no marker -> 'Warm it to 20C and wait.'
bold + code block -> 'Warm it to 20C and wait.'
182 tests (up from 181).
This commit is contained in:
@@ -25,6 +25,7 @@ from __future__ import annotations
|
|||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
import urllib.error
|
import urllib.error
|
||||||
import urllib.request
|
import urllib.request
|
||||||
|
|
||||||
@@ -271,12 +272,38 @@ def _strip_reasoning(text: str) -> str:
|
|||||||
working did leak in, the brewer sees a slightly rambling reply
|
working did leak in, the brewer sees a slightly rambling reply
|
||||||
rather than no advice at all.
|
rather than no advice at all.
|
||||||
"""
|
"""
|
||||||
index = text.find(_ADVICE_MARKER)
|
index = _find_marker(text)
|
||||||
if index == -1:
|
if index is None:
|
||||||
return text.strip()
|
return text.strip()
|
||||||
|
|
||||||
advice = text[index + len(_ADVICE_MARKER):].strip()
|
advice = text[index:].strip()
|
||||||
return advice or ""
|
|
||||||
|
# The model may wrap the marker in markdown or a code block. Strip
|
||||||
|
# the leftover formatting so the advice reads as prose.
|
||||||
|
advice = re.sub(r"^\*\*|\*\*$", "", advice).strip()
|
||||||
|
if advice.startswith("```"):
|
||||||
|
advice = advice[3:]
|
||||||
|
if advice.endswith("```"):
|
||||||
|
advice = advice[:-3]
|
||||||
|
if advice.startswith("##"):
|
||||||
|
advice = advice.lstrip("#").strip()
|
||||||
|
|
||||||
|
return advice.strip() or ""
|
||||||
|
|
||||||
|
|
||||||
|
def _find_marker(text: str) -> int | None:
|
||||||
|
"""Where the advice starts, or ``None`` if there is no marker.
|
||||||
|
|
||||||
|
The marker is matched case-insensitively and with the markdown
|
||||||
|
decoration the model might put around it - ``**ADVICE:**``,
|
||||||
|
``## ADVICE:`` - because the instruction asks for "ADVICE:" but the
|
||||||
|
model formats it its own way often enough to matter.
|
||||||
|
"""
|
||||||
|
match = re.search(r"^[\s#>\*]*advice\s*:[\s\*>]*", text,
|
||||||
|
re.IGNORECASE | re.MULTILINE)
|
||||||
|
if match:
|
||||||
|
return match.end()
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def generate_advice(run: dict, reading: dict, history: list[dict], *,
|
def generate_advice(run: dict, reading: dict, history: list[dict], *,
|
||||||
|
|||||||
@@ -274,6 +274,28 @@ def test_a_reply_with_the_marker_is_kept_whole():
|
|||||||
assert brew_advice._strip_reasoning(reply) == advice
|
assert brew_advice._strip_reasoning(reply) == advice
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_marker_is_found_however_the_model_formats_it():
|
||||||
|
"""The model formats the marker its own way often enough to matter.
|
||||||
|
|
||||||
|
The instruction asks for "ADVICE:", but the model writes
|
||||||
|
"**ADVICE:**", "## ADVICE:" or puts it in a code block. Each of
|
||||||
|
those has to extract the advice cleanly, with no leftover markdown.
|
||||||
|
"""
|
||||||
|
advice = "Warm it to 20C and wait."
|
||||||
|
|
||||||
|
for reply in (
|
||||||
|
f"ADVICE:\n{advice}",
|
||||||
|
f"**ADVICE:**\n{advice}",
|
||||||
|
f"## ADVICE:\n{advice}",
|
||||||
|
f"```\nADVICE:\n{advice}\n```",
|
||||||
|
f"**ADVICE:**\n```\n{advice}\n```",
|
||||||
|
f"advice:\n{advice}",
|
||||||
|
):
|
||||||
|
assert brew_advice._strip_reasoning(reply) == advice, (
|
||||||
|
f"the marker was not extracted cleanly from {reply!r}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_a_reply_without_the_marker_is_used_as_it_stands():
|
def test_a_reply_without_the_marker_is_used_as_it_stands():
|
||||||
"""The safe direction when the model does not use the marker.
|
"""The safe direction when the model does not use the marker.
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user