fix(advice): judge the working by its shape, not by its length
A length threshold is a blunt instrument: a legitimate advice reply can
run to a thousand characters, and a short working reply can be a few
hundred - so a threshold gets both wrong. A 995-character advice reply
with no marker word in it was being emptied.
The reliable signal is the SHAPE. The model's working is about
INTERPRETING the numbers - what the reading might mean, what could be
causing it, what it is unsure about. The advice is about what to DO.
So a reply is working when it is full of the model's hedging about the
data AND contains no instruction at all. Length is only a tie-breaker
for a reply that is both hedged and short.
995-char legitimate advice -> kept
long pure working -> dropped
short legitimate advice -> kept
long working + advice -> advice kept
Checked against the live model - a real reply comes back clean, with no
reasoning in it:
At two weeks in, dropping from 1.055 to 1.020 is a healthy 35-point
fall - about 64% attenuation - which is right in the normal range for
a pale ale, though you likely still have 5 to 8 points to go...
184 tests (up from 183).
This commit is contained in:
+32
-15
@@ -356,31 +356,48 @@ _WORKING_LENGTH = 600
|
|||||||
def _is_all_working(text: str, paragraphs: list[str]) -> bool:
|
def _is_all_working(text: str, paragraphs: list[str]) -> bool:
|
||||||
"""True when the reply is the model's working rather than advice.
|
"""True when the reply is the model's working rather than advice.
|
||||||
|
|
||||||
Two signals, because either alone is unreliable:
|
The reliable signal is SHAPE, not length. A legitimate advice reply
|
||||||
|
can be a thousand characters, and a short working reply can be a
|
||||||
|
few hundred - so length alone gets both wrong.
|
||||||
|
|
||||||
- **Length.** The working runs to thousands of characters; the
|
The model's working is about INTERPRETING the numbers: what the
|
||||||
advice is two to five sentences.
|
reading might mean, what could be causing it, what it is unsure
|
||||||
- **Shape.** The working interprets the numbers ("perhaps the yeast
|
about. The advice is about what to DO.
|
||||||
is stressed, it could be that..."); the advice tells the brewer
|
|
||||||
what to do.
|
|
||||||
|
|
||||||
A reply that is SHORT is kept even if it has no marker word - real
|
So a reply is working when it is full of the model's hedging about
|
||||||
advice is often phrased without one ("Actually the drop is fine.
|
the data AND contains no instruction at all.
|
||||||
Wait a couple of days and check again.").
|
|
||||||
"""
|
"""
|
||||||
if len(text) <= _WORKING_LENGTH:
|
|
||||||
# Too short to be the model's working. It is advice, whatever
|
|
||||||
# it is worded like.
|
|
||||||
return False
|
|
||||||
|
|
||||||
if _has_advice_marker(text):
|
if _has_advice_marker(text):
|
||||||
# It contains an instruction, so part of it is advice.
|
# It contains an instruction, so part of it is advice.
|
||||||
return False
|
return False
|
||||||
|
|
||||||
# Long, and nothing in it reads like an instruction.
|
lowered = text.lower()
|
||||||
|
hedging = sum(lowered.count(word) for word in _HEDGING_WORDS)
|
||||||
|
|
||||||
|
if hedging == 0:
|
||||||
|
# No instruction and no hedging about the data. This is advice,
|
||||||
|
# however long it is.
|
||||||
|
return False
|
||||||
|
|
||||||
|
if len(text) <= _WORKING_LENGTH:
|
||||||
|
# Some hedging, but it is short - that is a brewer's note that
|
||||||
|
# happens to use a hedging word, not a page of working.
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Hedging, no instruction, and long: the working.
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
#: The words the model uses when it is working through the data rather
|
||||||
|
#: than telling the brewer what to do.
|
||||||
|
_HEDGING_WORDS = (
|
||||||
|
"perhaps", "maybe", "possibly", "could be", "might be", "might",
|
||||||
|
"i think", "i am not sure", "i'm not sure", "actually", "hmm",
|
||||||
|
"wait,", "let me", "i need to", "the question is", "unclear",
|
||||||
|
"i am unsure", "not sure", "unfortunately",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _has_advice_marker(text: str) -> bool:
|
def _has_advice_marker(text: str) -> bool:
|
||||||
"""True when the text contains any advice marker at all.
|
"""True when the text contains any advice marker at all.
|
||||||
|
|
||||||
|
|||||||
@@ -688,6 +688,35 @@ def test_a_short_reply_is_advice_even_without_a_marker_word():
|
|||||||
assert brew_advice._strip_reasoning(reply) == reply
|
assert brew_advice._strip_reasoning(reply) == reply
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_long_legitimate_advice_reply_is_kept():
|
||||||
|
"""Length alone is a blunt instrument.
|
||||||
|
|
||||||
|
A legitimate advice reply can run to a thousand characters, and a
|
||||||
|
short working reply can be a few hundred - so a length threshold
|
||||||
|
gets both wrong. What separates them is the SHAPE: the working
|
||||||
|
hedges about the data, the advice tells the brewer what to do.
|
||||||
|
"""
|
||||||
|
advice = (
|
||||||
|
"The ferment has dropped steadily since brew day and the numbers "
|
||||||
|
"line up with a healthy fermentation, so the yeast is clearly "
|
||||||
|
"working through the wort rather than sitting on it. At 19 degrees "
|
||||||
|
"this is on the cool side for an ale yeast and it will finish "
|
||||||
|
"slowly, but that is not a problem in itself. Give the ferment "
|
||||||
|
"another three or four days before you draw any conclusions, and "
|
||||||
|
"take a reading on the eighteenth so you can see whether it is "
|
||||||
|
"still moving. Do not raise the temperature in a hurry either, "
|
||||||
|
"because a sudden change while the yeast is finishing can make it "
|
||||||
|
"give up what it has left to give. If it is still stuck at the "
|
||||||
|
"same gravity after that, then it is worth rousing the yeast with "
|
||||||
|
"a gentle swirl and considering a fresh sachet of dry yeast, "
|
||||||
|
"rehydrated in the usual way and pitched at the same temperature "
|
||||||
|
"the beer is sitting at."
|
||||||
|
)
|
||||||
|
assert len(advice) > 600, "the sample should be longer than the threshold"
|
||||||
|
|
||||||
|
assert brew_advice._strip_reasoning(advice) == advice
|
||||||
|
|
||||||
|
|
||||||
def test_a_long_reply_with_no_instruction_is_dropped():
|
def test_a_long_reply_with_no_instruction_is_dropped():
|
||||||
"""Long, and nothing in it reads like an instruction - all working."""
|
"""Long, and nothing in it reads like an instruction - all working."""
|
||||||
working = ("Let me think about this.\n\n" +
|
working = ("Let me think about this.\n\n" +
|
||||||
|
|||||||
Reference in New Issue
Block a user