teren de test TypeSafe: docs offline + script de proba
Separat de produsele ROA. docs/ = documentatia oficiala descarcata ca Markdown (111 pagini), reluabila cu update_docs.sh. typesafe_test.py face un apel cu cate o intrebare din fiecare tip (choice/noul/score) pe o linie de factura de furnizor. Cheia API se ia din TYPESAFE_API_KEY, nu se versioneaza. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01KHLUSsKP99G6ebv2fFUKQV
This commit is contained in:
291
docs/patterns/confidence-routing.md
Normal file
291
docs/patterns/confidence-routing.md
Normal file
@@ -0,0 +1,291 @@
|
||||
> ## Documentation Index
|
||||
> Fetch the complete documentation index at: https://docs.typesafe.ai/llms.txt
|
||||
> Use this file to discover all available pages before exploring further.
|
||||
|
||||
# Confidence-gated routing
|
||||
|
||||
> Use confidence as a second axis. The answer tells you what; confidence tells you whether to act.
|
||||
|
||||
export function TypesafeExample({example, display, title}) {
|
||||
const keyStrUriSafe = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-$";
|
||||
function compressToEncodedURIComponent(input) {
|
||||
if (input == null) return "";
|
||||
return _compress(input, 6, function (a) {
|
||||
return keyStrUriSafe.charAt(a);
|
||||
});
|
||||
}
|
||||
function _compress(uncompressed, bitsPerChar, getCharFromInt) {
|
||||
if (uncompressed == null) return "";
|
||||
var i, value, context_dictionary = {}, context_dictionaryToCreate = {}, context_c = "", context_wc = "", context_w = "", context_enlargeIn = 2, context_dictSize = 3, context_numBits = 2, context_data = [], context_data_val = 0, context_data_position = 0, ii;
|
||||
for (ii = 0; ii < uncompressed.length; ii += 1) {
|
||||
context_c = uncompressed.charAt(ii);
|
||||
if (!Object.prototype.hasOwnProperty.call(context_dictionary, context_c)) {
|
||||
context_dictionary[context_c] = context_dictSize++;
|
||||
context_dictionaryToCreate[context_c] = true;
|
||||
}
|
||||
context_wc = context_w + context_c;
|
||||
if (Object.prototype.hasOwnProperty.call(context_dictionary, context_wc)) {
|
||||
context_w = context_wc;
|
||||
} else {
|
||||
if (Object.prototype.hasOwnProperty.call(context_dictionaryToCreate, context_w)) {
|
||||
if (context_w.charCodeAt(0) < 256) {
|
||||
for (i = 0; i < context_numBits; i++) {
|
||||
context_data_val = context_data_val << 1;
|
||||
if (context_data_position == bitsPerChar - 1) {
|
||||
context_data_position = 0;
|
||||
context_data.push(getCharFromInt(context_data_val));
|
||||
context_data_val = 0;
|
||||
} else {
|
||||
context_data_position++;
|
||||
}
|
||||
}
|
||||
value = context_w.charCodeAt(0);
|
||||
for (i = 0; i < 8; i++) {
|
||||
context_data_val = context_data_val << 1 | value & 1;
|
||||
if (context_data_position == bitsPerChar - 1) {
|
||||
context_data_position = 0;
|
||||
context_data.push(getCharFromInt(context_data_val));
|
||||
context_data_val = 0;
|
||||
} else {
|
||||
context_data_position++;
|
||||
}
|
||||
value = value >> 1;
|
||||
}
|
||||
} else {
|
||||
value = 1;
|
||||
for (i = 0; i < context_numBits; i++) {
|
||||
context_data_val = context_data_val << 1 | value;
|
||||
if (context_data_position == bitsPerChar - 1) {
|
||||
context_data_position = 0;
|
||||
context_data.push(getCharFromInt(context_data_val));
|
||||
context_data_val = 0;
|
||||
} else {
|
||||
context_data_position++;
|
||||
}
|
||||
value = 0;
|
||||
}
|
||||
value = context_w.charCodeAt(0);
|
||||
for (i = 0; i < 16; i++) {
|
||||
context_data_val = context_data_val << 1 | value & 1;
|
||||
if (context_data_position == bitsPerChar - 1) {
|
||||
context_data_position = 0;
|
||||
context_data.push(getCharFromInt(context_data_val));
|
||||
context_data_val = 0;
|
||||
} else {
|
||||
context_data_position++;
|
||||
}
|
||||
value = value >> 1;
|
||||
}
|
||||
}
|
||||
context_enlargeIn--;
|
||||
if (context_enlargeIn == 0) {
|
||||
context_enlargeIn = Math.pow(2, context_numBits);
|
||||
context_numBits++;
|
||||
}
|
||||
delete context_dictionaryToCreate[context_w];
|
||||
} else {
|
||||
value = context_dictionary[context_w];
|
||||
for (i = 0; i < context_numBits; i++) {
|
||||
context_data_val = context_data_val << 1 | value & 1;
|
||||
if (context_data_position == bitsPerChar - 1) {
|
||||
context_data_position = 0;
|
||||
context_data.push(getCharFromInt(context_data_val));
|
||||
context_data_val = 0;
|
||||
} else {
|
||||
context_data_position++;
|
||||
}
|
||||
value = value >> 1;
|
||||
}
|
||||
}
|
||||
context_enlargeIn--;
|
||||
if (context_enlargeIn == 0) {
|
||||
context_enlargeIn = Math.pow(2, context_numBits);
|
||||
context_numBits++;
|
||||
}
|
||||
context_dictionary[context_wc] = context_dictSize++;
|
||||
context_w = String(context_c);
|
||||
}
|
||||
}
|
||||
if (context_w !== "") {
|
||||
if (Object.prototype.hasOwnProperty.call(context_dictionaryToCreate, context_w)) {
|
||||
if (context_w.charCodeAt(0) < 256) {
|
||||
for (i = 0; i < context_numBits; i++) {
|
||||
context_data_val = context_data_val << 1;
|
||||
if (context_data_position == bitsPerChar - 1) {
|
||||
context_data_position = 0;
|
||||
context_data.push(getCharFromInt(context_data_val));
|
||||
context_data_val = 0;
|
||||
} else {
|
||||
context_data_position++;
|
||||
}
|
||||
}
|
||||
value = context_w.charCodeAt(0);
|
||||
for (i = 0; i < 8; i++) {
|
||||
context_data_val = context_data_val << 1 | value & 1;
|
||||
if (context_data_position == bitsPerChar - 1) {
|
||||
context_data_position = 0;
|
||||
context_data.push(getCharFromInt(context_data_val));
|
||||
context_data_val = 0;
|
||||
} else {
|
||||
context_data_position++;
|
||||
}
|
||||
value = value >> 1;
|
||||
}
|
||||
} else {
|
||||
value = 1;
|
||||
for (i = 0; i < context_numBits; i++) {
|
||||
context_data_val = context_data_val << 1 | value;
|
||||
if (context_data_position == bitsPerChar - 1) {
|
||||
context_data_position = 0;
|
||||
context_data.push(getCharFromInt(context_data_val));
|
||||
context_data_val = 0;
|
||||
} else {
|
||||
context_data_position++;
|
||||
}
|
||||
value = 0;
|
||||
}
|
||||
value = context_w.charCodeAt(0);
|
||||
for (i = 0; i < 16; i++) {
|
||||
context_data_val = context_data_val << 1 | value & 1;
|
||||
if (context_data_position == bitsPerChar - 1) {
|
||||
context_data_position = 0;
|
||||
context_data.push(getCharFromInt(context_data_val));
|
||||
context_data_val = 0;
|
||||
} else {
|
||||
context_data_position++;
|
||||
}
|
||||
value = value >> 1;
|
||||
}
|
||||
}
|
||||
context_enlargeIn--;
|
||||
if (context_enlargeIn == 0) {
|
||||
context_enlargeIn = Math.pow(2, context_numBits);
|
||||
context_numBits++;
|
||||
}
|
||||
delete context_dictionaryToCreate[context_w];
|
||||
} else {
|
||||
value = context_dictionary[context_w];
|
||||
for (i = 0; i < context_numBits; i++) {
|
||||
context_data_val = context_data_val << 1 | value & 1;
|
||||
if (context_data_position == bitsPerChar - 1) {
|
||||
context_data_position = 0;
|
||||
context_data.push(getCharFromInt(context_data_val));
|
||||
context_data_val = 0;
|
||||
} else {
|
||||
context_data_position++;
|
||||
}
|
||||
value = value >> 1;
|
||||
}
|
||||
}
|
||||
context_enlargeIn--;
|
||||
if (context_enlargeIn == 0) {
|
||||
context_enlargeIn = Math.pow(2, context_numBits);
|
||||
context_numBits++;
|
||||
}
|
||||
}
|
||||
value = 2;
|
||||
for (i = 0; i < context_numBits; i++) {
|
||||
context_data_val = context_data_val << 1 | value & 1;
|
||||
if (context_data_position == bitsPerChar - 1) {
|
||||
context_data_position = 0;
|
||||
context_data.push(getCharFromInt(context_data_val));
|
||||
context_data_val = 0;
|
||||
} else {
|
||||
context_data_position++;
|
||||
}
|
||||
value = value >> 1;
|
||||
}
|
||||
while (true) {
|
||||
context_data_val = context_data_val << 1;
|
||||
if (context_data_position == bitsPerChar - 1) {
|
||||
context_data.push(getCharFromInt(context_data_val));
|
||||
break;
|
||||
} else context_data_position++;
|
||||
}
|
||||
return context_data.join("");
|
||||
}
|
||||
function buildHref(ex) {
|
||||
const documentText = ex.state === undefined ? "" : typeof ex.state === "string" ? ex.state : JSON.stringify(ex.state, null, 2);
|
||||
return "https://console.typesafe.ai/decode#share/" + compressToEncodedURIComponent(JSON.stringify({
|
||||
apiVersion: "v1",
|
||||
documentText,
|
||||
promptsText: JSON.stringify(ex.questions, null, 2),
|
||||
selectedModels: ex.selectedModels
|
||||
}));
|
||||
}
|
||||
const displayedExample = display === "questions" ? example.questions : example.state === undefined ? {
|
||||
questions: example.questions
|
||||
} : {
|
||||
state: example.state,
|
||||
questions: example.questions
|
||||
};
|
||||
const code = JSON.stringify(displayedExample, null, 2);
|
||||
const href = buildHref(example);
|
||||
return <div style={{
|
||||
margin: "1.25rem 0"
|
||||
}}>
|
||||
<CodeBlock language="json" filename={title ?? "request"}>
|
||||
{code}
|
||||
</CodeBlock>
|
||||
<div className="pb-8">
|
||||
<a href={href} target="_blank" rel="noreferrer" className="text-primary">
|
||||
Try it in the Playground →
|
||||
</a>
|
||||
</div>
|
||||
</div>;
|
||||
}
|
||||
|
||||
One of TypeSafe's most powerful features is [confidence](/confidence). By being intentional with the way you gate decisions on confidence, you can build systems that are both reliable and safe.
|
||||
|
||||
## Example: voice banking commands
|
||||
|
||||
Let's imagine you are building a voice banking interface to allow the user to interact with their account verbally. While you always want to have reasonable confidence in interpreting the user's intent, some actions are riskier than others and thus demand a higher confidence threshold.
|
||||
|
||||
### Step 1: determine the user's intent
|
||||
|
||||
<TypesafeExample
|
||||
title="questions"
|
||||
display="questions"
|
||||
example={{
|
||||
questions: {
|
||||
intent: {
|
||||
type: 'choice',
|
||||
instructions: 'What action is the user requesting?',
|
||||
criteria: {
|
||||
check_balance: 'Check the balance of an account',
|
||||
approve_transfer: 'Approve the pending transfer request',
|
||||
other: 'Something else',
|
||||
},
|
||||
},
|
||||
},
|
||||
}}
|
||||
/>
|
||||
|
||||
### Step 2: confidence-gated routing
|
||||
|
||||
```python theme={null}
|
||||
action = response.answers["intent"]
|
||||
|
||||
# Below 0.6 confidence on any action, route to a human
|
||||
if action.confidence < 0.6:
|
||||
route_to_support_agent(account_id)
|
||||
|
||||
elif action.choice == "check_balance":
|
||||
# Low stakes. 0.6 confidence is sufficient.
|
||||
show_balance(account_id)
|
||||
|
||||
elif action.choice == "approve_transfer":
|
||||
if action.confidence > 0.85:
|
||||
# High stakes, but high confidence. Safe to act automatically.
|
||||
approve_transfer(account_id)
|
||||
else:
|
||||
# High stakes, moderate confidence. Verify intent first.
|
||||
ask_user_to_confirm("Just to confirm: you would like to approve this transfer, is that correct?")
|
||||
|
||||
else:
|
||||
route_to_support_agent(account_id)
|
||||
```
|
||||
|
||||
The 0.6 floor catches anything the model is genuinely uncertain about. Above that floor, each action type has its own threshold based on the consequences of acting on a wrong classification. Checking a balance at 0.6 is fine because the worst case is the user having to listen to the balance read-out. But approving a transfer requires very high confidence (>0.85), otherwise the system should ask the user to confirm.
|
||||
|
||||
See [Confidence](/confidence) for more details on how to think about confidence in your systems.
|
||||
Reference in New Issue
Block a user