{}

{titlePrefix}Confusion Matrix (Normalized)

{}

Predicted

{}

{displayPredictedLabels.left}

{displayPredictedLabels.right}

{}

Actual

{displayActualLabels.top}

{showCounts &&

{displayMatrix.tl.count}

}

{formatValue(displayMatrix.tl.pct)}

{showCounts &&

{displayMatrix.tr.count}

}

{formatValue(displayMatrix.tr.pct)}

{}

{displayActualLabels.bottom}

{showCounts &&

{displayMatrix.bl.count}

}

{formatValue(displayMatrix.bl.pct)}

{showCounts &&

{displayMatrix.br.count}

}

{formatValue(displayMatrix.br.pct)}

{}

{displayFormat === "fraction" ? "0.0" : "0%"}

{palette.map((color, idx) =>

)}

{displayFormat === "fraction" ? "1.0" : "100%"}

; }; export const BooleanClassificationReport = ({report, negativeLabel = "Not Advanced", positiveLabel = "Advanced", negativeClass = "False", positiveClass = "True", maxWidth = 520}) => { const parseReport = reportStr => { const lines = reportStr.trim().split('\n').filter(line => line.trim()); const result = { classes: [], accuracy: null, macroAvg: null, weightedAvg: null, totalSupport: null }; for (const line of lines) { const parts = line.trim().split(/\s+/); if (parts[0] === 'precision') continue; if (parts.length >= 5 && !['accuracy', 'macro', 'weighted'].includes(parts[0])) { result.classes.push({ name: parts[0], precision: parseFloat(parts[1]), recall: parseFloat(parts[2]), f1: parseFloat(parts[3]), support: parseInt(parts[4], 10) }); } if (parts[0] === 'accuracy') { result.accuracy = parseFloat(parts[1]); result.totalSupport = parseInt(parts[2], 10); } if (parts[0] === 'macro' && parts[1] === 'avg') { result.macroAvg = { precision: parseFloat(parts[2]), recall: parseFloat(parts[3]), f1: parseFloat(parts[4]), support: parseInt(parts[5], 10) }; } if (parts[0] === 'weighted' && parts[1] === 'avg') { result.weightedAvg = { precision: parseFloat(parts[2]), recall: parseFloat(parts[3]), f1: parseFloat(parts[4]), support: parseInt(parts[5], 10) }; } } return result; }; const parsed = parseReport(report); if (parsed.classes.length < 2) { return

BooleanClassificationReport: Could not parse report. Expected at least 2 classes.

; } const negClass = parsed.classes.find(c => c.name === negativeClass) || parsed.classes[0]; const posClass = parsed.classes.find(c => c.name === positiveClass) || parsed.classes[1]; const tnPlusFp = negClass.support; const tpPlusFn = posClass.support; const tn = Math.round(negClass.recall * tnPlusFp); const fp = tnPlusFp - tn; const tp = Math.round(posClass.recall * tpPlusFn); const fn = tpPlusFn - tp; const tnPct = tn / tnPlusFp * 100; const fpPct = fp / tnPlusFp * 100; const fnPct = fn / tpPlusFn * 100; const tpPct = tp / tpPlusFn * 100; const rowStyle = { borderBottom: "1px solid rgba(148, 163, 184, 0.3)" }; const cellStyle = { padding: "0.5rem 0.125rem" }; const centerCellStyle = { textAlign: "center", padding: "0.5rem 0.125rem" }; return

{} {}

	Precision	Recall	F1-Score
{negativeLabel}	{negClass.precision.toFixed(2)}	{negClass.recall.toFixed(2)}	{negClass.f1.toFixed(2)}
{positiveLabel}	{posClass.precision.toFixed(2)}	{posClass.recall.toFixed(2)}	{posClass.f1.toFixed(2)}

{}

; }; export const DefinitionCard = ({children}) => { return

{children}

; }; export const Scale = ({low, mid, high, lowLabel = "Low", midLabel = "Mid", highLabel = "High", lowDescription, midDescription, highDescription, midColor = "yellow", inverted = false}) => { const lowColor = inverted ? "green" : "red"; const highColor = inverted ? "red" : "green"; const gradientId = inverted ? "greenToRed" : "redToGreen"; return

{low}

{mid &&

{mid}

}

{high}

{lowLabel}

{lowDescription &&

{lowDescription}

}

{mid &&

{midLabel}

{midDescription &&

{midDescription}

}

{highLabel}

{highDescription &&

{highDescription}

}

; }; SQL Adherence assesses whether the generated SQL query semantically aligns with the intent of the natural language query provided. ## Metric definition SQL Adherence — A binary metric that evaluates whether the generated SQL query accurately reflects the user's intent as expressed in the natural language request. * Type: Binary * 1 (Valid): The query semantically matches the intention of the natural language query and answers it accurately. * 0 (Invalid): The query doesn't answer the natural language query accurately. This metric focuses purely on the "meaning" and "intent" of the SQL query, independent of syntactic correctness or schema validity. It validates that the operations performed (filtering, sorting, aggregating) match what the user requested. This metric requires the original natural language query to be provided. Without the NL query, semantic alignment cannot be evaluated and the metric will not produce meaningful results. Here's a scale that shows the relationship between SQL Adherence and potential impact on your AI system: Scale is 0-100 and is derived from binary judgments converted into a confidence score. ## Calculation method SQL Adherence is computed through a multi-step evaluation process: One or more evaluation requests are sent to an LLM evaluator to analyze the semantic alignment between the natural language query and the generated SQL. A specialized chain-of-thought prompt guides the model to evaluate whether the SQL accurately reflects user intent across selection, filtering, aggregation, and logical relationships. The evaluator analyzes the query for semantic correctness, checking column selection alignment, filtering logic (under/over-filtering), aggregation functions, and proper use of ORDER BY/LIMIT clauses. Based on the evaluation, a binary score is assigned: 1 (Valid) if the query semantically matches user intent, or 0 (Invalid) if semantic mismatches are detected. This metric is computed by prompting an LLM and may require multiple LLM calls to compute, which can impact usage and billing. ## Supported nodes * LLM span Required inputs: * The natural language query posed by the user * The generated SQL query (output) Optional inputs: * SQL dialect, schema information, and domain knowledge hints This metric does NOT check syntactic correctness or schema validity—it focuses purely on semantic alignment with user intent. Use [SQL Correctness](/concepts/metrics/text2sql/sql-correctness) for syntactic and schematic validation. ## What constitutes adherent (1) * **Semantic Correctness**: The SQL accurately reflects user intent and retrieves the exact required data. * **Completeness**: The query includes all necessary components to fulfill the request. * **Minimal Redundancy** (preferred): Uses the most efficient/standard formulation, though returning the identical result set is the core requirement. ## What constitutes non-adherent (0) ### Semantic Mismatch (Incorrect Logic) The executable SQL doesn't match user intent, including: * Wrong column/table referenced for the requested data * Incorrect filtering/condition (wrong WHERE/HAVING operator or value) * Incorrect aggregation/grouping (wrong aggregate function, missing GROUP BY) * Incorrect ordering/limiting (ORDER BY/LIMIT error) ### Ambiguity Not Resolved * System failed to interpret an ambiguous query * No attempt to prompt for clarification when needed ### Missing Components * SQL omits necessary clauses required to fulfill user intent * Missing a necessary JOIN to connect related tables * Missing WHERE clause for a filtered request ## Example use cases * Validating that a natural language to SQL assistant correctly interprets user questions. * Ensuring data analytics queries return the data users actually requested. * Quality assurance for business intelligence tools before presenting query results. * Comparing different LLM models or prompts for semantic accuracy in SQL generation. **Example**: A data analytics assistant where a user asks "Show me the top 5 customers by total sales" — SQL Adherence validates that the query correctly orders by sales amount in descending order, limits to 5 results, and selects customer information rather than product data. ## Best practices Providing the SQL dialect and schema information helps the evaluator understand dialect-specific syntax and table relationships for more accurate assessment. Provide domain knowledge hints (e.g., "premium customer means tier = 'gold'") for more accurate intent matching. Use alongside SQL Correctness to validate both semantic intent and syntactic/schematic validity. Use continuous learning via human feedback to improve the evaluator's understanding of your domain-specific queries. ## Performance Benchmarks We evaluated SQL Adherence against human expert labels on an internal dataset of natural language to SQL samples using top frontier models. | Model | F1 (True) | | :---------------------- | :-------: | | GPT-4.1 | 0.91 | | GPT-4.1 Mini (judges=3) | 0.92 | | Gemini 3 Flash Preview | 0.94 | | Claude Sonnet 4.5 | 0.92 | ### GPT-4.1 Classification Report Benchmarks based on internal evaluation dataset. Performance may vary by use case. ## Related Resources If you would like to dive deeper or start implementing SQL Adherence, check out the following resources: ### Examples * [SQL Adherence Examples](https://app.galileo.ai) - Log in and explore the "SQL Adherence" Log Stream in the "Preset Metric Examples" Project to see this metric in action. ### Related Concepts * [SQL Correctness](/concepts/metrics/text2sql/sql-correctness) * [SQL Efficiency](/concepts/metrics/text2sql/sql-efficiency) * [SQL Injection](/concepts/metrics/text2sql/sql-injection)