/** * Admin Panel: AI Observability Center (GET /admin/ai-observability/*). * * Two real data sources, merged server-side (api_blueprint.py's * ai_observability_summary_endpoint), not duplicated here: * - logs/llm_observability.jsonl (existing, per-LLM-call token/cost/ * latency -- same data Dashboard.jsx's three stat tiles already show) * - logs/ai_traces.jsonl (one entry per AI PIPELINE SPAN -- a trace_id * groups every span of one end-to-end workflow run, e.g. one resume's * full parse: document extraction -> chunking -> embedding -> * retrieval ranking -> two LLM calls -> output validation -> * persistence, all correlated). This is the one real, single-process, * end-to-end AI workflow in this repo (ResumeParser.py) instrumented * start to finish -- see docs/AI_OBSERVABILITY_AZURE_HUB.md's * investigation notes on why that's the highest-value real workflow * to trace here (ResumeMEtrics.py runs as a separate process, so it * can't share one in-process trace with the parsing step). * * Never fabricates data: an empty log file returns real zeros, and a * latency percentile computed from fewer than 5 data points is shown as * an explicit "not enough data" state, never a number. In mock mode the * trace list is clearly illustrative demo data (see mockApiClient.js's * own comment on MOCK_AI_TRACES); the topbar's "API: mock" label is this * app's existing, already-established way of marking that. * * Deliberately NOT built this pass (stated here, not hidden): the AI * Evaluation Center (no eval dataset/run infrastructure exists), alert * rules (no real notification mechanism exists in this repo to wire * them to -- faking "active" alerts was explicitly out of scope), and * RAG quality metrics like context precision/faithfulness (no evaluation * harness exists to compute them from). See * docs/AI_OBSERVABILITY_AZURE_HUB.md for the full list. */ function AIObservabilityCenter(props) { var api = props.api; var tabState = React.useState("overview"); // "overview" | "traces" var tab = tabState[0]; var setTab = tabState[1]; var selectedTraceState = React.useState(null); var selectedTraceId = selectedTraceState[0]; var setSelectedTraceId = selectedTraceState[1]; return (

AI Observability Center

What happened during each AI workflow run, stage by stage -- latency, tokens, cost, and failures, correlated into real multi-stage traces, not just isolated LLM call metrics.

{tab === "overview" && } {tab === "traces" && ( selectedTraceId ? : )} {tab === "budget" && } {tab === "cache" && } {tab === "routing" && }
); } function AIObservabilityOverview(props) { var api = props.api; var onOpenTrace = props.onOpenTrace; var snapshot = CDPI_REPORT_SNAPSHOT.useReportSnapshot("ai_observability_overview", function () { return api.aiObservabilitySummary(); }); var data = { loading: snapshot.state.loading && !snapshot.state.data, error: snapshot.state.error, data: snapshot.state.data }; var load = snapshot.refresh; if (data.error && !data.data) return
Couldn't load observability summary: {data.error}
; if (data.loading) return

Loading…

; var d = data.data; return (
Snapshot from {snapshot.state.lastUpdated ? new Date(snapshot.state.lastUpdated).toLocaleString() : "just now"}

AI requests (end-to-end traces)

Trace latency (end-to-end, per request)

Span latency (per pipeline stage)

Cost

Pricing version {d.cost.pricing_version} (last updated {d.cost.pricing_last_updated}) — {d.cost.pricing_source}

{Object.keys(d.cost.by_workflow).length === 0 && } {Object.keys(d.cost.by_workflow).sort().map(function (wf) { return ; })}
WorkflowCost
No priced spans yet.
{wf}${d.cost.by_workflow[wf].toFixed(6)}

Instrumentation coverage

{d.instrumentation_coverage.covered_stage_count} of {d.instrumentation_coverage.documented_stage_count} documented pipeline stages have at least one recorded span ({d.instrumentation_coverage.coverage_pct}%).

{d.instrumentation_coverage.uninstrumented_stages.length > 0 && (

Not yet observed: {d.instrumentation_coverage.uninstrumented_stages.join(", ")}

)}

Not applicable to this repo (no such feature exists): {d.instrumentation_coverage.not_applicable_to_this_repo.join("; ")}

By LLM prompt label

{Object.keys(d.llm_calls.by_label).sort().map(function (label) { var stats = d.llm_calls.by_label[label]; return ( ); })} {Object.keys(d.llm_calls.by_label).length === 0 && ( )}
LabelCallsTokensCost
{label}{stats.calls}{stats.tokens.toLocaleString()} ${stats.cost_usd.toFixed(4)}
No LLM calls recorded yet.

Recent traces

{d.recent_traces.map(function (t) { return ( ); })} {d.recent_traces.length === 0 && }
TraceWorkflowStatusSpansDurationStarted
{t.correlation_id || t.trace_id} {t.workflow} {t.status} {t.span_count} {t.total_duration_ms !== null ? t.total_duration_ms + " ms" : "—"} {t.started_at ? new Date(t.started_at).toLocaleString() : "—"}
No traces recorded yet.
); } function LatencyRow(props) { var stats = props.stats; if (stats.insufficient_sample) { return (

Only {stats.sample_count} data point(s) recorded -- not enough to compute P50/P95/P99 (needs at least 5). {stats.avg_ms !== null && Average so far: {stats.avg_ms} ms.}

); } return (
); } function AITraceExplorer(props) { var api = props.api; var onOpenTrace = props.onOpenTrace; var filterState = React.useState({ workflow: "", status: "", correlation_id: "", trace_id: "" }); var filters = filterState[0]; var setFilters = filterState[1]; var pageState = React.useState(0); var page = pageState[0]; var setPage = pageState[1]; var PAGE_SIZE = 20; // Snapshot-cached per filter+page combination -- the first time a given // combination is viewed it fetches for real (so changing a filter still // gets fresh results, same as before); revisiting that exact // filter+page later shows its own cached snapshot instantly. var snapshotKey = "ai_trace_explorer:" + JSON.stringify([filters.workflow, filters.status, filters.correlation_id, filters.trace_id, page]); var snapshot = CDPI_REPORT_SNAPSHOT.useReportSnapshot(snapshotKey, function () { var query = { limit: PAGE_SIZE, offset: page * PAGE_SIZE }; if (filters.workflow) query.workflow = filters.workflow; if (filters.status) query.status = filters.status; if (filters.correlation_id) query.correlation_id = filters.correlation_id; if (filters.trace_id) query.trace_id = filters.trace_id; return api.listAiTraces(query); }); var load = snapshot.refresh; var data = { loading: snapshot.state.loading && !snapshot.state.data, error: snapshot.state.error, traces: (snapshot.state.data && snapshot.state.data.traces) || [], count: (snapshot.state.data && snapshot.state.data.count) || 0, }; function setFilter(name, value) { setPage(0); setFilters(function (f) { return Object.assign({}, f, { [name]: value }); }); } var totalPages = Math.max(1, Math.ceil(data.count / PAGE_SIZE)); return (
{data.error &&
Couldn't load traces: {data.error}
} {data.loading &&

Loading…

} {!data.loading && !data.error && (
{data.traces.map(function (t) { return ( ); })} {data.traces.length === 0 && }
TraceWorkflowStatusSpansStagesDurationStarted
{t.correlation_id || t.trace_id} {t.workflow} {t.status} {t.span_count} {t.stages.join(" → ")} {t.total_duration_ms !== null ? t.total_duration_ms + " ms" : "—"} {t.started_at ? new Date(t.started_at).toLocaleString() : "—"}
No traces match these filters.
Page {page + 1} of {totalPages} ({data.count} total)
)}
); } function AITraceDetail(props) { var api = props.api; var traceId = props.traceId; var onBack = props.onBack; var state = React.useState({ loading: true, error: null, trace: null, spans: [] }); var data = state[0]; var setData = state[1]; React.useEffect(function () { setData({ loading: true, error: null, trace: null, spans: [] }); api.getAiTrace(traceId) .then(function (result) { setData({ loading: false, error: null, trace: result.trace, spans: result.spans }); }) .catch(function (err) { setData({ loading: false, error: err.message, trace: null, spans: [] }); }); }, [traceId]); return (
{data.error &&
Couldn't load trace: {data.error}
} {data.loading &&

Loading…

} {!data.loading && !data.error && data.trace && (

{data.trace.correlation_id || data.trace.trace_id}

Trace ID {data.trace.trace_id} — workflow {data.trace.workflow} —{" "} {data.trace.status} {" "}— {data.trace.span_count} span(s), {data.trace.total_duration_ms !== null ? data.trace.total_duration_ms + " ms total" : "duration unavailable"} {data.trace.total_cost_usd !== null && , ${data.trace.total_cost_usd.toFixed(6)} estimated cost}

{data.spans.map(function (s, i) { var meta = s.metadata || {}; return ( ); })}
#StageStatusDurationModelTokensCostError
{i + 1} {s.stage} {s.status} {s.duration_ms !== null && s.duration_ms !== undefined ? s.duration_ms + " ms" : "—"} {meta.model || "—"}{meta.prompt_version ? " (" + meta.prompt_version + ")" : ""} {meta.total_tokens !== undefined ? meta.total_tokens : "—"} {meta.cost_usd !== undefined && meta.cost_usd !== null ? "$" + meta.cost_usd.toFixed(6) : "—"} {s.error || ""}
)}
); } /** * F-02 (Tokenomics calculator, minimal slice): real, period-scoped * (current UTC calendar month) usage + a real, versioned, admin- * configurable budget/warning-threshold check -- see tokenomics.py's * module docstring and docs/fde/IMPLEMENTATION_STATUS.md for exactly * what's in and out of scope for this slice. This is deliberately added * as a tab on the EXISTING AI Observability Center (which already shows * all-time cost/token data) rather than a new, disconnected page. */ function TokenomicsBudgetPanel(props) { var api = props.api; var state = React.useState({ loading: true, error: null, active: null, draft: null, status: null }); var data = state[0]; var setData = state[1]; var formState = React.useState({ monthly_budget_usd: "", warning_threshold_pct: "80" }); var form = formState[0]; var setForm = formState[1]; var saveState = React.useState({ saving: false, error: null }); var save = saveState[0]; var setSave = saveState[1]; function load() { setData(function (d) { return Object.assign({}, d, { loading: true, error: null }); }); Promise.all([api.getTokenomicsConfig(), api.getTokenomicsStatus()]) .then(function (results) { var cfg = results[0]; var status = results[1]; var effective = cfg.draft || cfg.active; setForm({ monthly_budget_usd: effective.monthly_budget_usd === null || effective.monthly_budget_usd === undefined ? "" : String(effective.monthly_budget_usd), warning_threshold_pct: String(effective.warning_threshold_pct != null ? effective.warning_threshold_pct : 80), }); setData({ loading: false, error: null, active: cfg.active, draft: cfg.draft, status: status }); }) .catch(function (err) { setData(function (d) { return Object.assign({}, d, { loading: false, error: err.message }); }); }); } React.useEffect(function () { load(); }, []); function saveDraft() { setSave({ saving: true, error: null }); var budget = form.monthly_budget_usd.trim() === "" ? null : Number(form.monthly_budget_usd); var threshold = parseInt(form.warning_threshold_pct, 10); api.saveTokenomicsConfigDraft({ monthly_budget_usd: budget, warning_threshold_pct: threshold }) .then(function () { setSave({ saving: false, error: null }); load(); }) .catch(function (err) { setSave({ saving: false, error: err.message }); }); } function activate() { setSave({ saving: true, error: null }); api.activateTokenomicsConfig() .then(function () { setSave({ saving: false, error: null }); load(); }) .catch(function (err) { setSave({ saving: false, error: err.message }); }); } if (data.error) return
Couldn't load budget configuration: {data.error}
; if (data.loading) return

Loading…

; var status = data.status.budget; var usage = data.status.usage; var statusTone = status.status === "exceeded" ? "bad" : status.status === "warning" ? "warn" : status.status === "ok" ? "good" : "neutral"; return (

This month's usage ({usage.period})

{usage.unknown_cost_call_count > 0 && (

{usage.unknown_cost_call_count} call(s) this month have no recorded cost -- excluded from the total above, not counted as $0.

)} {status.configured && (

${status.spent_usd.toFixed(4)} of ${status.budget_usd} budget ({status.percent_used}% used).

)} {!status.configured &&

No monthly budget configured -- set one below to enable warning/exceeded alerts.

}

Budget configuration

{data.draft &&

An unsaved draft exists below. Activate it to make it the real budget check.

} {save.error &&
{save.error}
}

Active: {data.active.monthly_budget_usd === null ? "no budget set" : "$" + data.active.monthly_budget_usd + "/mo"}, warning at {data.active.warning_threshold_pct}%.

); } /** * F-18 (LLM caching, minimal slice). Exact-match caching only, scoped * to ResumeParser.py's two cacheable call sites -- see llm_cache.py's * module docstring for exactly why only those two, and why this is * safe against cross-candidate leakage by construction (the cache key * already includes each candidate's own resume content). */ function LlmCachePanel(props) { var api = props.api; var state = React.useState({ loading: true, error: null, data: null }); var data = state[0]; var setData = state[1]; var clearState = React.useState({ clearing: false, error: null }); var clearing = clearState[0]; var setClearing = clearState[1]; function load() { setData(function (d) { return Object.assign({}, d, { loading: true, error: null }); }); api.getLlmCacheStats() .then(function (result) { setData({ loading: false, error: null, data: result }); }) .catch(function (err) { setData(function (d) { return Object.assign({}, d, { loading: false, error: err.message }); }); }); } React.useEffect(function () { load(); }, []); function clearCache() { setClearing({ clearing: true, error: null }); api.clearLlmCache() .then(function () { setClearing({ clearing: false, error: null }); load(); }) .catch(function (err) { setClearing({ clearing: false, error: err.message }); }); } if (data.error) return
Couldn't load cache stats: {data.error}
; if (data.loading) return

Loading…

; var d = data.data; return (
{clearing.error &&
{clearing.error}
}

Exact-match caching for resume-processing LLM calls only (chunk summarization and structured extraction) -- the one real case where identical input (reprocessing the same resume/JD) genuinely recurs and the result is only ever written back into that same candidate's own record. Chatbots, interview-question generation, and evaluation judges are not cached.

{Object.keys(d.cache.by_label).map(function (label) { var s = d.cache.by_label[label]; return ; })} {Object.keys(d.cache.by_label).length === 0 && }
LabelCached EntriesHits
{label}{s.entry_count}{s.hit_count}
Cache is empty.
); } var MODEL_ROUTING_LABEL_TEXT = { chat_candidate_qa: "Candidate Chat Q&A", interview_questions_generation: "Interview Question Generation", interview_bot_answer_scoring: "Interview Answer Scoring (voicebot CLI -- not live-wired, see note below)", rag_groundedness_judge: "RAG Groundedness Judge", ai_match_quality_judge: "AI Match-Quality Judge", jd_generation: "JD Generation (AI draft)", jd_document_extraction: "JD Document Extraction (upload)", }; /** * F-17 (model routing, minimal slice). A centralized override for the 6 * real LLM call sites that had NO admin-configurable model before this * feature -- resume-processing's 2 steps already have their own model * field in the RAG Configuration Console and are deliberately not * duplicated here. See model_routing.py's module docstring. */ function ModelRoutingPanel(props) { var api = props.api; var state = React.useState({ loading: true, error: null, data: null }); var data = state[0]; var setData = state[1]; var formState = React.useState({}); var form = formState[0]; var setForm = formState[1]; var saveState = React.useState({ saving: false, error: null, savedAt: null }); var save = saveState[0]; var setSave = saveState[1]; function load() { setData(function (d) { return Object.assign({}, d, { loading: true, error: null }); }); api.getModelRoutingConfig() .then(function (result) { setForm(Object.assign({}, result.routes)); setData({ loading: false, error: null, data: result }); }) .catch(function (err) { setData(function (d) { return Object.assign({}, d, { loading: false, error: err.message }); }); }); } React.useEffect(function () { load(); }, []); function saveRoutes() { setSave({ saving: true, error: null, savedAt: null }); var routes = {}; Object.keys(form).forEach(function (label) { if (form[label] && form[label].trim()) routes[label] = form[label].trim(); }); api.saveModelRoutingConfig(routes) .then(function () { setSave({ saving: false, error: null, savedAt: new Date().toISOString() }); load(); }) .catch(function (err) { setSave({ saving: false, error: err.message, savedAt: null }); }); } if (data.error) return
Couldn't load model routing config: {data.error}
; if (data.loading) return

Loading…

; var d = data.data; return (

Sets which model each of these real LLM call sites uses. Leave a field blank to keep using that site's real default ({"gpt-3.5-turbo"} today for every one of these -- unchanged unless overridden here). Resume processing's chunk-summarization and structured-extraction steps are configured separately, in the RAG Configuration Console.

{save.error &&
{save.error}
} {save.savedAt &&
Saved.
}
{d.routable_labels.map(function (label) { return ( ); })}
Call SiteModel OverrideDefault
{MODEL_ROUTING_LABEL_TEXT[label] || label} {d.default_models[label]}
{d.updated_by &&

Last updated by {d.updated_by} at {d.updated_at ? new Date(d.updated_at).toLocaleString() : "—"}.

}
); } // StatTile is defined globally by Dashboard.jsx -- reused here (no // bundler/module system in this app, see frontend/README.md), same // pattern RecruiterReports.jsx already uses to reuse AdminReports.jsx's // ChartCanvas/CHART_COLORS/countBy.