/** * Admin Panel: Service Observability Report -- covers all of CDPI's real, * discovered subsystems (not just Azure/AI), reusing every existing real * endpoint rather than re-implementing them: * - Real Azure Monitor / Log Analytics QUERY-BACK (azure_monitor_query.py, * manage/observability-report/*) -- the one genuinely NEW telemetry * source this feature adds; everything else below already existed. * - Per-service Azure connectivity/health -> manage/azure-readiness (same * data Azure Readiness' Services tab shows; reused here as a compact * matrix, not duplicated logic). * - Document/resume processing -> manage/reports/resume-processing (real * document_extraction spans, see that endpoint's own docstring). * - External integrations (WhatsApp/email) -> * manage/reports/communication-delivery (real comms_delivery_log * records; email has NO delivery logging at all today -- a real, * stated gap, not a zero-volume coincidence, see that endpoint). * - Local LLM/trace token-cost-latency detail + the full Trace Explorer -> * AI Observability Center (link out, not duplicated). * - Real Azure resource cost -> Azure Readiness' Cost & Usage tab (link out). * * Every number is labeled with its real source. `overall_status` is * computed server-side from real ingestion evidence, never just "the page * loaded" or "a connection string is set". One section failing to load * (e.g. a role lacking comms.delivery.read) never blanks the rest of the * page -- each section fetches and fails independently. * * CUSTOMIZATION: which sections render is user-controlled (Customize panel * below) and persisted to localStorage ONLY -- this is a per-browser, * per-device convenience, explicitly NOT a server-persisted/per-account * preference (no backend dashboard-config model exists in this repo; see * docs/AZURE_OBSERVABILITY.md for the scoping decision to skip full * drag-and-drop/resize/named-layouts given this app's scale). Hidden * widgets are still fetched as "available" if their data loaded -- hiding * is a display preference, not a fetch skip, so toggling one back on is * instant. */ var OBS_REPORT_WIDGETS = [ { id: "kpis", label: "Executive KPIs" }, { id: "request_volume", label: "Request Volume (Azure Monitor)" }, { id: "ai_usage", label: "AI Model Usage (Azure Monitor)" }, { id: "ai_usage_chart", label: "AI Token Usage by Model (chart)" }, { id: "recent_errors", label: "Error Analysis (Azure Monitor)" }, { id: "service_health", label: "Azure Service Health Matrix" }, { id: "service_health_chart", label: "Service Health Breakdown (chart)" }, { id: "workflow_breakdown", label: "Traces by Workflow: Succeeded vs Failed (chart)" }, { id: "workflow_latency", label: "Average Duration by Workflow (chart)" }, { id: "document_processing", label: "Document / Resume Processing" }, { id: "external_integrations", label: "External Integrations (WhatsApp / Email)" }, { id: "local_recap", label: "Local Observability Recap" }, ]; var OBS_REPORT_WIDGETS_STORAGE_KEY = "cdpi_observability_report_hidden_widgets"; function _loadHiddenWidgets() { try { var raw = window.localStorage.getItem(OBS_REPORT_WIDGETS_STORAGE_KEY); return raw ? JSON.parse(raw) : []; } catch (e) { return []; } } function _saveHiddenWidgets(ids) { try { window.localStorage.setItem(OBS_REPORT_WIDGETS_STORAGE_KEY, JSON.stringify(ids)); } catch (e) { /* per-device convenience only; ignore quota/availability errors */ } } /** Adapts a persisted observability_snapshots.py snapshot into the same * summary/telemetry/serviceHealth/docProcessing/comms shapes the existing * chart/table sub-components below already consume -- so this snapshot * architecture change doesn't require rewriting every widget, only how * data gets INTO them. A source with status != "success" (not_configured, * failed, or simply absent from a snapshot taken before that source * existed) maps to an honest "unavailable" shape, never a fabricated * zero/empty-but-successful one. A "failed" source that has a real * last_known_good result from an earlier refresh uses THAT instead of * going blank -- a bad refresh never erases good data (see * observability_snapshots.run_refresh()'s docstring). */ function _sourceData(snapshot, sourceId) { var src = snapshot && snapshot.sources && snapshot.sources[sourceId]; if (!src) return null; if (src.status === "success") return src.data; if (src.status === "failed" && src.last_known_good && src.last_known_good.status === "success") return src.last_known_good.data; return null; } function _sourceUnavailableReason(snapshot, sourceId) { var src = snapshot && snapshot.sources && snapshot.sources[sourceId]; if (!src) return "No refresh has collected this source yet."; if (src.status === "not_configured") return "Not configured."; if (src.status === "failed") return "Last refresh failed: " + (src.error || "unknown error") + (src.last_known_good ? " (showing last known good data from an earlier refresh)" : ""); return null; } function _adaptSnapshot(snapshot) { var reqVol = _sourceData(snapshot, "azure_request_volume") || { available: false, buckets: [] }; var latency = _sourceData(snapshot, "azure_latency") || { available: false }; var aiUsage = _sourceData(snapshot, "azure_ai_usage") || { available: false, by_model: [] }; var recentErrors = _sourceData(snapshot, "azure_recent_errors") || { available: false, errors: [] }; var telHealth = _sourceData(snapshot, "azure_telemetry_health") || { azure_configured: false, ingestion_confirmed: false, detail: _sourceUnavailableReason(snapshot, "azure_telemetry_health") || "Unavailable." }; var localTraces = _sourceData(snapshot, "local_ai_traces") || { total_spans: 0, by_workflow: {} }; var localLlm = _sourceData(snapshot, "local_llm_calls") || { total_calls: 0, total_tokens: 0, total_cost_usd: 0, by_label: {} }; var totalReq = reqVol.available ? reqVol.buckets.reduce(function (s, b) { return s + b.total; }, 0) : null; var succeeded = reqVol.available ? reqVol.buckets.reduce(function (s, b) { return s + b.succeeded; }, 0) : null; var failedReq = reqVol.available ? reqVol.buckets.reduce(function (s, b) { return s + b.failed; }, 0) : null; var aiRequests = aiUsage.available ? aiUsage.by_model.reduce(function (s, m) { return s + m.calls; }, 0) : null; var aiTokens = aiUsage.available ? aiUsage.by_model.reduce(function (s, m) { return s + (m.total_tokens || 0); }, 0) : null; var aiCost = aiUsage.available ? Math.round(aiUsage.by_model.reduce(function (s, m) { return s + (m.total_cost_usd || 0); }, 0) * 1000000) / 1000000 : null; var overallStatus = "UNKNOWN"; if (!snapshot || !snapshot.collected_at) overallStatus = "NOT_CONFIGURED"; else if (!telHealth.azure_configured) overallStatus = "UNKNOWN"; else if (!telHealth.ingestion_confirmed) overallStatus = "CONFIGURED_NO_INGESTION"; else if (failedReq && totalReq && failedReq / totalReq > 0.1) overallStatus = "DEGRADED"; else overallStatus = "HEALTHY"; var summary = { reporting_period_hours: snapshot && snapshot.hours_window, generated_at: snapshot && snapshot.collected_at, overall_status: overallStatus, telemetry_health: telHealth, azure_telemetry: { configured: telHealth.azure_configured, total_requests: totalReq, succeeded_requests: succeeded, failed_requests: failedReq, success_rate: (totalReq ? succeeded / totalReq : null), failure_rate: (totalReq ? failedReq / totalReq : null), latency: latency, ai_requests: aiRequests, ai_total_tokens: aiTokens, ai_total_cost_usd: aiCost, unavailable_reason: reqVol.available ? null : _sourceUnavailableReason(snapshot, "azure_request_volume"), }, local_observability: { llm_calls: localLlm, ai_traces: localTraces, note: "Sourced from this process's local JSONL logs, as of the last refresh -- always available regardless of Azure Monitor configuration.", }, }; var telemetry = { reporting_period_hours: snapshot && snapshot.hours_window, request_volume: reqVol, ai_usage: aiUsage, recent_errors: recentErrors, }; return { summary: summary, telemetry: telemetry }; } function AzureObservabilityReport(props) { var api = props.api; var hoursState = React.useState(24); var hours = hoursState[0]; var setHours = hoursState[1]; // Persistent frontend snapshot of the already-persisted backend // snapshot (api.getObservabilitySnapshot() -- itself a pure read of // observability_snapshots.py's cached last-known-good data, never a // real Azure collection). This is a second, frontend-side cache layer // on top of that: per the "no loading flash on revisit" requirement, // opening this page never shows "Loading..." once it's been opened // once in this browser, even though the backend read is already fast. var pageSnapshot = CDPI_REPORT_SNAPSHOT.useReportSnapshot("azure_observability_report:" + hours, function () { return api.getObservabilitySnapshot(); }); var snapshot = pageSnapshot.state.data; var refreshingState = React.useState(false); var refreshing = refreshingState[0]; var setRefreshing = refreshingState[1]; var exportingState = React.useState(false); var exporting = exportingState[0]; var setExporting = exportingState[1]; var historyState = React.useState([]); var refreshHistory = historyState[0]; var setRefreshHistory = historyState[1]; var showHistoryState = React.useState(false); var showHistory = showHistoryState[0]; var setShowHistory = showHistoryState[1]; var serviceHealth = { services: (_sourceData(snapshot, "service_health") || {}).services || null }; var serviceHealthError = _sourceUnavailableReason(snapshot, "service_health"); var docProcessing = _sourceData(snapshot, "document_processing"); var docProcessingError = _sourceUnavailableReason(snapshot, "document_processing"); var comms = _sourceData(snapshot, "comms_delivery"); var commsError = _sourceUnavailableReason(snapshot, "comms_delivery"); var customizingState = React.useState(false); var customizing = customizingState[0]; var setCustomizing = customizingState[1]; var hiddenWidgetsState = React.useState(_loadHiddenWidgets); var hiddenWidgets = hiddenWidgetsState[0]; var setHiddenWidgets = hiddenWidgetsState[1]; function isVisible(widgetId) { return hiddenWidgets.indexOf(widgetId) === -1; } function toggleWidget(widgetId) { var next = isVisible(widgetId) ? hiddenWidgets.concat([widgetId]) : hiddenWidgets.filter(function (id) { return id !== widgetId; }); setHiddenWidgets(next); _saveHiddenWidgets(next); } function resetWidgets() { setHiddenWidgets([]); _saveHiddenWidgets([]); } // The persisted-data read itself (NEVER a real Azure/local collection // -- per the "manual refresh is the default; opening the dashboard // must not auto-collect" requirement) is `pageSnapshot` above -- // fetched once per `hours` the first time that window is viewed in // this browser, cached after that, so no "Loading..." flash on revisit. var loading = pageSnapshot.state.loading && !pageSnapshot.state.data; var error = pageSnapshot.state.error; /** The ONLY action that collects real data -- explicit user click, * never automatic. */ function refreshAll() { setRefreshing(true); api.refreshObservability(hours) .then(function () { window.CDPI_AZURE_CACHE.invalidatePrefix("azure-readiness:"); // keep Azure Readiness/Service Hub pages consistent with this real refresh return pageSnapshot.refresh(); // re-reads what the refresh just persisted, updating this page's own snapshot too }) .catch(function (err) { window.alert(err.message); }) .then(function () { setRefreshing(false); }); } function loadHistory() { api.getObservabilityRefreshHistory(20).then(function (r) { setRefreshHistory(r.history || []); }).catch(function () { setRefreshHistory([]); }); } var adapted = _adaptSnapshot(snapshot); var summary = adapted.summary; var telemetry = adapted.telemetry; function doExport() { setExporting(true); api.exportObservabilityReportCsv(hours) .then(function (csvText) { var blob = new Blob([csvText], { type: "text/csv" }); var url = URL.createObjectURL(blob); var a = document.createElement("a"); a.href = url; a.download = "azure_observability_request_volume_" + hours + "h.csv"; document.body.appendChild(a); a.click(); document.body.removeChild(a); URL.revokeObjectURL(url); }) .catch(function (err) { window.alert("Export failed: " + err.message); }) .then(function () { setExporting(false); }); } return (

Service Observability Report

Real telemetry across CDPI's actual subsystems -- Azure Monitor-ingested metrics, per-service Azure connectivity, document/resume processing, and external-integration delivery -- composed from this app's existing real endpoints, not re-implemented. See the AI Observability Center for full local trace detail, and Azure Readiness for live per-service checks and real cost.

{snapshot && snapshot.collected_at ? Showing data collected {new Date(snapshot.collected_at).toLocaleString()} ({snapshot.succeeded_count} succeeded, {snapshot.failed_count} failed, {snapshot.skipped_count} not configured) -- opening this page never re-queries Azure; click "Refresh All" to collect current data. : No refresh has ever been run. Click "Refresh All" to collect real data for the first time.}

{showHistory && (

Refresh History

{refreshHistory.length === 0 &&

No past refreshes recorded.

} {refreshHistory.length > 0 && ( {refreshHistory.map(function (h) { return ; })}
CollectedTriggered bySucceededFailedNot configured
{new Date(h.collected_at).toLocaleString()}{h.triggered_by}{h.succeeded_count}{h.failed_count}{h.skipped_count}
)}
)} {customizing && (

Customize this dashboard

Which sections to show -- saved in this browser only (not a server-side, per-account preference; see docs/AZURE_OBSERVABILITY.md).

{OBS_REPORT_WIDGETS.map(function (w) { return ( ); })}
)} {error &&
{error}
} {loading &&

Loading…

} {!loading && ( {isVisible("kpis") && }
{isVisible("request_volume") && } {isVisible("ai_usage") && } {isVisible("ai_usage_chart") && } {isVisible("service_health_chart") && } {isVisible("workflow_breakdown") && } {isVisible("workflow_latency") && }
{isVisible("recent_errors") && } {isVisible("service_health") && } {isVisible("document_processing") && } {isVisible("external_integrations") && } {isVisible("local_recap") && }
)}
); } var OBS_REPORT_STATUS_BADGE = { HEALTHY: "badge-good", DEGRADED: "badge-neutral", CONFIGURED_NO_INGESTION: "badge-neutral", NOT_CONFIGURED: "badge-bad", UNKNOWN: "badge-neutral", }; function ReportHeader(props) { var s = props.summary; return (

Overall status: {s.overall_status}

Reporting period: last {props.hours}h · Generated: {s.generated_at ? new Date(s.generated_at).toLocaleString() : "never (no refresh yet)"} ·{" "} Data source: Azure Monitor (Log Analytics) + local JSONL · Environment: this app's configured Azure resource group

Last successful Azure telemetry ingestion:{" "} {s.telemetry_health.last_ingested_at ? new Date(s.telemetry_health.last_ingested_at).toLocaleString() : none confirmed -- {s.telemetry_health.detail}}

); } function Kpi(props) { var displayValue = (props.value === null || props.value === undefined) ? "—" : props.value; return (
{displayValue}
{props.label}
{props.unavailable &&
Unavailable: {props.unavailable}
}
); } function KpiRow(props) { var at = props.summary.azure_telemetry; var reason = at.unavailable_reason; return (
); } function RequestVolumeChart(props) { var t = props.telemetry; if (!t) return null; var rv = t.request_volume; if (!rv.available) { return

Unavailable — {rv.detail}

; } if (rv.buckets.length === 0) { return

No requests recorded in this window.

; } var config = { type: "line", data: { labels: rv.buckets.map(function (b) { return new Date(b.timestamp).toLocaleString(); }), datasets: [ { label: "Succeeded", data: rv.buckets.map(function (b) { return b.succeeded; }), borderColor: CHART_COLORS.good, backgroundColor: CHART_COLORS.good, tension: 0.2 }, { label: "Failed", data: rv.buckets.map(function (b) { return b.failed; }), borderColor: CHART_COLORS.bad, backgroundColor: CHART_COLORS.bad, tension: 0.2 }, ], }, options: { responsive: true, maintainAspectRatio: false, scales: { y: { beginAtZero: true } } }, }; return ( ); } function AiUsageTable(props) { var t = props.telemetry; if (!t) return null; var usage = t.ai_usage; return (

AI Model Usage

{!usage.available &&

Unavailable — {usage.detail}

} {usage.available && usage.by_model.length === 0 &&

No AI calls recorded in this window.

} {usage.available && usage.by_model.length > 0 && ( {usage.by_model.map(function (m, i) { return ( ); })}
ModelProviderCallsFailedTokensCost (USD)
{m.model || "—"} {m.provider || "—"} {m.calls} {m.failed} {m.total_tokens !== null ? m.total_tokens : "—"} {m.total_cost_usd !== null ? ("$" + m.total_cost_usd) : "—"}
)}
); } /** Bar chart of real token usage by model -- same ai_usage data AiUsageTable * shows as a table, just visualized, so a reader can see at a glance which * model dominates usage without reading rows. */ function AiUsageChart(props) { var t = props.telemetry; if (!t) return null; var usage = t.ai_usage; if (!usage.available) { return

Unavailable — {usage.detail}

; } if (usage.by_model.length === 0) { return

No AI calls recorded in this window.

; } var config = { type: "bar", data: { labels: usage.by_model.map(function (m) { return (m.model || "unknown") + (m.provider ? " (" + m.provider + ")" : ""); }), datasets: [{ label: "Tokens", data: usage.by_model.map(function (m) { return m.total_tokens || 0; }), backgroundColor: PALETTE }], }, options: { responsive: true, maintainAspectRatio: false, plugins: { legend: { display: false } }, scales: { y: { beginAtZero: true } } }, }; return ; } var SERVICE_HEALTH_CHART_COLORS = { READY: CHART_COLORS.good, PARTIALLY_VERIFIED: CHART_COLORS.amber, DEGRADED: CHART_COLORS.amber, NOT_CONFIGURED: CHART_COLORS.bad, AUTHENTICATION_REQUIRED: CHART_COLORS.bad, PERMISSION_DENIED: CHART_COLORS.bad, CONNECTION_FAILED: CHART_COLORS.bad, NOT_INTEGRATED: CHART_COLORS.gray, UNKNOWN: CHART_COLORS.gray, }; /** Donut of real Azure service counts by readiness status -- a reader can * see at a glance "how many of our Azure integrations are actually READY" * without reading the full matrix table below it. A service with UNKNOWN * health is its own distinct slice, never folded into "healthy". */ function ServiceHealthChart(props) { if (props.error) { return

Unavailable — {props.error}

; } if (!props.data) { return

Loading…

; } var counts = {}; (props.data.services || []).forEach(function (s) { counts[s.status] = (counts[s.status] || 0) + 1; }); var statuses = Object.keys(counts); if (statuses.length === 0) { return

No Azure services registered.

; } var config = { type: "doughnut", data: { labels: statuses, datasets: [{ data: statuses.map(function (s) { return counts[s]; }), backgroundColor: statuses.map(function (s) { return SERVICE_HEALTH_CHART_COLORS[s] || CHART_COLORS.gray; }), borderWidth: 0 }], }, options: { responsive: true, maintainAspectRatio: false, plugins: { legend: { position: "bottom" } } }, }; return ; } /** Stacked bar: real succeeded-vs-failed span counts per workflow, sourced * from local_observability.ai_traces.by_workflow -- this data (7 real * instrumented workflows: resume parsing, resume matching, LLM calls, * Azure config checks, Azure ephemeral tests, Azure AI Foundry test * requests, observability self-tests) existed in the API response already * but had no chart at all before this -- a real gap this adds a chart for. */ function WorkflowBreakdownChart(props) { var byWorkflow = props.summary.local_observability.ai_traces.by_workflow || {}; var workflows = Object.keys(byWorkflow); if (workflows.length === 0) { return

No traces recorded locally.

; } var config = { type: "bar", data: { labels: workflows, datasets: [ { label: "Succeeded", data: workflows.map(function (w) { return byWorkflow[w].succeeded; }), backgroundColor: CHART_COLORS.good }, { label: "Failed", data: workflows.map(function (w) { return byWorkflow[w].failed; }), backgroundColor: CHART_COLORS.bad }, ], }, options: { responsive: true, maintainAspectRatio: false, scales: { x: { stacked: true }, y: { stacked: true, beginAtZero: true } } }, }; return ; } /** Bar chart of real average span duration per workflow -- same * by_workflow data as above, a different cut (where is time actually * going, not just pass/fail). */ function WorkflowLatencyChart(props) { var byWorkflow = props.summary.local_observability.ai_traces.by_workflow || {}; var workflows = Object.keys(byWorkflow); if (workflows.length === 0) { return

No traces recorded locally.

; } var config = { type: "bar", data: { labels: workflows, datasets: [{ label: "Avg duration (ms)", data: workflows.map(function (w) { return byWorkflow[w].avg_duration_ms || 0; }), backgroundColor: CHART_COLORS.blue }], }, options: { responsive: true, maintainAspectRatio: false, plugins: { legend: { display: false } }, scales: { y: { beginAtZero: true } } }, }; return ; } function RecentErrorsTable(props) { var t = props.telemetry; if (!t) return null; var errs = t.recent_errors; return (

Error Analysis — Recent Failures (Azure-ingested)

{!errs.available &&

Unavailable — {errs.detail}

} {errs.available && errs.errors.length === 0 &&

No failures found in this window.

} {errs.available && errs.errors.length > 0 && (
{errs.errors.map(function (e, i) { return ( ); })}
TimeSourceWorkflow / LabelStageErrorCorrelation ID
{new Date(e.timestamp).toLocaleString()} {e.source} {e.workflow || e.label || "—"} {e.stage || "—"} {e.error || "—"} {e.correlation_id || "—"}
)}
); } function LocalObservabilityRecap(props) { var lo = props.summary.local_observability; return (

Local Observability (this process's JSONL logs)

{lo.note}

{lo.llm_calls.total_calls} LLM call(s) logged locally, {lo.llm_calls.total_tokens} token(s), ${lo.llm_calls.total_cost_usd} estimated cost. Full detail, per-trace spans, and the Trace Explorer live on the AI Observability Center page.

); } var SERVICE_HEALTH_BADGE = { READY: "badge-good", PARTIALLY_VERIFIED: "badge-neutral", DEGRADED: "badge-neutral", NOT_CONFIGURED: "badge-bad", AUTHENTICATION_REQUIRED: "badge-bad", PERMISSION_DENIED: "badge-bad", CONNECTION_FAILED: "badge-bad", NOT_INTEGRATED: "badge-neutral", UNKNOWN: "badge-neutral", }; /** Compact matrix reusing manage/azure-readiness (the same real data * Azure Readiness' Services tab shows, computed from already-cached * checks -- this does not trigger a new live check). A 403 here (role * lacks azure.config.read) is shown as "no permission", not an error * that blanks the rest of the page. */ function ServiceHealthMatrix(props) { if (props.error) { return

Azure Service Health Matrix

Unavailable — {props.error}

; } if (!props.data) { return

Azure Service Health Matrix

Loading…

; } var services = props.data.services || []; return (

Azure Service Health Matrix

Composed status per Azure integration (configuration, authentication, connectivity) — a successful application health check never implies every dependency below is healthy. See Azure Readiness to run a live check.

{services.map(function (s, i) { return ( ); })}
ServiceStatusReason
{s.service_name} {s.status} {s.reason || "—"}
); } /** Reuses manage/reports/resume-processing -- real document_extraction * spans, not a separately-maintained counter. pending_queue_depth is * honestly null (this repo's batch pipeline has no queue concept), never * invented. */ function DocumentProcessingSection(props) { if (props.error) { return

Document / Resume Processing

Unavailable — {props.error}

; } if (!props.data) { return

Document / Resume Processing

Loading…

; } var d = props.data; return (

Document / Resume Processing

Source: {d.data_source}

{d.total_processing_attempts}
Processing attempts
{d.succeeded}
Succeeded
{d.failed_count}
Failed
{d.failure_rate !== null ? (Math.round(d.failure_rate * 10000) / 100) + "%" : "—"}
Failure rate
{d.pending_queue_depth === null ? "N/A" : d.pending_queue_depth}
Pending queue depth
{d.recent_failures && d.recent_failures.length > 0 && ( {d.recent_failures.map(function (f, i) { return ; })}
Logged atCorrelation IDError
{f.logged_at ? new Date(f.logged_at).toLocaleString() : "—"}{f.correlation_id || "—"}{f.error || "—"}
)}
); } /** Reuses manage/reports/communication-delivery -- real * comms_delivery_log records. Email has NO delivery logging anywhere in * this repo today; that gap is shown explicitly via `note`, not hidden. */ function ExternalIntegrationsSection(props) { if (props.error) { return

External Integrations

Unavailable — {props.error}

; } if (!props.data) { return

External Integrations

Loading…

; } var channels = Object.keys(props.data.by_channel || {}); return (

External Integrations (WhatsApp / Email)

{props.data.note}

{channels.length === 0 &&

No delivery records found.

} {channels.length > 0 && ( {channels.map(function (ch) { var stats = props.data.by_channel[ch]; return ; })}
ChannelSubmittedProvider acceptedFailed
{ch}{stats.submitted}{stats.provider_accepted}{stats.failed}
)}
); }