From c7f2bfc5e95c0d03427bfaa56257101bc55e396c Mon Sep 17 00:00:00 2001 From: Kemal Soylu Date: Wed, 15 Jan 2025 17:14:43 +0300 Subject: [PATCH 1/4] initial popovers for runs in threads table --- oa/templates/analytics.html | 121 ++++++++++++++++++++++++++++++------ 1 file changed, 103 insertions(+), 18 deletions(-) diff --git a/oa/templates/analytics.html b/oa/templates/analytics.html index 9c5da2a..4267d8b 100644 --- a/oa/templates/analytics.html +++ b/oa/templates/analytics.html @@ -426,31 +426,109 @@

}); const runsData = await runsResponse.json(); + // Calculate token usage const tokensTotal = runsData.runs.reduce((sum, run) => sum + run.usage.total_tokens, 0); const tokensPrompt = runsData.runs.reduce((sum, run) => sum + run.usage.prompt_tokens, 0); const tokensCompletion = runsData.runs.reduce((sum, run) => sum + run.usage.completion_tokens, 0); + + // Calculate tool usage const toolsUsed = runsData.runs.reduce((tools, run) => { run.tools.forEach(tool => { tools[tool.type] = (tools[tool.type] || 0) + 1; }); return tools; }, {}); - const toolsUsedString = Object.entries(toolsUsed) .map(([tool, count]) => `${tool} (${count})`) .join(', '); - const cancelledRuns = runsData.runs.filter(run => run.cancelled_at !== null).length; - const failedRuns = runsData.runs.filter(run => run.failed_at !== null).length; + // Runs count + const totalRunsCount = runsData.runs.length; + + // Identify special runs + const incompleteRuns = runsData.runs.filter(run => run.status === 'incomplete'); + const failedRuns = runsData.runs.filter(run => run.failed_at !== null); + const cancelledRuns = runsData.runs.filter(run => run.cancelled_at !== null); + + // Build popovers for incomplete runs + const incompleteCount = incompleteRuns.length; + let incompleteSpan = ''; + if (incompleteCount > 0) { + // Build the content for the popover + const incompleteDetails = incompleteRuns + .map(run => `- ${run.incomplete_details?.reason}`) + .join('
'); + incompleteSpan = ` + + ${incompleteCount} incomplete + + `; + } - let runsSummary = `${runsData.runs.length}`; - if (cancelledRuns > 0 || failedRuns > 0) { - const details = []; - if (cancelledRuns > 0) details.push(`${cancelledRuns} cancelled`); - if (failedRuns > 0) details.push(`${failedRuns} failed`); - runsSummary += ` (${details.join(', ')})`; + // Build popovers for failed runs + const failedCount = failedRuns.length; + let failedSpan = ''; + if (failedCount > 0) { + // Build the content for the popover + const failedDetails = failedRuns + .map(run => { + const code = run.last_error?.code ?? 'unknown_code'; + const message = run.last_error?.message ?? 'No message'; + return `- ${code}: ${message}`; + }) + .join('
'); + failedSpan = ` + + ${failedCount} failed + + `; } + // Build popovers for cancelled runs + const cancelledCount = cancelledRuns.length; + let cancelledSpan = ''; + if (cancelledCount > 0) { + // Build the content for the popover + const cancelledDetails = cancelledRuns + .map(run => `- Cancelled by user - ${formatUnixTimestamp(run.cancelled_at)}`) + .join('
'); + cancelledSpan = ` + + ${cancelledCount} cancelled + + `; + } + + // Build the runs summary "X (details...)" + let runsSummary = `${totalRunsCount}`; + const subDetails = []; + // Add incomplete, failed, cancelled if they exist + if (incompleteCount > 0) subDetails.push(incompleteSpan); + if (failedCount > 0) subDetails.push(failedSpan); + if (cancelledCount > 0) subDetails.push(cancelledSpan); + + if (subDetails.length > 0) { + runsSummary += ` (`; + runsSummary += subDetails.join(', '); + runsSummary += `)`; + } + + // Build tokens summary "X (prompt, completion)" let tokensSummary = `${tokensTotal}`; if (tokensTotal > 0) { tokensSummary += ` (${tokensPrompt} prompt, ${tokensCompletion} completion)`; @@ -459,16 +537,23 @@

// Add a "User" column — thread.user.username or 'anonymous' const userName = (thread.user && thread.user.username) ? thread.user.username : 'anonymous'; - const threadRow = ` - ...${thread.id.slice(-5)} - ${new Date(thread.created_at).toLocaleString()} - ${runsSummary} - ${tokensSummary} - ${toolsUsedString} - ${userName} - `; - + // Build the row + const threadRow = ` + + ...${thread.id.slice(-5)} + ${new Date(thread.created_at).toLocaleString()} + ${runsSummary} + ${tokensSummary} + ${toolsUsedString} + ${userName} + + `; + + // Append row to the table body document.querySelector(`#${tableId} tbody`).innerHTML += threadRow; + + initializePopovers(); + } catch (error) { console.error(`Error fetching runs for thread ${thread.id}:`, error); } finally { From f963f96dd18658d571bb81254fb32b5d3d1a6dec Mon Sep 17 00:00:00 2001 From: Kemal Soylu Date: Thu, 16 Jan 2025 14:04:24 +0300 Subject: [PATCH 2/4] updated popovers --- oa/templates/analytics.html | 66 +++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/oa/templates/analytics.html b/oa/templates/analytics.html index 4267d8b..4029fce 100644 --- a/oa/templates/analytics.html +++ b/oa/templates/analytics.html @@ -407,7 +407,7 @@

document.getElementById(`stats-${assistantId}`).innerHTML = `Error loading stats.`; } } - + async function fetchThreadRuns(threads, progressId, tableId, sectionId) { let completedThreads = 0; const totalThreads = threads.length; @@ -450,66 +450,76 @@

const failedRuns = runsData.runs.filter(run => run.failed_at !== null); const cancelledRuns = runsData.runs.filter(run => run.cancelled_at !== null); - // Build popovers for incomplete runs + function buildPopoverContent(listItems) { + const html = `
    ${listItems.join('')}
`; + // Escape double quotes so the string won't break the data-bs-content attribute + return html.replace(/"/g, '"'); + } + + // Build popover for incomplete runs const incompleteCount = incompleteRuns.length; let incompleteSpan = ''; if (incompleteCount > 0) { - // Build the content for the popover - const incompleteDetails = incompleteRuns - .map(run => `- ${run.incomplete_details?.reason}`) - .join('
'); + const incompleteList = incompleteRuns.map(run => { + const reason = run.incomplete_details?.reason ?? 'No details'; + return `
  • ${reason}
    • `; + }); + const incompleteHTML = buildPopoverContent(incompleteList); + incompleteSpan = ` - ${incompleteCount} incomplete + data-bs-content="${incompleteHTML}"> + ${incompleteCount} incomplete `; } - // Build popovers for failed runs + // Build popover for failed runs const failedCount = failedRuns.length; let failedSpan = ''; if (failedCount > 0) { - // Build the content for the popover - const failedDetails = failedRuns - .map(run => { - const code = run.last_error?.code ?? 'unknown_code'; - const message = run.last_error?.message ?? 'No message'; - return `- ${code}: ${message}`; - }) - .join('
    '); + const failedList = failedRuns.map(run => { + const code = run.last_error?.code ?? 'unknown_code'; + const message = run.last_error?.message ?? 'No message'; + const failedDate = `${formatUnixTimestamp(run.failed_at)}`; + return `
  • ${code}: ${message} - ${failedDate}
    • `; + }); + const failedHTML = buildPopoverContent(failedList); + failedSpan = ` - ${failedCount} failed + data-bs-content="${failedHTML}"> + ${failedCount} failed `; } - // Build popovers for cancelled runs + // Build popover for cancelled runs const cancelledCount = cancelledRuns.length; let cancelledSpan = ''; if (cancelledCount > 0) { - // Build the content for the popover - const cancelledDetails = cancelledRuns - .map(run => `- Cancelled by user - ${formatUnixTimestamp(run.cancelled_at)}`) - .join('
    '); + const cancelledList = cancelledRuns.map(run => { + const cancelledDate = `${formatUnixTimestamp(run.cancelled_at)}`; + return `
  • Cancelled - ${cancelledDate}
    • `; + }); + const cancelledHTML = buildPopoverContent(cancelledList); + cancelledSpan = ` - ${cancelledCount} cancelled + data-bs-content="${cancelledHTML}"> + ${cancelledCount} cancelled `; } @@ -534,10 +544,10 @@

    tokensSummary += ` (${tokensPrompt} prompt, ${tokensCompletion} completion)`; } - // Add a "User" column — thread.user.username or 'anonymous' + // Determine user name const userName = (thread.user && thread.user.username) ? thread.user.username : 'anonymous'; - // Build the row + // Build row const threadRow = ` ...${thread.id.slice(-5)} From 9ad81ea7c207a2dd1c7a43dc39253ce2e6f83097 Mon Sep 17 00:00:00 2001 From: Kemal Soylu Date: Thu, 16 Jan 2025 15:53:32 +0300 Subject: [PATCH 3/4] aggregated info in popovers --- oa/templates/analytics.html | 44 ++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/oa/templates/analytics.html b/oa/templates/analytics.html index 4029fce..92e821a 100644 --- a/oa/templates/analytics.html +++ b/oa/templates/analytics.html @@ -460,12 +460,20 @@

    const incompleteCount = incompleteRuns.length; let incompleteSpan = ''; if (incompleteCount > 0) { - const incompleteList = incompleteRuns.map(run => { + // Group by reason and count occurrences + const groupedReasons = incompleteRuns.reduce((acc, run) => { const reason = run.incomplete_details?.reason ?? 'No details'; - return `
  • ${reason}
    • `; + acc[reason] = (acc[reason] || 0) + 1; // Increment the count for this reason + return acc; + }, {}); + + // Prepare the aggregated list for the popover + const incompleteList = Object.entries(groupedReasons).map(([reason, count]) => { + return `
  • ${count} ${reason}
    • `; }); + const incompleteHTML = buildPopoverContent(incompleteList); - + incompleteSpan = ` const failedCount = failedRuns.length; let failedSpan = ''; if (failedCount > 0) { - const failedList = failedRuns.map(run => { + // Group by error code and count occurrences + const groupedErrors = failedRuns.reduce((acc, run) => { const code = run.last_error?.code ?? 'unknown_code'; const message = run.last_error?.message ?? 'No message'; - const failedDate = `${formatUnixTimestamp(run.failed_at)}`; - return `
  • ${code}: ${message} - ${failedDate}
    • `; + + if (!acc[code]) { + acc[code] = { count: 0, messages: new Set() }; + } + + acc[code].count += 1; // Increment the count for this code + acc[code].messages.add(message); // Add the unique message + return acc; + }, {}); + + // Prepare the aggregated list for the popover + const failedList = Object.entries(groupedErrors).map(([code, data]) => { + const aggregatedMessages = Array.from(data.messages).join('; '); + return `
  • ${data.count} ${code}: ${aggregatedMessages}
    • `; }); + const failedHTML = buildPopoverContent(failedList); - + failedSpan = ` const cancelledCount = cancelledRuns.length; let cancelledSpan = ''; if (cancelledCount > 0) { - const cancelledList = cancelledRuns.map(run => { - const cancelledDate = `${formatUnixTimestamp(run.cancelled_at)}`; - return `
  • Cancelled - ${cancelledDate}
    • `; - }); - const cancelledHTML = buildPopoverContent(cancelledList); - + const cancelledHTML = `${cancelledCount} cancelled by user`; + cancelledSpan = ` Date: Thu, 16 Jan 2025 18:38:48 +0300 Subject: [PATCH 4/4] added failed icon to accordion-header --- oa/templates/analytics.html | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/oa/templates/analytics.html b/oa/templates/analytics.html index 92e821a..0a0a64e 100644 --- a/oa/templates/analytics.html +++ b/oa/templates/analytics.html @@ -264,6 +264,7 @@

    ${assistant.name || 'Untitled Assistant'}

    const sectionId = `collapse-test-${assistantId}-${userIndex}`; const progressId = `progress-test-${assistantId}-${userIndex}`; const tableId = `table-test-${assistantId}-${userIndex}`; + const iconId = `icon-test-${assistantId}-${userIndex}`; htmlOutput += `
    @@ -275,7 +276,10 @@

    aria-controls="${sectionId}" disabled>
    - Test threads by ${userName} (${groupThreads.length} threads) + + Test threads by ${userName} (${groupThreads.length} threads) + +
    @@ -307,7 +311,7 @@

    `; // Fetch runs for these threads - fetchThreadRuns(groupThreads, progressId, tableId, sectionId); + fetchThreadRuns(groupThreads, progressId, tableId, sectionId, iconId); userIndex++; } @@ -348,6 +352,7 @@

    const sectionId = `collapse-shared-${assistantId}-${shareIndex}`; const progressId = `progress-shared-${assistantId}-${shareIndex}`; const tableId = `table-shared-${assistantId}-${shareIndex}`; + const iconId = `icon-shared-${assistantId}-${shareIndex}`; htmlOutput += `
    @@ -359,7 +364,10 @@

    aria-controls="${sectionId}" disabled>
    - Shared: ${share} by ${shareUser} (${groupThreads.length} threads) + + Shared: ${share} by ${shareUser} (${groupThreads.length} threads) + +
    @@ -391,7 +399,7 @@

    `; // Fetch runs for these threads - fetchThreadRuns(groupThreads, progressId, tableId, sectionId); + fetchThreadRuns(groupThreads, progressId, tableId, sectionId, iconId); shareIndex++; } @@ -408,8 +416,10 @@

    } } - async function fetchThreadRuns(threads, progressId, tableId, sectionId) { + async function fetchThreadRuns(threads, progressId, tableId, sectionId, iconId) { let completedThreads = 0; + let totalFailedCount = 0; + const totalThreads = threads.length; for (const thread of threads) { @@ -490,6 +500,9 @@

    const failedCount = failedRuns.length; let failedSpan = ''; if (failedCount > 0) { + // Increment totalFailedCount by failedCount + totalFailedCount += failedCount; + // Group by error code and count occurrences const groupedErrors = failedRuns.reduce((acc, run) => { const code = run.last_error?.code ?? 'unknown_code'; @@ -593,6 +606,12 @@

    // Enable expand button once all threads in this group are loaded if (progress === 100) { + const icon = document.getElementById(iconId); + if (totalFailedCount > 0) { + icon.classList.remove('bi-0-circle-fill'); + icon.classList.add(`bi-${totalFailedCount}-circle-fill`); + icon.classList.remove('d-none'); // Reveal the icon + } document.querySelector(`[data-bs-target="#${sectionId}"]`).removeAttribute('disabled'); } }