Skip to content

Commit

Permalink
Merge pull request #5 from rekognize/analytics-updates
Browse files Browse the repository at this point in the history 
Analytics updates
  • Loading branch information
@onurmatik
onurmatik authored Jan 16, 2025
2 parents f284865 + 7a18d7a commit c6e7574
Showing 1 changed file with 157 additions and 25 deletions.
182 changes: 157 additions & 25 deletions oa/templates/analytics.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,7 @@ <h4>${assistant.name || 'Untitled Assistant'}</h4>
const sectionId = `collapse-test-${assistantId}-${userIndex}`;
const progressId = `progress-test-${assistantId}-${userIndex}`;
const tableId = `table-test-${assistantId}-${userIndex}`;
const iconId = `icon-test-${assistantId}-${userIndex}`;

htmlOutput += `
<div class="accordion-item">
Expand All @@ -275,7 +276,10 @@ <h2 class="accordion-header" id="heading-test-${assistantId}-${userIndex}">
aria-controls="${sectionId}"
disabled>
<div class="d-flex w-100 justify-content-between align-items-center">
<span>Test threads by <strong>${userName}</strong> (${groupThreads.length} threads)</span>
<span>
Test threads by <strong>${userName}</strong> (${groupThreads.length} threads)
<i id="${iconId}" class="bi bi-0-circle-fill text-danger d-none"></i>
</span>
<div class="progress w-50 me-3">
<div id="${progressId}" class="progress-bar" role="progressbar"
style="width: 0%;" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100">
Expand Down Expand Up @@ -307,7 +311,7 @@ <h2 class="accordion-header" id="heading-test-${assistantId}-${userIndex}">
`;

// Fetch runs for these threads
fetchThreadRuns(groupThreads, progressId, tableId, sectionId);
fetchThreadRuns(groupThreads, progressId, tableId, sectionId, iconId);
userIndex++;
}

Expand Down Expand Up @@ -348,6 +352,7 @@ <h2 class="accordion-header" id="heading-test-${assistantId}-${userIndex}">
const sectionId = `collapse-shared-${assistantId}-${shareIndex}`;
const progressId = `progress-shared-${assistantId}-${shareIndex}`;
const tableId = `table-shared-${assistantId}-${shareIndex}`;
const iconId = `icon-shared-${assistantId}-${shareIndex}`;

htmlOutput += `
<div class="accordion-item">
Expand All @@ -359,7 +364,10 @@ <h2 class="accordion-header" id="heading-shared-${assistantId}-${shareIndex}">
aria-controls="${sectionId}"
disabled>
<div class="d-flex w-100 justify-content-between align-items-center">
<span>Shared: <strong>${share}</strong> by <strong>${shareUser}</strong> (${groupThreads.length} threads)</span>
<span>
Shared: <strong>${share}</strong> by <strong>${shareUser}</strong> (${groupThreads.length} threads)
<i id="${iconId}" class="bi bi-0-circle-fill text-danger d-none"></i>
</span>
<div class="progress w-50 me-3">
<div id="${progressId}" class="progress-bar" role="progressbar"
style="width: 0%;" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100">
Expand Down Expand Up @@ -391,7 +399,7 @@ <h2 class="accordion-header" id="heading-shared-${assistantId}-${shareIndex}">
`;

// Fetch runs for these threads
fetchThreadRuns(groupThreads, progressId, tableId, sectionId);
fetchThreadRuns(groupThreads, progressId, tableId, sectionId, iconId);
shareIndex++;
}

Expand All @@ -407,9 +415,11 @@ <h2 class="accordion-header" id="heading-shared-${assistantId}-${shareIndex}">
document.getElementById(`stats-${assistantId}`).innerHTML = `<strong>Error loading stats.</strong>`;
}
}
async function fetchThreadRuns(threads, progressId, tableId, sectionId) {

async function fetchThreadRuns(threads, progressId, tableId, sectionId, iconId) {
let completedThreads = 0;
let totalFailedCount = 0;

const totalThreads = threads.length;

for (const thread of threads) {
Expand All @@ -426,49 +436,165 @@ <h2 class="accordion-header" id="heading-shared-${assistantId}-${shareIndex}">
});
const runsData = await runsResponse.json();

// Calculate token usage
const tokensTotal = runsData.runs.reduce((sum, run) => sum + run.usage.total_tokens, 0);
const tokensPrompt = runsData.runs.reduce((sum, run) => sum + run.usage.prompt_tokens, 0);
const tokensCompletion = runsData.runs.reduce((sum, run) => sum + run.usage.completion_tokens, 0);

// Calculate tool usage
const toolsUsed = runsData.runs.reduce((tools, run) => {
run.tools.forEach(tool => {
tools[tool.type] = (tools[tool.type] || 0) + 1;
});
return tools;
}, {});

const toolsUsedString = Object.entries(toolsUsed)
.map(([tool, count]) => `${tool} (${count})`)
.join(', ');

const cancelledRuns = runsData.runs.filter(run => run.cancelled_at !== null).length;
const failedRuns = runsData.runs.filter(run => run.failed_at !== null).length;
// Runs count
const totalRunsCount = runsData.runs.length;

// Identify special runs
const incompleteRuns = runsData.runs.filter(run => run.status === 'incomplete');
const failedRuns = runsData.runs.filter(run => run.failed_at !== null);
const cancelledRuns = runsData.runs.filter(run => run.cancelled_at !== null);

function buildPopoverContent(listItems) {
const html = `<ul class="mb-0">${listItems.join('')}</ul>`;
// Escape double quotes so the string won't break the data-bs-content attribute
return html.replace(/"/g, '&quot;');
}

let runsSummary = `${runsData.runs.length}`;
if (cancelledRuns > 0 || failedRuns > 0) {
const details = [];
if (cancelledRuns > 0) details.push(`${cancelledRuns} cancelled`);
if (failedRuns > 0) details.push(`${failedRuns} failed`);
runsSummary += ` (${details.join(', ')})`;
// Build popover for incomplete runs
const incompleteCount = incompleteRuns.length;
let incompleteSpan = '';
if (incompleteCount > 0) {
// Group by reason and count occurrences
const groupedReasons = incompleteRuns.reduce((acc, run) => {
const reason = run.incomplete_details?.reason ?? 'No details';
acc[reason] = (acc[reason] || 0) + 1; // Increment the count for this reason
return acc;
}, {});

// Prepare the aggregated list for the popover
const incompleteList = Object.entries(groupedReasons).map(([reason, count]) => {
return `<li>${count} ${reason}</li>`;
});

const incompleteHTML = buildPopoverContent(incompleteList);

incompleteSpan = `
<span
class="text-decoration-underline text-danger-emphasis"
data-bs-toggle="popover"
data-bs-trigger="hover"
data-bs-html="true"
data-bs-content="${incompleteHTML}">
${incompleteCount} incomplete
</span>
`;
}

// Build popover for failed runs
const failedCount = failedRuns.length;
let failedSpan = '';
if (failedCount > 0) {
// Increment totalFailedCount by failedCount
totalFailedCount += failedCount;

// Group by error code and count occurrences
const groupedErrors = failedRuns.reduce((acc, run) => {
const code = run.last_error?.code ?? 'unknown_code';
const message = run.last_error?.message ?? 'No message';

if (!acc[code]) {
acc[code] = { count: 0, messages: new Set() };
}

acc[code].count += 1; // Increment the count for this code
acc[code].messages.add(message); // Add the unique message
return acc;
}, {});

// Prepare the aggregated list for the popover
const failedList = Object.entries(groupedErrors).map(([code, data]) => {
const aggregatedMessages = Array.from(data.messages).join('; ');
return `<li>${data.count} ${code}: ${aggregatedMessages}</li>`;
});

const failedHTML = buildPopoverContent(failedList);

failedSpan = `
<span
class="text-decoration-underline text-danger"
data-bs-toggle="popover"
data-bs-trigger="hover"
data-bs-html="true"
data-bs-content="${failedHTML}">
${failedCount} failed
</span>
`;
}

// Build popover for cancelled runs
const cancelledCount = cancelledRuns.length;
let cancelledSpan = '';
if (cancelledCount > 0) {
const cancelledHTML = `<span>${cancelledCount} cancelled by user</span>`;

cancelledSpan = `
<span
class="text-decoration-underline text-warning"
data-bs-toggle="popover"
data-bs-trigger="hover"
data-bs-html="true"
data-bs-content="${cancelledHTML}">
${cancelledCount} cancelled
</span>
`;
}

// Build the runs summary "X (details...)"
let runsSummary = `${totalRunsCount}`;
const subDetails = [];
// Add incomplete, failed, cancelled if they exist
if (incompleteCount > 0) subDetails.push(incompleteSpan);
if (failedCount > 0) subDetails.push(failedSpan);
if (cancelledCount > 0) subDetails.push(cancelledSpan);

if (subDetails.length > 0) {
runsSummary += ` (`;
runsSummary += subDetails.join(', ');
runsSummary += `)`;
}

// Build tokens summary "X (prompt, completion)"
let tokensSummary = `${tokensTotal}`;
if (tokensTotal > 0) {
tokensSummary += ` (${tokensPrompt} prompt, ${tokensCompletion} completion)`;
}

// Add a "User" column — thread.user.username or 'anonymous'
// Determine user name
const userName = (thread.user && thread.user.username) ? thread.user.username : 'anonymous';

const threadRow = `<tr>
<td>...${thread.id.slice(-5)}</td>
<td>${new Date(thread.created_at).toLocaleString()}</td>
<td>${runsSummary}</td>
<td>${tokensSummary}</td>
<td>${toolsUsedString}</td>
<td>${userName}</td>
</tr>`;

// Build row
const threadRow = `
<tr>
<td>...${thread.id.slice(-5)}</td>
<td>${new Date(thread.created_at).toLocaleString()}</td>
<td>${runsSummary}</td>
<td>${tokensSummary}</td>
<td>${toolsUsedString}</td>
<td>${userName}</td>
</tr>
`;

// Append row to the table body
document.querySelector(`#${tableId} tbody`).innerHTML += threadRow;

initializePopovers();

} catch (error) {
console.error(`Error fetching runs for thread ${thread.id}:`, error);
} finally {
Expand All @@ -480,6 +606,12 @@ <h2 class="accordion-header" id="heading-shared-${assistantId}-${shareIndex}">

// Enable expand button once all threads in this group are loaded
if (progress === 100) {
const icon = document.getElementById(iconId);
if (totalFailedCount > 0) {
icon.classList.remove('bi-0-circle-fill');
icon.classList.add(`bi-${totalFailedCount}-circle-fill`);
icon.classList.remove('d-none'); // Reveal the icon
}
document.querySelector(`[data-bs-target="#${sectionId}"]`).removeAttribute('disabled');
}
}
Expand Down

0 comments on commit c6e7574

Please sign in to comment.