Skip to content

Commit 37bc0da

Browse files
dlucdluc
authored
Merge branch 'main' into feature/pages-in-memoryrecords
2 parents c220010 + dd89a8e commit 37bc0da

File tree

29 files changed

+388
-81
lines changed

29 files changed

+388
-81
lines changed

‎.editorconfig

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -658,6 +658,25 @@ dotnet_naming_rule.private_fields_underscored.symbols = private_fields
658658
dotnet_naming_rule.private_fields_underscored.style = underscored
659659
dotnet_naming_rule.private_fields_underscored.severity = error
660660

661+
#####################################
662+
# Resharper #
663+
#####################################
664+
665+
# disable ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract
666+
resharper_Condition_Is_Always_True_Or_False_According_To_Nullable_API_Contract_highlighting = none
667+
668+
# disable RedundantTypeArgumentsOfMethod
669+
resharper_Redundant_Type_Arguments_Of_Method_highlighting = none
670+
671+
# disable NullCoalescingConditionIsAlwaysNotNullAccordingToAPIContract
672+
resharper_Null_Coalescing_Condition_Is_Always_Not_Null_According_To_API_Contract_highlighting = none
673+
674+
# disable PartialTypeWithSinglePart
675+
resharper_Partial_Type_With_Single_Part_highlighting = none
676+
677+
# disable RedundantDefaultMemberInitializer
678+
resharper_Redundant_Default_Member_Initializer_highlighting = none
679+
661680
#####################################################################################################
662681
# Naming Conventions by folder #
663682
# See also https://www.jetbrains.com/help/resharper/Coding_Assistance__Naming_Style.html#configure #

‎KernelMemory.sln.DotSettings

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,14 +88,19 @@
8888
<s:Int64 x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/WRAP_LIMIT/@EntryValue">512</s:Int64>
8989
<s:Boolean x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/WRAP_LINES/@EntryValue">True</s:Boolean>
9090
<s:String x:Key="/Default/CodeStyle/FileHeader/FileHeaderText/@EntryValue">Copyright (c) Microsoft. All rights reserved.</s:String>
91+
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=AAC/@EntryIndexedValue">AAC</s:String>
9192
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=ACS/@EntryIndexedValue">ACS</s:String>
9293
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=AI/@EntryIndexedValue">AI</s:String>
9394
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=AIGPT/@EntryIndexedValue">AIGPT</s:String>
9495
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=AMQP/@EntryIndexedValue">AMQP</s:String>
9596
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=API/@EntryIndexedValue">API</s:String>
97+
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=AWS/@EntryIndexedValue">AWS</s:String>
98+
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=AWSS/@EntryIndexedValue">AWSS</s:String>
99+
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=AWSS3/@EntryIndexedValue">AWSS3</s:String>
96100
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=BOM/@EntryIndexedValue">BOM</s:String>
97101
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=CL/@EntryIndexedValue">CL</s:String>
98102
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=CORS/@EntryIndexedValue">CORS</s:String>
103+
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=CSV/@EntryIndexedValue">CSV</s:String>
99104
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=DB/@EntryIndexedValue">DB</s:String>
100105
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=DI/@EntryIndexedValue">DI</s:String>
101106
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=GPT/@EntryIndexedValue">GPT</s:String>
@@ -106,28 +111,38 @@
106111
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=IO/@EntryIndexedValue">IO</s:String>
107112
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=IOS/@EntryIndexedValue">IOS</s:String>
108113
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=JSON/@EntryIndexedValue">JSON</s:String>
114+
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=JSONLD/@EntryIndexedValue">JSONLD</s:String>
109115
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=JWT/@EntryIndexedValue">JWT</s:String>
110116
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=KM/@EntryIndexedValue">KM</s:String>
117+
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=LLM/@EntryIndexedValue">LLM</s:String>
118+
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=MP/@EntryIndexedValue">MP</s:String>
119+
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=MPEG/@EntryIndexedValue">MPEG</s:String>
111120
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=MQ/@EntryIndexedValue">MQ</s:String>
112121
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=MQTT/@EntryIndexedValue">MQTT</s:String>
113122
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=MS/@EntryIndexedValue">MS</s:String>
114123
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=MSAL/@EntryIndexedValue">MSAL</s:String>
115124
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=NLF/@EntryIndexedValue">NLF</s:String>
116125
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=OCR/@EntryIndexedValue">OCR</s:String>
126+
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=OGG/@EntryIndexedValue">OGG</s:String>
117127
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=OID/@EntryIndexedValue">OID</s:String>
118128
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=OK/@EntryIndexedValue">OK</s:String>
119129
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=OS/@EntryIndexedValue">OS</s:String>
120130
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=PR/@EntryIndexedValue">PR</s:String>
121131
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=QA/@EntryIndexedValue">QA</s:String>
132+
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=QL/@EntryIndexedValue">QL</s:String>
133+
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=RTF/@EntryIndexedValue">RTF</s:String>
122134
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=SHA/@EntryIndexedValue">SHA</s:String>
123135
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=SK/@EntryIndexedValue">SK</s:String>
124136
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=SKHTTP/@EntryIndexedValue">SKHTTP</s:String>
125137
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=SSE/@EntryIndexedValue">SSE</s:String>
126138
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=SSL/@EntryIndexedValue">SSL</s:String>
139+
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=SVG/@EntryIndexedValue">SVG</s:String>
127140
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=TTL/@EntryIndexedValue">TTL</s:String>
128141
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=UI/@EntryIndexedValue">UI</s:String>
129142
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=UID/@EntryIndexedValue">UID</s:String>
130143
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=URL/@EntryIndexedValue">URL</s:String>
144+
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=WEBM/@EntryIndexedValue">WEBM</s:String>
145+
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=XHTML/@EntryIndexedValue">XHTML</s:String>
131146
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=XML/@EntryIndexedValue">XML</s:String>
132147
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=YAML/@EntryIndexedValue">YAML</s:String>
133148
<s:Boolean x:Key="/Default/CodeStyle/Naming/CSharpNaming/ApplyAutoDetectedRules/@EntryValue">False</s:Boolean>

‎examples/001-dotnet-WebClient/Program.cs

Lines changed: 35 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -253,31 +253,58 @@ private static async Task AskSimpleQuestionStreamingTheAnswer()
253253
{
254254
var question = "What's E = m*c^2?";
255255
Console.WriteLine($"Question: {question}");
256-
Console.WriteLine($"Expected result: formula explanation using the information loaded");
256+
Console.WriteLine("Expected result: formula explanation using the information loaded");
257257

258258
Console.Write("\nAnswer: ");
259+
var tokenUsage = new List<TokenUsage>();
259260
var answerStream = s_memory.AskStreamingAsync(question, options: new SearchOptions { Stream = true });
260261

261262
await foreach (var answer in answerStream)
262263
{
263264
// Print token received by LLM
264265
Console.Write(answer.Result);
266+
267+
// Collect token usage
268+
if (answer.TokenUsage?.Count > 0)
269+
{
270+
tokenUsage = tokenUsage.Union(answer.TokenUsage).ToList();
271+
}
272+
265273
// Slow down the stream for demo purpose
266274
await Task.Delay(25);
267275
}
268276

277+
Console.WriteLine("\n\nToken usage report:");
278+
foreach (var report in tokenUsage)
279+
{
280+
Console.WriteLine($"{report.ServiceType}: {report.ModelName} [{report.ModelType}]");
281+
Console.WriteLine($"- Input : {report.TokenizerTokensIn} tokens (measured by KM tokenizer)");
282+
Console.WriteLine($"- Input : {report.ServiceTokensIn} tokens (measured by remote service)");
283+
Console.WriteLine($"- Output: {report.ServiceTokensOut} tokens (measured by remote service)");
284+
Console.WriteLine($"- Output: {report.TokenizerTokensOut} tokens (measured by KM tokenizer)");
285+
Console.WriteLine();
286+
}
287+
269288
Console.WriteLine("\n\n====================================\n");
270289

271290
/* OUTPUT
272291
273292
Question: What's E = m*c^2?
274-
275-
Answer: E = m*c^2 is the formula representing the principle of mass-energy equivalence, which was introduced by Albert Einstein. In this equation,
276-
E stands for energy, m represents mass, and c is the speed of light in a vacuum, which is approximately 299,792,458 meters per second (m/s).
277-
The equation states that the energy (E) of a system in its rest frame is equal to its mass (m) multiplied by the square of the speed of light (c^2).
278-
This implies that mass and energy are interchangeable; a small amount of mass can be converted into a large amount of energy and vice versa,
279-
due to the speed of light being a very large number when squared. This concept is a fundamental principle in physics and has important implications
280-
in various fields, including nuclear physics and cosmology.
293+
Expected result: formula explanation using the information loaded
294+
295+
Answer: E = m*c^2 is a formula derived by the physicist Albert Einstein, which describes the principle of
296+
mass–energy equivalence. In this equation, E represents energy, m represents mass, and c represents the
297+
speed of light in a vacuum (approximately 3 x 10^8 meters per second). The formula indicates that mass and
298+
energy are interchangeable; they are different forms of the same thing and can be converted into each other.
299+
This principle is fundamental in physics and has significant implications in various fields, including nuclear
300+
physics and cosmology.
301+
302+
Token usage report:
303+
Azure OpenAI: gpt-4o [TextGeneration]
304+
- Input : 15657 tokens (measured by KM tokenizer)
305+
- Input : 15664 tokens (measured by remote service)
306+
- Output: 110 tokens (measured by remote service)
307+
- Output: 110 tokens (measured by KM tokenizer)
281308
282309
*/
283310
}

‎examples/002-dotnet-Serverless/Program.cs

Lines changed: 34 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -311,31 +311,57 @@ private static async Task AskSimpleQuestionStreamingTheAnswer()
311311
{
312312
var question = "What's E = m*c^2?";
313313
Console.WriteLine($"Question: {question}");
314-
Console.WriteLine($"Expected result: formula explanation using the information loaded");
314+
Console.WriteLine("Expected result: formula explanation using the information loaded");
315315

316316
Console.Write("\nAnswer: ");
317+
var tokenUsage = new List<TokenUsage>();
317318
var answerStream = s_memory.AskStreamingAsync(question, options: new SearchOptions { Stream = true });
318319

319320
await foreach (var answer in answerStream)
320321
{
321322
// Print token received by LLM
322323
Console.Write(answer.Result);
324+
325+
// Collect token usage
326+
if (answer.TokenUsage?.Count > 0)
327+
{
328+
tokenUsage = tokenUsage.Union(answer.TokenUsage).ToList();
329+
}
330+
323331
// Slow down the stream for demo purpose
324332
await Task.Delay(25);
325333
}
326334

335+
Console.WriteLine("\n\nToken usage report:");
336+
foreach (var report in tokenUsage)
337+
{
338+
Console.WriteLine($"{report.ServiceType}: {report.ModelName} [{report.ModelType}]");
339+
Console.WriteLine($"- Input : {report.TokenizerTokensIn} tokens (measured by KM tokenizer)");
340+
Console.WriteLine($"- Input : {report.ServiceTokensIn} tokens (measured by remote service)");
341+
Console.WriteLine($"- Output: {report.ServiceTokensOut} tokens (measured by remote service)");
342+
Console.WriteLine($"- Output: {report.TokenizerTokensOut} tokens (measured by KM tokenizer)");
343+
Console.WriteLine();
344+
}
345+
327346
Console.WriteLine("\n\n====================================\n");
328347

329348
/* OUTPUT
330349
331350
Question: What's E = m*c^2?
332-
333-
Answer: E = m*c^2 is the formula representing the principle of mass-energy equivalence, which was introduced by Albert Einstein. In this equation,
334-
E stands for energy, m represents mass, and c is the speed of light in a vacuum, which is approximately 299,792,458 meters per second (m/s).
335-
The equation states that the energy (E) of a system in its rest frame is equal to its mass (m) multiplied by the square of the speed of light (c^2).
336-
This implies that mass and energy are interchangeable; a small amount of mass can be converted into a large amount of energy and vice versa,
337-
due to the speed of light being a very large number when squared. This concept is a fundamental principle in physics and has important implications
338-
in various fields, including nuclear physics and cosmology.
351+
Expected result: formula explanation using the information loaded
352+
353+
Answer: E = m*c^2 is a formula derived by physicist Albert Einstein, which expresses the principle of
354+
mass–energy equivalence. In this equation, E represents energy, m represents mass, and c represents the
355+
speed of light in a vacuum (approximately 3 x 10^8 meters per second). The formula indicates that mass and
356+
energy are interchangeable; a small amount of mass can be converted into a large amount of energy, and vice
357+
versa, differing only by a multiplicative constant (c^2).
358+
359+
Token usage report:
360+
Azure OpenAI: gpt-4o [TextGeneration]
361+
- Input : 24349 tokens (measured by KM tokenizer)
362+
- Input : 24356 tokens (measured by remote service)
363+
- Output: 103 tokens (measured by remote service)
364+
- Output: 103 tokens (measured by KM tokenizer)
339365
340366
*/
341367
}

‎examples/104-dotnet-custom-LLM/Program.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ public IReadOnlyList<string> GetTokens(string text)
6868
}
6969

7070
/// <inheritdoc />
71-
public async IAsyncEnumerable<string> GenerateTextAsync(
71+
public async IAsyncEnumerable<GeneratedTextContent> GenerateTextAsync(
7272
string prompt,
7373
TextGenerationOptions options,
7474
[EnumeratorCancellation] CancellationToken cancellationToken = default)

‎extensions/Anthropic/AnthropicTextGeneration.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ public IReadOnlyList<string> GetTokens(string text)
9797
}
9898

9999
/// <inheritdoc />
100-
public async IAsyncEnumerable<string> GenerateTextAsync(
100+
public async IAsyncEnumerable<GeneratedTextContent> GenerateTextAsync(
101101
string prompt,
102102
TextGenerationOptions options,
103103
[EnumeratorCancellation] CancellationToken cancellationToken = default)

‎extensions/AzureOpenAI/AzureOpenAI/AzureOpenAITextGenerator.cs

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
// Copyright (c) Microsoft. All rights reserved.
22

3+
using System;
34
using System.Collections.Generic;
45
using System.Diagnostics.CodeAnalysis;
56
using System.Net.Http;
@@ -12,6 +13,7 @@
1213
using Microsoft.KernelMemory.Diagnostics;
1314
using Microsoft.SemanticKernel;
1415
using Microsoft.SemanticKernel.Connectors.AzureOpenAI;
16+
using OpenAI.Chat;
1517

1618
namespace Microsoft.KernelMemory.AI.AzureOpenAI;
1719

@@ -28,6 +30,8 @@ public sealed class AzureOpenAITextGenerator : ITextGenerator
2830
private readonly ITextTokenizer _textTokenizer;
2931
private readonly ILogger<AzureOpenAITextGenerator> _log;
3032

33+
private readonly string _deployment;
34+
3135
/// <inheritdoc/>
3236
public int MaxTokenTotal { get; }
3337

@@ -87,6 +91,7 @@ public AzureOpenAITextGenerator(
8791
{
8892
this._client = skClient;
8993
this._log = (loggerFactory ?? DefaultLogger.Factory).CreateLogger<AzureOpenAITextGenerator>();
94+
this._deployment = config.Deployment;
9095
this.MaxTokenTotal = config.MaxTokenTotal;
9196

9297
textTokenizer ??= TokenizerFactory.GetTokenizerForEncoding(config.Tokenizer);
@@ -114,7 +119,7 @@ public IReadOnlyList<string> GetTokens(string text)
114119
}
115120

116121
/// <inheritdoc/>
117-
public async IAsyncEnumerable<string> GenerateTextAsync(
122+
public async IAsyncEnumerable<GeneratedTextContent> GenerateTextAsync(
118123
string prompt,
119124
TextGenerationOptions options,
120125
[EnumeratorCancellation] CancellationToken cancellationToken = default)
@@ -153,9 +158,33 @@ public async IAsyncEnumerable<string> GenerateTextAsync(
153158

154159
await foreach (StreamingTextContent x in result.WithCancellation(cancellationToken))
155160
{
156-
if (x.Text == null) { continue; }
157-
158-
yield return x.Text;
161+
TokenUsage? tokenUsage = null;
162+
163+
// The last message includes tokens usage metadata.
164+
// https://platform.openai.com/docs/api-reference/chat/create#chat-create-stream_options
165+
if (x.Metadata?["Usage"] is ChatTokenUsage usage)
166+
{
167+
this._log.LogTrace("Usage report: input tokens: {InputTokenCount}, output tokens: {OutputTokenCount}, output reasoning tokens: {ReasoningTokenCount}",
168+
usage.InputTokenCount, usage.OutputTokenCount, usage.OutputTokenDetails?.ReasoningTokenCount ?? 0);
169+
170+
tokenUsage = new TokenUsage
171+
{
172+
Timestamp = (DateTimeOffset?)x.Metadata["CreatedAt"] ?? DateTimeOffset.UtcNow,
173+
ServiceType = "Azure OpenAI",
174+
ModelType = Constants.ModelType.TextGeneration,
175+
ModelName = this._deployment,
176+
ServiceTokensIn = usage.InputTokenCount,
177+
ServiceTokensOut = usage.OutputTokenCount,
178+
ServiceReasoningTokens = usage.OutputTokenDetails?.ReasoningTokenCount
179+
};
180+
}
181+
182+
// NOTE: as stated at https://platform.openai.com/docs/api-reference/chat/streaming#chat/streaming-choices,
183+
// the Choice can also be empty for the last chunk if we set stream_options: { "include_usage": true} to get token counts, so it is possible that
184+
// x.Text is null, but tokenUsage is not (token usage statistics for the entire request are included in the last chunk).
185+
if (x.Text is null && tokenUsage is null) { continue; }
186+
187+
yield return new(x.Text ?? string.Empty, tokenUsage);
159188
}
160189
}
161190
}

‎extensions/LlamaSharp/LlamaSharp.FunctionalTests/LlamaSharpTextGeneratorTest.cs

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ public void ItCountsTokens()
4040

4141
// Assert
4242
Console.WriteLine("Phi3 token count: " + tokenCount);
43-
Console.WriteLine("GPT4 token count: " + (new CL100KTokenizer()).CountTokens(text));
43+
Console.WriteLine("GPT4 token count: " + new CL100KTokenizer().CountTokens(text));
4444
Console.WriteLine($"Time: {this._timer.ElapsedMilliseconds / 1000} secs");
4545

4646
// Expected result with Phi-3-mini-4k-instruct-q4.gguf, without BoS (https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf)
@@ -90,9 +90,8 @@ public async Task ItGeneratesText()
9090
this._timer.Restart();
9191
var tokens = this._target.GenerateTextAsync(prompt, options);
9292
var result = new StringBuilder();
93-
await foreach (string token in tokens)
93+
await foreach (var token in tokens)
9494
{
95-
// Console.WriteLine(token);
9695
result.Append(token);
9796
}
9897

0 commit comments

Comments
 (0)