diff --git a/FEATURE_MATRIX.md b/FEATURE_MATRIX.md index d657eb635a19..441a88d739ab 100644 --- a/FEATURE_MATRIX.md +++ b/FEATURE_MATRIX.md @@ -2,7 +2,7 @@ ## AI Services | | C# | Python | Notes | -|---|---|---|---| +|---|---|---|---| | TextGeneration | ✅ | ✅ | Example: Text-Davinci-003 | | TextEmbeddings | ✅ | ✅ | Example: Text-Embeddings-Ada-002 | | ChatCompletion | ✅ | ❌ | Example: GPT4, Chat-GPT | @@ -10,35 +10,40 @@ ## AI Service Endpoints | | C# | Python | Notes | -|---|---|---|---| +|---|---|---|---| | OpenAI | ✅ | ✅ | | | AzureOpenAI | ✅ | ✅ | | -| Hugging Face | ✅ | ❌ | Coming soon to Python - both native and web endpoint support | +| Hugging Face | ✅ | ❌ | Coming soon to Python - both native and web endpoint support | | Custom | ✅ | ❌ | Requires the user to define the service schema in their application | ## Tokenizers | | C# | Python | Notes | -|---|---|---|---| +|---|---|---|---| | GPT2 | ✅ | 🔄 | Can be manually added to Python via `pip install transformers` | -| GPT3 | ✅ | ❌ | | -| tiktoken | 🔄 | ❌ | Coming soon to Python and C#. Can be manually added to Python via `pip install tiktoken` | +| GPT3 | ✅ | ❌ | | +| tiktoken | 🔄 | ❌ | Coming soon to Python and C#. Can be manually added to Python via `pip install tiktoken` | ## Core Skills | | C# | Python | Notes | -|---|---|---|---| +|---|---|---|---| | TextMemorySkill | ✅ | ✅ | | -| PlannerSkill | ✅ | 🔄 | | -| ConversationSummarySkill | ✅ | ❌ | | +| ConversationSummarySkill | ✅ | ❌ | | | FileIOSkill | ✅ | ✅ | | | HttpSkill | ✅ | ❌ | | | MathSkill | ✅ | ❌ | | | TextSkill | ✅ | ✅ | | | TimeSkill | ✅ | ✅ | | -## Connectors and Skill Libraries +## Planning | | C# | Python | Notes | -|---|---|---|---| -| Qdrant (Memory) | ✅ | ❌ | Vector optimized | +|---|---|---|---| +| Plan | ✅ | ❌ | | +| SequentialPlanner | ✅ | ❌ | | + +## Connectors and Skill Libraries +| | C# | Python | Notes | +|---|---|---|---| +| Qdrant (Memory) | ✅ | ❌ | Vector optimized | | ChromaDb (Memory) | ❌ | 🔄 | | | Milvus (Memory) | ❌ | ❌ | Vector optimized | | Pinecone (Memory) | ❌ | ❌ | Vector optimized | diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index eea4f25e77e3..1dbd651e0ab0 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -9,26 +9,26 @@ - + - + - + - + diff --git a/dotnet/src/SemanticKernel.Abstractions/SemanticKernel.Abstractions.csproj b/dotnet/src/SemanticKernel.Abstractions/SemanticKernel.Abstractions.csproj index cb6ad42ddf88..8361049cdd46 100644 --- a/dotnet/src/SemanticKernel.Abstractions/SemanticKernel.Abstractions.csproj +++ b/dotnet/src/SemanticKernel.Abstractions/SemanticKernel.Abstractions.csproj @@ -26,6 +26,9 @@ <_Parameter1>Microsoft.SemanticKernel.Core + + <_Parameter1>Microsoft.SemanticKernel + <_Parameter1>Microsoft.SemanticKernel.Connectors.AI.OpenAI diff --git a/dotnet/src/SemanticKernel.IntegrationTests/CoreSkills/PlannerSkillTests.cs b/dotnet/src/SemanticKernel.IntegrationTests/CoreSkills/PlannerSkillTests.cs deleted file mode 100644 index 574e37969b46..000000000000 --- a/dotnet/src/SemanticKernel.IntegrationTests/CoreSkills/PlannerSkillTests.cs +++ /dev/null @@ -1,267 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Text.RegularExpressions; -using System.Threading.Tasks; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.CoreSkills; -using Microsoft.SemanticKernel.Memory; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.Planning; -using SemanticKernel.IntegrationTests.Fakes; -using SemanticKernel.IntegrationTests.TestSettings; -using Xunit; -using Xunit.Abstractions; - -namespace SemanticKernel.IntegrationTests.CoreSkills; - -public sealed class PlannerSkillTests : IDisposable -{ - public PlannerSkillTests(ITestOutputHelper output) - { - this._logger = new XunitLogger(output); - this._testOutputHelper = new RedirectOutput(output); - - // Load configuration - this._configuration = new ConfigurationBuilder() - .AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true) - .AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true) - .AddEnvironmentVariables() - .AddUserSecrets() - .Build(); - } - - [Theory] - [InlineData("Write a poem or joke and send it in an e-mail to Kai.", "function._GLOBAL_FUNCTIONS_.SendEmail")] - public async Task CreatePlanWithEmbeddingsTestAsync(string prompt, string expectedAnswerContains) - { - // Arrange - AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); - Assert.NotNull(azureOpenAIConfiguration); - - AzureOpenAIConfiguration? azureOpenAIEmbeddingsConfiguration = this._configuration.GetSection("AzureOpenAIEmbeddings").Get(); - Assert.NotNull(azureOpenAIEmbeddingsConfiguration); - - IKernel target = Kernel.Builder - .WithLogger(this._logger) - .Configure(config => - { - config.AddAzureTextCompletionService( - serviceId: azureOpenAIConfiguration.ServiceId, - deploymentName: azureOpenAIConfiguration.DeploymentName, - endpoint: azureOpenAIConfiguration.Endpoint, - apiKey: azureOpenAIConfiguration.ApiKey); - - config.AddAzureTextEmbeddingGenerationService( - serviceId: azureOpenAIEmbeddingsConfiguration.ServiceId, - deploymentName: azureOpenAIEmbeddingsConfiguration.DeploymentName, - endpoint: azureOpenAIEmbeddingsConfiguration.Endpoint, - apiKey: azureOpenAIEmbeddingsConfiguration.ApiKey); - - config.SetDefaultTextCompletionService(azureOpenAIConfiguration.ServiceId); - }) - .WithMemoryStorage(new VolatileMemoryStore()) - .Build(); - - // Import all sample skills available for demonstration purposes. - TestHelpers.ImportSampleSkills(target); - - var emailSkill = target.ImportSkill(new EmailSkillFake()); - - var plannerSKill = target.ImportSkill(new PlannerSkill(target)); - - // Act - ContextVariables variables = new(prompt); - variables.Set(PlannerSkill.Parameters.ExcludedSkills, "IntentDetectionSkill,FunSkill,CodingSkill"); - variables.Set(PlannerSkill.Parameters.ExcludedFunctions, "EmailTo"); - variables.Set(PlannerSkill.Parameters.IncludedFunctions, "Continue"); - variables.Set(PlannerSkill.Parameters.MaxRelevantFunctions, "19"); - variables.Set(PlannerSkill.Parameters.RelevancyThreshold, "0.5"); - SKContext actual = await target.RunAsync(variables, plannerSKill["CreatePlan"]).ConfigureAwait(true); - - // Assert - Assert.Empty(actual.LastErrorDescription); - Assert.False(actual.ErrorOccurred); - - this._logger.LogTrace("RESULT: {0}", actual.Result); - Assert.Contains(expectedAnswerContains, actual.Result, StringComparison.OrdinalIgnoreCase); - } - - [Theory] - [InlineData("If is morning tell me a joke about coffee", - "function.FunSkill.Joke", 1, - "", 1, - "", 0, - "", 0)] - [InlineData("If is morning tell me a joke about coffee, otherwise tell me a joke about the sun ", - "function.FunSkill.Joke", 2, - "", 1, - "", 1, - "", 1)] - [InlineData("If is morning tell me a joke about coffee, otherwise tell me a joke about the sun, but if its night I want a joke about the moon", - "function.FunSkill.Joke", 3, - "", 2, - "", 2, - "", 2)] - public async Task CreatePlanShouldHaveIfElseConditionalStatementsAndBeAbleToExecuteAsync(string prompt, params object[] expectedAnswerContainsAtLeast) - { - // Arrange - - Dictionary expectedAnswerContainsDictionary = new(); - for (int i = 0; i < expectedAnswerContainsAtLeast.Length; i += 2) - { - string? key = expectedAnswerContainsAtLeast[i].ToString(); - int value = Convert.ToInt32(expectedAnswerContainsAtLeast[i + 1], CultureInfo.InvariantCulture); - expectedAnswerContainsDictionary.Add(key!, value); - } - - AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); - Assert.NotNull(azureOpenAIConfiguration); - - IKernel target = Kernel.Builder - .WithLogger(this._logger) - .Configure(config => - { - config.AddAzureTextCompletionService( - serviceId: azureOpenAIConfiguration.ServiceId, - deploymentName: azureOpenAIConfiguration.DeploymentName, - endpoint: azureOpenAIConfiguration.Endpoint, - apiKey: azureOpenAIConfiguration.ApiKey); - - config.SetDefaultTextCompletionService(azureOpenAIConfiguration.ServiceId); - }) - .Build(); - - TestHelpers.GetSkill("FunSkill", target); - target.ImportSkill(new TimeSkill()); - var plannerSKill = target.ImportSkill(new PlannerSkill(target)); - - // Act - var context = new ContextVariables(prompt); - context.Set(PlannerSkill.Parameters.UseConditionals, "true"); - SKContext createdPlanContext = await target.RunAsync(context, plannerSKill["CreatePlan"]).ConfigureAwait(true); - await target.RunAsync(createdPlanContext.Variables.Clone(), plannerSKill["ExecutePlan"]).ConfigureAwait(false); - var planResult = createdPlanContext.Variables[SkillPlan.PlanKey]; - - // Assert - Assert.Empty(createdPlanContext.LastErrorDescription); - Assert.False(createdPlanContext.ErrorOccurred); - await this._testOutputHelper.WriteLineAsync(planResult); - - foreach ((string? matchingExpression, int minimumExpectedCount) in expectedAnswerContainsDictionary) - { - if (minimumExpectedCount > 0) - { - Assert.Contains(matchingExpression, planResult, StringComparison.OrdinalIgnoreCase); - } - - var numberOfMatches = Regex.Matches(planResult, matchingExpression, RegexOptions.IgnoreCase).Count; - Assert.True(numberOfMatches >= minimumExpectedCount, - $"Minimal number of matches below expected. Current: {numberOfMatches} Expected: {minimumExpectedCount} - Match: {matchingExpression}"); - } - } - - [Theory] - [InlineData( - "Start with a X number equals to the current minutes of the clock and remove 20 from this number until it becomes 0. After that tell me a math style joke where the input is X number + \"bananas\"", - "function.TimeSkill.Minute", 1, - "function.FunSkill.Joke", 1, - "", 1)] - [InlineData("Until time is not noon wait 5 seconds after that check again and if it is create a creative joke", - "function.TimeSkill", 1, - "function.FunSkill.Joke", 1, - "function.WaitSkill.Seconds", 1, - "", 1)] - public async Task CreatePlanShouldHaveWhileConditionalStatementsAndBeAbleToExecuteAsync(string prompt, params object[] expectedAnswerContainsAtLeast) - { - // Arrange - - Dictionary expectedAnswerContainsDictionary = new(); - for (int i = 0; i < expectedAnswerContainsAtLeast.Length; i += 2) - { - string? key = expectedAnswerContainsAtLeast[i].ToString(); - int value = Convert.ToInt32(expectedAnswerContainsAtLeast[i + 1], CultureInfo.InvariantCulture); - expectedAnswerContainsDictionary.Add(key!, value); - } - - AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); - Assert.NotNull(azureOpenAIConfiguration); - - IKernel target = Kernel.Builder - .WithLogger(this._logger) - .Configure(config => - { - config.AddAzureTextCompletionService( - serviceId: azureOpenAIConfiguration.ServiceId, - deploymentName: azureOpenAIConfiguration.DeploymentName, - endpoint: azureOpenAIConfiguration.Endpoint, - apiKey: azureOpenAIConfiguration.ApiKey); - - config.SetDefaultTextCompletionService(azureOpenAIConfiguration.ServiceId); - }) - .Build(); - - TestHelpers.GetSkill("FunSkill", target); - target.ImportSkill(new TimeSkill(), "TimeSkill"); - target.ImportSkill(new MathSkill(), "MathSkill"); - target.ImportSkill(new WaitSkill(), "WaitSkill"); - var plannerSKill = target.ImportSkill(new PlannerSkill(target)); - - // Act - var context = new ContextVariables(prompt); - context.Set(PlannerSkill.Parameters.UseConditionals, "true"); - SKContext createdPlanContext = await target.RunAsync(context, plannerSKill["CreatePlan"]).ConfigureAwait(true); - await target.RunAsync(createdPlanContext.Variables.Clone(), plannerSKill["ExecutePlan"]).ConfigureAwait(false); - var planResult = createdPlanContext.Variables[SkillPlan.PlanKey]; - - // Assert - Assert.Empty(createdPlanContext.LastErrorDescription); - Assert.False(createdPlanContext.ErrorOccurred); - await this._testOutputHelper.WriteLineAsync(planResult); - - foreach ((string? matchingExpression, int minimumExpectedCount) in expectedAnswerContainsDictionary) - { - if (minimumExpectedCount > 0) - { - Assert.Contains(matchingExpression, planResult, StringComparison.OrdinalIgnoreCase); - } - - var numberOfMatches = Regex.Matches(planResult, matchingExpression, RegexOptions.IgnoreCase).Count; - Assert.True(numberOfMatches >= minimumExpectedCount, - $"Minimal number of matches below expected. Current: {numberOfMatches} Expected: {minimumExpectedCount} - Match: {matchingExpression}"); - } - } - - private readonly XunitLogger _logger; - private readonly RedirectOutput _testOutputHelper; - private readonly IConfigurationRoot _configuration; - - public void Dispose() - { - this.Dispose(true); - GC.SuppressFinalize(this); - } - - ~PlannerSkillTests() - { - this.Dispose(false); - } - - private void Dispose(bool disposing) - { - if (disposing) - { - this._logger.Dispose(); - this._testOutputHelper.Dispose(); - } - } -} diff --git a/dotnet/src/SemanticKernel.IntegrationTests/Orchestration/PlanTests.cs b/dotnet/src/SemanticKernel.IntegrationTests/Orchestration/PlanTests.cs new file mode 100644 index 000000000000..38b4f013230c --- /dev/null +++ b/dotnet/src/SemanticKernel.IntegrationTests/Orchestration/PlanTests.cs @@ -0,0 +1,417 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Memory; +using Microsoft.SemanticKernel.Orchestration; +using SemanticKernel.IntegrationTests.Fakes; +using SemanticKernel.IntegrationTests.TestSettings; +using Xunit; +using Xunit.Abstractions; + +namespace SemanticKernel.IntegrationTests.Orchestration; + +public sealed class PlanTests : IDisposable +{ + public PlanTests(ITestOutputHelper output) + { + this._logger = NullLogger.Instance; //new XunitLogger(output); + this._testOutputHelper = new RedirectOutput(output); + + // Load configuration + this._configuration = new ConfigurationBuilder() + .AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true) + .AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true) + .AddEnvironmentVariables() + .AddUserSecrets() + .Build(); + } + + [Theory] + [InlineData("Write a poem or joke and send it in an e-mail to Kai.")] + public void CreatePlan(string prompt) + { + // Arrange + + // Act + var plan = new Plan(prompt); + + // Assert + Assert.Equal(prompt, plan.Description); + Assert.Equal(prompt, plan.Name); + Assert.Equal("Microsoft.SemanticKernel.Orchestration.Plan", plan.SkillName); + Assert.Empty(plan.Steps); + } + + [Theory] + [InlineData("This is a story about a dog.", "kai@email.com")] + public async Task CanExecuteRunSimpleAsync(string inputToEmail, string expectedEmail) + { + // Arrange + IKernel target = this.InitializeKernel(); + var emailSkill = target.ImportSkill(new EmailSkillFake()); + var expectedBody = $"Sent email to: {expectedEmail}. Body: {inputToEmail}".Trim(); + + var plan = new Plan(emailSkill["SendEmailAsync"]); + + // Act + var cv = new ContextVariables(); + cv.Update(inputToEmail); + cv.Set("email_address", expectedEmail); + var result = await target.RunAsync(cv, plan); + + // Assert + Assert.Equal(expectedBody, result.Result); + } + + [Theory] + [InlineData("Send a story to kai.", "This is a story about a dog.", "French", "kai@email.com")] + public async Task CanExecuteRunSimpleStepsAsync(string goal, string inputToTranslate, string language, string expectedEmail) + { + // Arrange + IKernel target = this.InitializeKernel(); + var emailSkill = target.ImportSkill(new EmailSkillFake()); + var writerSkill = TestHelpers.GetSkill("WriterSkill", target); + var expectedBody = $"Sent email to: {expectedEmail}. Body:".Trim(); + + var plan = new Plan(goal); + plan.AddSteps(writerSkill["Translate"], emailSkill["SendEmailAsync"]); + + // Act + var cv = new ContextVariables(); + cv.Update(inputToTranslate); + cv.Set("email_address", expectedEmail); + cv.Set("language", language); + var result = await target.RunAsync(cv, plan); + + // Assert + Assert.Contains(expectedBody, result.Result, StringComparison.OrdinalIgnoreCase); + Assert.True(expectedBody.Length < result.Result.Length); + } + + [Fact] + public async Task CanExecutePanWithTreeStepsAsync() + { + // Arrange + IKernel target = this.InitializeKernel(); + var goal = "Write a poem or joke and send it in an e-mail to Kai."; + var plan = new Plan(goal); + var subPlan = new Plan("Write a poem or joke"); + + var emailSkill = target.ImportSkill(new EmailSkillFake()); + + // Arrange + var returnContext = target.CreateNewContext(); + + subPlan.AddSteps(emailSkill["WritePoemAsync"], emailSkill["WritePoemAsync"], emailSkill["WritePoemAsync"]); + plan.AddSteps(subPlan, emailSkill["SendEmailAsync"]); + plan.State.Set("email_address", "something@email.com"); + + // Act + var result = await target.RunAsync("PlanInput", plan); + + // Assert + Assert.NotNull(result); + Assert.Equal( + $"Sent email to: something@email.com. Body: Roses are red, violets are blue, Roses are red, violets are blue, Roses are red, violets are blue, PlanInput is hard, so is this test. is hard, so is this test. is hard, so is this test.", + result.Result); + } + + [Theory] + [InlineData(null, "Write a poem or joke and send it in an e-mail to Kai.", null)] + [InlineData("", "Write a poem or joke and send it in an e-mail to Kai.", "")] + [InlineData("Hello World!", "Write a poem or joke and send it in an e-mail to Kai.", "some_email@email.com")] + public async Task CanExecuteRunPlanSimpleManualStateAsync(string input, string goal, string email) + { + // Arrange + IKernel target = this.InitializeKernel(); + var emailSkill = target.ImportSkill(new EmailSkillFake()); + + // Create the input mapping from parent (plan) plan state to child plan (sendEmailPlan) state. + var cv = new ContextVariables(); + cv.Set("email_address", "$TheEmailFromState"); + var sendEmailPlan = new Plan(emailSkill["SendEmailAsync"]) + { + NamedParameters = cv, + }; + + var plan = new Plan(goal); + plan.AddSteps(sendEmailPlan); + plan.State.Set("TheEmailFromState", email); // manually prepare the state + + // Act + var result = await target.StepAsync(input, plan); + + // Assert + var expectedBody = string.IsNullOrEmpty(input) ? goal : input; + Assert.Single(result.Steps); + Assert.Equal(1, result.NextStepIndex); + Assert.False(result.HasNextStep); + Assert.Equal(goal, plan.Description); + Assert.Equal($"Sent email to: {email}. Body: {expectedBody}".Trim(), plan.State.ToString()); + } + + [Theory] + [InlineData(null, "Write a poem or joke and send it in an e-mail to Kai.", null)] + [InlineData("", "Write a poem or joke and send it in an e-mail to Kai.", "")] + [InlineData("Hello World!", "Write a poem or joke and send it in an e-mail to Kai.", "some_email@email.com")] + public async Task CanExecuteRunPlanManualStateAsync(string input, string goal, string email) + { + // Arrange + IKernel target = this.InitializeKernel(); + + var emailSkill = target.ImportSkill(new EmailSkillFake()); + + // Create the input mapping from parent (plan) plan state to child plan (sendEmailPlan) state. + var cv = new ContextVariables(); + cv.Set("email_address", "$TheEmailFromState"); + var sendEmailPlan = new Plan(emailSkill["SendEmailAsync"]) + { + NamedParameters = cv + }; + + var plan = new Plan(goal); + plan.AddSteps(sendEmailPlan); + plan.State.Set("TheEmailFromState", email); // manually prepare the state + + // Act + var result = await target.StepAsync(input, plan); + + // Assert + var expectedBody = string.IsNullOrEmpty(input) ? goal : input; + Assert.False(plan.HasNextStep); + Assert.Equal(goal, plan.Description); + Assert.Equal($"Sent email to: {email}. Body: {expectedBody}".Trim(), plan.State.ToString()); + } + + [Theory] + [InlineData("Summarize an input, translate to french, and e-mail to Kai", "This is a story about a dog.", "French", "Kai", "Kai@example.com")] + public async Task CanExecuteRunPlanAsync(string goal, string inputToSummarize, string inputLanguage, string inputName, string expectedEmail) + { + // Arrange + IKernel target = this.InitializeKernel(); + + var summarizeSkill = TestHelpers.GetSkill("SummarizeSkill", target); + var writerSkill = TestHelpers.GetSkill("WriterSkill", target); + var emailSkill = target.ImportSkill(new EmailSkillFake()); + + var expectedBody = $"Sent email to: {expectedEmail}. Body:".Trim(); + + var summarizePlan = new Plan(summarizeSkill["Summarize"]); + + var cv = new ContextVariables(); + cv.Set("language", inputLanguage); + var outputs = new ContextVariables(); + outputs.Set("TRANSLATED_SUMMARY", string.Empty); + var translatePlan = new Plan(writerSkill["Translate"]) + { + NamedParameters = cv, + NamedOutputs = outputs, + }; + + cv = new ContextVariables(); + cv.Update(inputName); + outputs = new ContextVariables(); + outputs.Set("TheEmailFromState", string.Empty); + var getEmailPlan = new Plan(emailSkill["GetEmailAddressAsync"]) + { + NamedParameters = cv, + NamedOutputs = outputs, + }; + + cv = new ContextVariables(); + cv.Set("email_address", "$TheEmailFromState"); + cv.Set("input", "$TRANSLATED_SUMMARY"); + var sendEmailPlan = new Plan(emailSkill["SendEmailAsync"]) + { + NamedParameters = cv + }; + + var plan = new Plan(goal); + plan.AddSteps(summarizePlan, translatePlan, getEmailPlan, sendEmailPlan); + + // Act + var result = await target.StepAsync(inputToSummarize, plan); + Assert.Equal(4, result.Steps.Count); + Assert.Equal(1, result.NextStepIndex); + Assert.True(result.HasNextStep); + result = await target.StepAsync(result); + Assert.Equal(4, result.Steps.Count); + Assert.Equal(2, result.NextStepIndex); + Assert.True(result.HasNextStep); + result = await target.StepAsync(result); + Assert.Equal(4, result.Steps.Count); + Assert.Equal(3, result.NextStepIndex); + Assert.True(result.HasNextStep); + result = await target.StepAsync(result); + + // Assert + Assert.Equal(4, result.Steps.Count); + Assert.Equal(4, result.NextStepIndex); + Assert.False(result.HasNextStep); + Assert.Equal(goal, plan.Description); + Assert.Contains(expectedBody, plan.State.ToString(), StringComparison.OrdinalIgnoreCase); + Assert.True(expectedBody.Length < plan.State.ToString().Length); + } + + [Theory] + [InlineData("Summarize an input, translate to french, and e-mail to Kai", "This is a story about a dog.", "French", "Kai", "Kai@example.com")] + public async Task CanExecuteRunSequentialAsync(string goal, string inputToSummarize, string inputLanguage, string inputName, string expectedEmail) + { + // Arrange + IKernel target = this.InitializeKernel(); + var summarizeSkill = TestHelpers.GetSkill("SummarizeSkill", target); + var writerSkill = TestHelpers.GetSkill("WriterSkill", target); + var emailSkill = target.ImportSkill(new EmailSkillFake()); + + var expectedBody = $"Sent email to: {expectedEmail}. Body:".Trim(); + + var summarizePlan = new Plan(summarizeSkill["Summarize"]); + + var cv = new ContextVariables(); + cv.Set("language", inputLanguage); + var outputs = new ContextVariables(); + outputs.Set("TRANSLATED_SUMMARY", string.Empty); + + var translatePlan = new Plan(writerSkill["Translate"]) + { + NamedParameters = cv, + NamedOutputs = outputs, + }; + + cv = new ContextVariables(); + cv.Update(inputName); + outputs = new ContextVariables(); + outputs.Set("TheEmailFromState", string.Empty); + var getEmailPlan = new Plan(emailSkill["GetEmailAddressAsync"]) + { + NamedParameters = cv, + NamedOutputs = outputs, + }; + + cv = new ContextVariables(); + cv.Set("email_address", "$TheEmailFromState"); + cv.Set("input", "$TRANSLATED_SUMMARY"); + var sendEmailPlan = new Plan(emailSkill["SendEmailAsync"]) + { + NamedParameters = cv + }; + + var plan = new Plan(goal); + plan.AddSteps(summarizePlan, translatePlan, getEmailPlan, sendEmailPlan); + + // Act + var result = await target.RunAsync(inputToSummarize, plan); + + // Assert + Assert.Contains(expectedBody, result.Result, StringComparison.OrdinalIgnoreCase); + Assert.True(expectedBody.Length < result.Result.Length); + } + + [Theory] + [InlineData("Summarize an input, translate to french, and e-mail to Kai", "This is a story about a dog.", "French", "kai@email.com")] + public async Task CanExecuteRunSequentialFunctionsAsync(string goal, string inputToSummarize, string inputLanguage, string expectedEmail) + { + // Arrange + IKernel target = this.InitializeKernel(); + + var summarizeSkill = TestHelpers.GetSkill("SummarizeSkill", target); + var writerSkill = TestHelpers.GetSkill("WriterSkill", target); + var emailSkill = target.ImportSkill(new EmailSkillFake()); + + var expectedBody = $"Sent email to: {expectedEmail}. Body:".Trim(); + + var summarizePlan = new Plan(summarizeSkill["Summarize"]); + var translatePlan = new Plan(writerSkill["Translate"]); + var sendEmailPlan = new Plan(emailSkill["SendEmailAsync"]); + + var plan = new Plan(goal); + plan.AddSteps(summarizePlan, translatePlan, sendEmailPlan); + + // Act + var cv = new ContextVariables(); + cv.Update(inputToSummarize); + cv.Set("email_address", expectedEmail); + cv.Set("language", inputLanguage); + var result = await target.RunAsync(cv, plan); + + // Assert + Assert.Contains(expectedBody, result.Result, StringComparison.OrdinalIgnoreCase); + } + + private IKernel InitializeKernel(bool useEmbeddings = false) + { + AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); + Assert.NotNull(azureOpenAIConfiguration); + + AzureOpenAIConfiguration? azureOpenAIEmbeddingsConfiguration = this._configuration.GetSection("AzureOpenAIEmbeddings").Get(); + Assert.NotNull(azureOpenAIEmbeddingsConfiguration); + + var builder = Kernel.Builder + .WithLogger(this._logger) + .Configure(config => + { + config.AddAzureTextCompletionService( + serviceId: azureOpenAIConfiguration.ServiceId, + deploymentName: azureOpenAIConfiguration.DeploymentName, + endpoint: azureOpenAIConfiguration.Endpoint, + apiKey: azureOpenAIConfiguration.ApiKey); + + if (useEmbeddings) + { + config.AddAzureTextEmbeddingGenerationService( + serviceId: azureOpenAIEmbeddingsConfiguration.ServiceId, + deploymentName: azureOpenAIEmbeddingsConfiguration.DeploymentName, + endpoint: azureOpenAIEmbeddingsConfiguration.Endpoint, + apiKey: azureOpenAIEmbeddingsConfiguration.ApiKey); + } + + config.SetDefaultTextCompletionService(azureOpenAIConfiguration.ServiceId); + }); + + if (useEmbeddings) + { + builder = builder.WithMemoryStorage(new VolatileMemoryStore()); + } + + var kernel = builder.Build(); + + // Import all sample skills available for demonstration purposes. + TestHelpers.ImportSampleSkills(kernel); + + _ = kernel.ImportSkill(new EmailSkillFake()); + return kernel; + } + + private readonly ILogger _logger; + private readonly RedirectOutput _testOutputHelper; + private readonly IConfigurationRoot _configuration; + + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + ~PlanTests() + { + this.Dispose(false); + } + + private void Dispose(bool disposing) + { + if (disposing) + { + if (this._logger is IDisposable ld) + { + ld.Dispose(); + } + + this._testOutputHelper.Dispose(); + } + } +} diff --git a/dotnet/src/SemanticKernel.IntegrationTests/Planning/SequentialPlanParserTests.cs b/dotnet/src/SemanticKernel.IntegrationTests/Planning/SequentialPlanParserTests.cs new file mode 100644 index 000000000000..311a472f8bb6 --- /dev/null +++ b/dotnet/src/SemanticKernel.IntegrationTests/Planning/SequentialPlanParserTests.cs @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.Configuration; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Planning; +using SemanticKernel.IntegrationTests.Fakes; +using SemanticKernel.IntegrationTests.TestSettings; +using Xunit; +using Xunit.Abstractions; + +namespace SemanticKernel.IntegrationTests.Planning; + +public class SequentialPlanParserTests +{ + public SequentialPlanParserTests(ITestOutputHelper output) + { + // Load configuration + this._configuration = new ConfigurationBuilder() + .AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true) + .AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true) + .AddEnvironmentVariables() + .AddUserSecrets() + .Build(); + } + + [Fact] + public void CanCallToPlanFromXml() + { + // Arrange + AzureOpenAIConfiguration? azureOpenAIConfiguration = this._configuration.GetSection("AzureOpenAI").Get(); + Assert.NotNull(azureOpenAIConfiguration); + + IKernel kernel = Kernel.Builder + .Configure(config => + { + config.AddAzureTextCompletionService( + serviceId: azureOpenAIConfiguration.ServiceId, + deploymentName: azureOpenAIConfiguration.DeploymentName, + endpoint: azureOpenAIConfiguration.Endpoint, + apiKey: azureOpenAIConfiguration.ApiKey); + config.SetDefaultTextCompletionService(azureOpenAIConfiguration.ServiceId); + }) + .Build(); + kernel.ImportSkill(new EmailSkillFake(), "email"); + var summarizeSkill = TestHelpers.GetSkill("SummarizeSkill", kernel); + var writerSkill = TestHelpers.GetSkill("WriterSkill", kernel); + + var planString = + @" +Summarize an input, translate to french, and e-mail to John Doe + + + + + + +"; + + // Act + var plan = planString.ToPlanFromXml(kernel.CreateNewContext()); + + // Assert + Assert.NotNull(plan); + Assert.Equal("Summarize an input, translate to french, and e-mail to John Doe", plan.Description); + + Assert.Equal(4, plan.Steps.Count); + Assert.Collection(plan.Steps, + step => + { + Assert.Equal("SummarizeSkill", step.SkillName); + Assert.Equal("Summarize", step.Name); + }, + step => + { + Assert.Equal("WriterSkill", step.SkillName); + Assert.Equal("Translate", step.Name); + Assert.Equal("French", step.NamedParameters["language"]); + Assert.True(step.NamedOutputs.ContainsKey("TRANSLATED_SUMMARY")); + }, + step => + { + Assert.Equal("email", step.SkillName); + Assert.Equal("GetEmailAddressAsync", step.Name); + Assert.Equal("John Doe", step.NamedParameters["input"]); + Assert.True(step.NamedOutputs.ContainsKey("EMAIL_ADDRESS")); + }, + step => + { + Assert.Equal("email", step.SkillName); + Assert.Equal("SendEmailAsync", step.Name); + Assert.Equal("$TRANSLATED_SUMMARY", step.NamedParameters["input"]); + Assert.Equal("$EMAIL_ADDRESS", step.NamedParameters["email_address"]); + } + ); + } + + private readonly IConfigurationRoot _configuration; +} diff --git a/dotnet/src/SemanticKernel.IntegrationTests/Planning/PlanTests.cs b/dotnet/src/SemanticKernel.IntegrationTests/Planning/SequentialPlannerTests.cs similarity index 51% rename from dotnet/src/SemanticKernel.IntegrationTests/Planning/PlanTests.cs rename to dotnet/src/SemanticKernel.IntegrationTests/Planning/SequentialPlannerTests.cs index dcbcb3da3e70..76c91d59be58 100644 --- a/dotnet/src/SemanticKernel.IntegrationTests/Planning/PlanTests.cs +++ b/dotnet/src/SemanticKernel.IntegrationTests/Planning/SequentialPlannerTests.cs @@ -7,7 +7,7 @@ using Microsoft.Extensions.Logging.Abstractions; using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.Memory; -using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.Planning.Planners; using SemanticKernel.IntegrationTests.Fakes; using SemanticKernel.IntegrationTests.TestSettings; using Xunit; @@ -15,9 +15,9 @@ namespace SemanticKernel.IntegrationTests.Planning; -public sealed class PlanTests : IDisposable +public sealed class SequentialPlannerTests : IDisposable { - public PlanTests(ITestOutputHelper output) + public SequentialPlannerTests(ITestOutputHelper output) { this._logger = NullLogger.Instance; //new XunitLogger(output); this._testOutputHelper = new RedirectOutput(output); @@ -27,98 +27,51 @@ public PlanTests(ITestOutputHelper output) .AddJsonFile(path: "testsettings.json", optional: false, reloadOnChange: true) .AddJsonFile(path: "testsettings.development.json", optional: true, reloadOnChange: true) .AddEnvironmentVariables() - .AddUserSecrets() + .AddUserSecrets() .Build(); } [Theory] - [InlineData("Write a poem or joke and send it in an e-mail to Kai.")] - public void CreatePlan(string prompt) + [InlineData("Write a joke and send it in an e-mail to Kai.", "SendEmailAsync", "_GLOBAL_FUNCTIONS_")] + public async Task CreatePlanFunctionFlowAsync(string prompt, string expectedFunction, string expectedSkill) { // Arrange + IKernel kernel = this.InitializeKernel(); + TestHelpers.GetSkill("FunSkill", kernel); - // Act - var plan = new Plan(prompt); - - // Assert - Assert.Equal(prompt, plan.Description); - Assert.Equal(prompt, plan.Name); - Assert.Equal("Microsoft.SemanticKernel.Orchestration.Plan", plan.SkillName); - Assert.Empty(plan.Steps); - } - - [Theory] - [InlineData("This is a story about a dog.", "kai@email.com")] - public async Task CanExecuteRunSimpleAsync(string inputToEmail, string expectedEmail) - { - // Arrange - IKernel target = this.InitializeKernel(); - var emailSkill = target.ImportSkill(new EmailSkillFake()); - var expectedBody = $"Sent email to: {expectedEmail}. Body: {inputToEmail}".Trim(); - - var plan = new Plan(emailSkill["SendEmailAsync"]); + var planner = new SequentialPlanner(kernel); // Act - var cv = new ContextVariables(); - cv.Update(inputToEmail); - cv.Set("email_address", expectedEmail); - var result = await target.RunAsync(cv, plan); - + var plan = await planner.CreatePlanAsync(prompt); // Assert - Assert.Equal(expectedBody, result.Result); + Assert.Contains( + plan.Steps, + step => + step.Name.Equals(expectedFunction, StringComparison.OrdinalIgnoreCase) && + step.SkillName.Equals(expectedSkill, StringComparison.OrdinalIgnoreCase)); } [Theory] - [InlineData("Send a story to kai.", "This is a story about a dog.", "French", "kai@email.com")] - public async Task CanExecuteRunSimpleStepsAsync(string goal, string inputToTranslate, string language, string expectedEmail) - { - // Arrange - IKernel target = this.InitializeKernel(); - var emailSkill = target.ImportSkill(new EmailSkillFake()); - var writerSkill = TestHelpers.GetSkill("WriterSkill", target); - var expectedBody = $"Sent email to: {expectedEmail}. Body:".Trim(); - - var plan = new Plan(goal); - plan.AddSteps(writerSkill["Translate"], emailSkill["SendEmailAsync"]); - - // Act - var cv = new ContextVariables(); - cv.Update(inputToTranslate); - cv.Set("email_address", expectedEmail); - cv.Set("language", language); - var result = await target.RunAsync(cv, plan); - - // Assert - Assert.Contains(expectedBody, result.Result, StringComparison.OrdinalIgnoreCase); - Assert.True(expectedBody.Length < result.Result.Length); - } - - [Fact] - public async Task CanExecutePanWithTreeStepsAsync() + [InlineData("Write a poem or joke and send it in an e-mail to Kai.", "SendEmailAsync", "_GLOBAL_FUNCTIONS_")] + public async Task CreatePlanGoalRelevantAsync(string prompt, string expectedFunction, string expectedSkill) { // Arrange - IKernel target = this.InitializeKernel(); - var goal = "Write a poem or joke and send it in an e-mail to Kai."; - var plan = new Plan(goal); - var subPlan = new Plan("Write a poem or joke"); - - var emailSkill = target.ImportSkill(new EmailSkillFake()); + IKernel kernel = this.InitializeKernel(true); - // Arrange - var returnContext = target.CreateNewContext(); + // Import all sample skills available for demonstration purposes. + TestHelpers.ImportSampleSkills(kernel); - subPlan.AddSteps(emailSkill["WritePoemAsync"], emailSkill["WritePoemAsync"], emailSkill["WritePoemAsync"]); - plan.AddSteps(subPlan, emailSkill["SendEmailAsync"]); - plan.State.Set("email_address", "something@email.com"); + var planner = new SequentialPlanner(kernel, new PlannerConfig() { RelevancyThreshold = 0.70, MaxRelevantFunctions = 20 }); // Act - var result = await target.RunAsync("PlanInput", plan); + var plan = await planner.CreatePlanAsync(prompt); // Assert - Assert.NotNull(result); - Assert.Equal( - $"Sent email to: something@email.com. Body: Roses are red, violets are blue, Roses are red, violets are blue, Roses are red, violets are blue, PlanInput is hard, so is this test. is hard, so is this test. is hard, so is this test.", - result.Result); + Assert.Contains( + plan.Steps, + step => + step.Name.Equals(expectedFunction, StringComparison.OrdinalIgnoreCase) && + step.SkillName.Equals(expectedSkill, StringComparison.OrdinalIgnoreCase)); } private IKernel InitializeKernel(bool useEmbeddings = false) @@ -158,10 +111,7 @@ private IKernel InitializeKernel(bool useEmbeddings = false) var kernel = builder.Build(); - // Import all sample skills available for demonstration purposes. - TestHelpers.ImportSampleSkills(kernel); - - var emailSkill = kernel.ImportSkill(new EmailSkillFake()); + _ = kernel.ImportSkill(new EmailSkillFake()); return kernel; } @@ -175,7 +125,7 @@ public void Dispose() GC.SuppressFinalize(this); } - ~PlanTests() + ~SequentialPlannerTests() { this.Dispose(false); } diff --git a/dotnet/src/SemanticKernel.IntegrationTests/WebSkill/WebSkillTests.cs b/dotnet/src/SemanticKernel.IntegrationTests/WebSkill/WebSkillTests.cs index d532b364e3b8..c29614519e1e 100644 --- a/dotnet/src/SemanticKernel.IntegrationTests/WebSkill/WebSkillTests.cs +++ b/dotnet/src/SemanticKernel.IntegrationTests/WebSkill/WebSkillTests.cs @@ -38,7 +38,7 @@ public WebSkillTests(ITestOutputHelper output) this._bingApiKey = bingApiKeyCandidate; } - [Theory] + [Theory(Skip = "Bing search results not consistent enough for testing.")] [InlineData("What is generally recognized as the tallest building in Seattle, Washington, USA?", "Columbia Center")] public async Task BingSkillTestAsync(string prompt, string expectedAnswerContains) { diff --git a/dotnet/src/SemanticKernel.UnitTests/CoreSkills/PlannerSkillTests.cs b/dotnet/src/SemanticKernel.UnitTests/CoreSkills/PlannerSkillTests.cs deleted file mode 100644 index 20b5c224e946..000000000000 --- a/dotnet/src/SemanticKernel.UnitTests/CoreSkills/PlannerSkillTests.cs +++ /dev/null @@ -1,516 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading.Tasks; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.AI.TextCompletion; -using Microsoft.SemanticKernel.CoreSkills; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.Planning; -using Microsoft.SemanticKernel.SkillDefinition; -using Moq; -using Xunit; -using Xunit.Abstractions; - -namespace SemanticKernel.UnitTests.CoreSkills; - -public class PlannerSkillTests -{ - private readonly ITestOutputHelper _testOutputHelper; - - private const string FunctionFlowRunnerText = @" - -Solve the equation x^2 = 2. - - - - -"; - - private const string GoalText = "Solve the equation x^2 = 2."; - - public PlannerSkillTests(ITestOutputHelper testOutputHelper) - { - this._testOutputHelper = testOutputHelper; - this._testOutputHelper.WriteLine("Tests initialized"); - } - - [Fact] - public void ItCanBeInstantiated() - { - // Arrange - var kernel = KernelBuilder.Create(); - var factory = new Mock>(); - kernel.Config.AddTextCompletionService("test", factory.Object); - - // Act - Assert no exception occurs - _ = new PlannerSkill(kernel); - } - - [Fact] - public void ItCanBeImported() - { - // Arrange - var kernel = KernelBuilder.Create(); - var factory = new Mock>(); - kernel.Config.AddTextCompletionService("test", factory.Object); - - // Act - Assert no exception occurs e.g. due to reflection - _ = kernel.ImportSkill(new PlannerSkill(kernel), "planner"); - } - - // TODO: fix the tests - they are not mocking the AI connector and should have been failing. - // Looks like they've been passing only because the planner is swallowing HTTP errors. - // [Fact] - // public async Task ItCanCreatePlanAsync() - // { - // // Arrange - // var kernel = KernelBuilder.Create(); - // var factory = new Mock>(); - // kernel.Config.AddTextCompletion("test", factory.Object, true); - // var plannerSkill = new PlannerSkill(kernel); - // var planner = kernel.ImportSkill(plannerSkill, "planner"); - // - // // Act - // var context = await kernel.RunAsync(GoalText, planner["CreatePlan"]); - // - // // Assert - // var plan = context.Variables.ToPlan(); - // Assert.NotNull(plan); - // Assert.NotNull(plan.Id); - // Assert.Equal(GoalText, plan.Goal); - // Assert.StartsWith("\nSolve the equation x^2 = 2.\n", plan.PlanString, StringComparison.OrdinalIgnoreCase); - // } - - [Fact] - public async Task ItCanExecutePlanTextAsync() - { - // Arrange - var kernel = KernelBuilder.Create(); - var factory = new Mock>(); - kernel.Config.AddTextCompletionService("test", factory.Object); - var plannerSkill = kernel.ImportSkill(new PlannerSkill(kernel)); - - // Act - var context = await kernel.RunAsync(FunctionFlowRunnerText, plannerSkill["ExecutePlan"]); - - // Assert - var plan = context.Variables.ToPlan(); - Assert.NotNull(plan); - Assert.NotNull(plan.Id); - - // Since not using SkillPlan or PlanExecution object, this won't be present. - // Maybe we do work to parse this out. Not doing too much though since we might move to json instead of xml. - // Assert.Equal(GoalText, plan.Goal); - } - - [Fact] - public async Task ItCanExecutePlanAsync() - { - // Arrange - var kernel = KernelBuilder.Create(); - var factory = new Mock>(); - kernel.Config.AddTextCompletionService("test", factory.Object); - var plannerSkill = kernel.ImportSkill(new PlannerSkill(kernel)); - SkillPlan createdPlan = new() - { - Goal = GoalText, - PlanString = FunctionFlowRunnerText - }; - - // Act - var variables = new ContextVariables(); - _ = variables.UpdateWithPlanEntry(createdPlan); - var context = await kernel.RunAsync(variables, plannerSkill["ExecutePlan"]); - - // Assert - var plan = context.Variables.ToPlan(); - Assert.NotNull(plan); - Assert.NotNull(plan.Id); - Assert.Equal(GoalText, plan.Goal); - } - - // TODO: fix the tests - they are not mocking the AI connector and should have been failing. - // Looks like they've been passing only because the planner is swallowing HTTP errors. - // [Fact] - // public async Task ItCanCreateSkillPlanAsync() - // { - // // Arrange - // var kernel = KernelBuilder.Create(); - // var factory = new Mock>(); - // kernel.Config.AddTextCompletion("test", factory.Object, true); - // var plannerSkill = kernel.ImportSkill(new PlannerSkill(kernel)); - // - // // Act - // var context = await kernel.RunAsync(GoalText, plannerSkill["CreatePlan"]); - // - // // Assert - // var plan = context.Variables.ToPlan(); - // Assert.NotNull(plan); - // Assert.NotNull(plan.Id); - // Assert.Equal(GoalText, plan.Goal); - // Assert.StartsWith("\nSolve the equation x^2 = 2.\n", plan.PlanString, StringComparison.OrdinalIgnoreCase); - // } - - [Fact] - public async Task ItCanExecutePlanJsonAsync() - { - // Arrange - var kernel = KernelBuilder.Create(); - var factory = new Mock>(); - kernel.Config.AddTextCompletionService("test", factory.Object); - var plannerSkill = kernel.ImportSkill(new PlannerSkill(kernel)); - SkillPlan createdPlan = new() - { - Goal = GoalText, - PlanString = FunctionFlowRunnerText - }; - - // Act - var context = await kernel.RunAsync(createdPlan.ToJson(), plannerSkill["ExecutePlan"]); - - // Assert - var plan = context.Variables.ToPlan(); - Assert.NotNull(plan); - Assert.NotNull(plan.Id); - Assert.Equal(GoalText, plan.Goal); - } - - [Fact] - public async Task NoGoalExecutePlanReturnsInvalidResultAsync() - { - // Arrange - var kernel = KernelBuilder.Create(); - var factory = new Mock>(); - kernel.Config.AddTextCompletionService("test", factory.Object); - var plannerSkill = kernel.ImportSkill(new PlannerSkill(kernel)); - - // Act - var context = await kernel.RunAsync(GoalText, plannerSkill["ExecutePlan"]); - - // Assert - var plan = context.Variables.ToPlan(); - Assert.NotNull(plan); - Assert.NotNull(plan.Id); - Assert.Equal(string.Empty, plan.Goal); - Assert.Equal(GoalText, plan.PlanString); - Assert.False(plan.IsSuccessful); - Assert.True(plan.IsComplete); - Assert.Contains("No goal found.", plan.Result, StringComparison.OrdinalIgnoreCase); - } - - [Fact] - public async Task InvalidPlanExecutePlanReturnsInvalidResultAsync() - { - // Arrange - var kernel = KernelBuilder.Create(); - var factory = new Mock>(); - kernel.Config.AddTextCompletionService("test", factory.Object); - var plannerSkill = kernel.ImportSkill(new PlannerSkill(kernel)); - - // Act - var context = await kernel.RunAsync("" + GoalText, plannerSkill["ExecutePlan"]); - - // Assert - var plan = context.Variables.ToPlan(); - Assert.NotNull(plan); - Assert.NotNull(plan.Id); - Assert.Equal(string.Empty, plan.Goal); - Assert.Equal("" + GoalText, plan.PlanString); - Assert.False(plan.IsSuccessful); - Assert.True(plan.IsComplete); - Assert.Contains("Failed to parse plan xml.", plan.Result, StringComparison.OrdinalIgnoreCase); - } - - // - // Advanced tests for ExecutePlan that use mock sk functions to test the flow - // - [Theory] - [InlineData("Test the functionFlowRunner", @"Test the functionFlowRunner - - -")] - public async Task ExecutePlanCanCallFunctionAsync(string goalText, string planText) - { - // Arrange - var kernel = KernelBuilder.Create(); - var factory = new Mock>(); - kernel.Config.AddTextCompletionService("test", factory.Object); - var plannerSkill = kernel.ImportSkill(new PlannerSkill(kernel)); - _ = kernel.ImportSkill(new MockSkill(this._testOutputHelper), "MockSkill"); - SkillPlan createdPlan = new() - { - Goal = goalText, - PlanString = planText - }; - - // Act - var context = await kernel.RunAsync(createdPlan.ToJson(), plannerSkill["ExecutePlan"]); - - // Assert - var plan = context.Variables.ToPlan(); - Assert.NotNull(plan); - Assert.NotNull(plan.Id); - Assert.Equal(goalText, plan.Goal); - Assert.True(plan.IsSuccessful); - Assert.True(plan.IsComplete); - Assert.Equal("Echo Result: Hello World", plan.Result, true); - } - - // Test that contains a #text node in the plan - [Theory] - [InlineData("Test the functionFlowRunner", @"Test the functionFlowRunner - - -This is some text -")] - public async Task ExecutePlanCanCallFunctionWithTextAsync(string goalText, string planText) - { - // Arrange - var kernel = KernelBuilder.Create(); - var factory = new Mock>(); - kernel.Config.AddTextCompletionService("test", factory.Object); - var plannerSkill = kernel.ImportSkill(new PlannerSkill(kernel)); - _ = kernel.ImportSkill(new MockSkill(this._testOutputHelper), "MockSkill"); - SkillPlan createdPlan = new() - { - Goal = goalText, - PlanString = planText - }; - - // Act - var context = await kernel.RunAsync(createdPlan.ToJson(), plannerSkill["ExecutePlan"]); - - // Assert - var plan = context.Variables.ToPlan(); - Assert.NotNull(plan); - Assert.NotNull(plan.Id); - Assert.Equal(goalText, plan.Goal); - Assert.True(plan.IsSuccessful); - Assert.True(plan.IsComplete); - Assert.Equal("Echo Result: Hello World", plan.Result, true); - } - - // use SplitInput after Echo - [Theory] - [InlineData("Test the functionFlowRunner", @"Test the functionFlowRunner - - - - - -")] - public async Task ExecutePlanCanCallFunctionWithVariablesAsync(string goalText, string planText) - { - // Arrange - var kernel = KernelBuilder.Create(); - var factory = new Mock>(); - kernel.Config.AddTextCompletionService("test", factory.Object); - var plannerSkill = kernel.ImportSkill(new PlannerSkill(kernel)); - _ = kernel.ImportSkill(new MockSkill(this._testOutputHelper), "MockSkill"); - SkillPlan createdPlan = new() - { - Goal = goalText, - PlanString = planText - }; - - // Act - run the plan 4 times to run all steps - var context = await kernel.RunAsync(createdPlan.ToJson(), plannerSkill["ExecutePlan"]); - context = await kernel.RunAsync(context.Variables, plannerSkill["ExecutePlan"]); - context = await kernel.RunAsync(context.Variables, plannerSkill["ExecutePlan"]); - context = await kernel.RunAsync(context.Variables, plannerSkill["ExecutePlan"]); - - // Assert - var plan = context.Variables.ToPlan(); - Assert.NotNull(plan); - Assert.NotNull(plan.Id); - Assert.Equal(goalText, plan.Goal); - Assert.True(plan.IsSuccessful); - Assert.True(plan.IsComplete); - Assert.Equal("Echo Result: Echo Result", plan.Result, true); - } - - [Theory] - [InlineData("Test the functionFlowRunner", @"Test the functionFlowRunner - - - - - - - -")] - public async Task ExecutePlanCanCallFunctionWithVariablesAndResultAsync(string goalText, string planText) - { - // Arrange - var kernel = KernelBuilder.Create(); - var factory = new Mock>(); - kernel.Config.AddTextCompletionService("test", factory.Object); - var plannerSkill = kernel.ImportSkill(new PlannerSkill(kernel)); - _ = kernel.ImportSkill(new MockSkill(this._testOutputHelper), "MockSkill"); - SkillPlan createdPlan = new() - { - Goal = goalText, - PlanString = planText - }; - - // Act - run the plan 6 times to run all steps - var context = await kernel.RunAsync(createdPlan.ToJson(), plannerSkill["ExecutePlan"]); - context = await kernel.RunAsync(context.Variables, plannerSkill["ExecutePlan"]); - context = await kernel.RunAsync(context.Variables, plannerSkill["ExecutePlan"]); - context = await kernel.RunAsync(context.Variables, plannerSkill["ExecutePlan"]); - context = await kernel.RunAsync(context.Variables, plannerSkill["ExecutePlan"]); - context = await kernel.RunAsync(context.Variables, plannerSkill["ExecutePlan"]); - - // Assert - var plan = context.Variables.ToPlan(); - Assert.NotNull(plan); - Assert.NotNull(plan.Id); - Assert.Equal(goalText, plan.Goal); - Assert.True(plan.IsSuccessful); - Assert.True(plan.IsComplete); - var separator = Environment.NewLine + Environment.NewLine; - Assert.Equal($"RESULT__1{separator}Echo Result: Echo Result: Echo Result{separator}RESULT__2{separator}Echo Result: Echo Result: Hello World", - plan.Result, true); - } - - [Theory] - [InlineData("Test the functionFlowRunner", @"Test the functionFlowRunner - - - - - - -")] - public async Task ExecutePlanCanCallFunctionWithChainedVariablesAsync(string goalText, string planText) - { - // Arrange - var kernel = KernelBuilder.Create(); - var factory = new Mock>(); - kernel.Config.AddTextCompletionService("test", factory.Object); - var plannerSkill = kernel.ImportSkill(new PlannerSkill(kernel)); - _ = kernel.ImportSkill(new MockSkill(this._testOutputHelper), "MockSkill"); - SkillPlan createdPlan = new() - { - Goal = goalText, - PlanString = planText - }; - - // Act - run the plan 5 times to run all steps - var context = await kernel.RunAsync(createdPlan.ToJson(), plannerSkill["ExecutePlan"]); - context = await kernel.RunAsync(context.Variables, plannerSkill["ExecutePlan"]); - context = await kernel.RunAsync(context.Variables, plannerSkill["ExecutePlan"]); - context = await kernel.RunAsync(context.Variables, plannerSkill["ExecutePlan"]); - context = await kernel.RunAsync(context.Variables, plannerSkill["ExecutePlan"]); - - // Assert - var plan = context.Variables.ToPlan(); - Assert.NotNull(plan); - Assert.NotNull(plan.Id); - Assert.Equal(goalText, plan.Goal); - Assert.True(plan.IsSuccessful); - Assert.True(plan.IsComplete); - Assert.Equal("Echo Result: Echo Result: Hello WorldEcho Result: Echo Result", plan.Result, true); - } - - // test that a that is not will just get skipped - [Theory] - [InlineData("Test the functionFlowRunner", @"Test the functionFlowRunner - - -Some other tag - -")] - public async Task ExecutePlanCanSkipTagsAsync(string goalText, string planText) - { - // Arrange - var kernel = KernelBuilder.Create(); - var factory = new Mock>(); - kernel.Config.AddTextCompletionService("test", factory.Object); - var plannerSkill = kernel.ImportSkill(new PlannerSkill(kernel)); - _ = kernel.ImportSkill(new MockSkill(this._testOutputHelper), "MockSkill"); - SkillPlan createdPlan = new() - { - Goal = goalText, - PlanString = planText - }; - - // Act - run the plan 2 times to run all steps - var context = await kernel.RunAsync(createdPlan.ToJson(), plannerSkill["ExecutePlan"]); - context = await kernel.RunAsync(context.Variables, plannerSkill["ExecutePlan"]); - - // Assert - var plan = context.Variables.ToPlan(); - Assert.NotNull(plan); - Assert.NotNull(plan.Id); - Assert.Equal(goalText, plan.Goal); - Assert.True(plan.IsSuccessful); - Assert.True(plan.IsComplete); - Assert.Equal("Echo Result: Echo Result: Hello World", plan.Result, true); - } - - [Theory] - [InlineData("Test the functionFlowRunner", @"Test the functionFlowRunner - - - -")] - public async Task ExecutePlanCanSkipOutputAsync(string goalText, string planText) - { - // Arrange - var kernel = KernelBuilder.Create(); - var factory = new Mock>(); - kernel.Config.AddTextCompletionService("test", factory.Object); - var plannerSkill = kernel.ImportSkill(new PlannerSkill(kernel)); - _ = kernel.ImportSkill(new MockSkill(this._testOutputHelper), "MockSkill"); - SkillPlan createdPlan = new() - { - Goal = goalText, - PlanString = planText - }; - - // Act - run the plan 2 times to run all steps - var context = await kernel.RunAsync(createdPlan.ToJson(), plannerSkill["ExecutePlan"]); - context = await kernel.RunAsync(context.Variables, plannerSkill["ExecutePlan"]); - - // Assert - var plan = context.Variables.ToPlan(); - Assert.NotNull(plan); - Assert.NotNull(plan.Id); - Assert.Equal(goalText, plan.Goal); - Assert.True(plan.IsSuccessful); - Assert.True(plan.IsComplete); - Assert.Equal("Echo Result: Echo Result: Hello World", plan.Result, true); - } - - public class MockSkill - { - private readonly ITestOutputHelper _testOutputHelper; - - public MockSkill(ITestOutputHelper testOutputHelper) - { - this._testOutputHelper = testOutputHelper; - } - - [SKFunction("Split the input into two parts")] - [SKFunctionName("SplitInput")] - [SKFunctionInput(Description = "The input text to split")] - public Task SplitInputAsync(string input, SKContext context) - { - var parts = input.Split(':'); - context.Variables.Set("First", parts[0]); - context.Variables.Set("Second", parts[1]); - return Task.FromResult(context); - } - - [SKFunction("Echo the input text")] - [SKFunctionName("Echo")] - public Task EchoAsync(string text, SKContext context) - { - this._testOutputHelper.WriteLine(text); - _ = context.Variables.Update("Echo Result: " + text); - return Task.FromResult(context); - } - } -} diff --git a/dotnet/src/SemanticKernel.UnitTests/Orchestration/PlanTests.cs b/dotnet/src/SemanticKernel.UnitTests/Orchestration/PlanTests.cs index 22e31607f437..4464f28313bb 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Orchestration/PlanTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Orchestration/PlanTests.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -316,6 +317,8 @@ public async Task CanStepPlanWithStepsAndContextAsync() returnContext.Variables.Update(returnContext.Variables.Input + c.Variables.Input + v); }) .Returns(() => Task.FromResult(returnContext)); + mockFunction.Setup(x => x.Describe()).Returns(new FunctionView() + { Parameters = new List() { new ParameterView() { Name = "variables" } } }); plan.AddSteps(mockFunction.Object, mockFunction.Object); @@ -330,6 +333,7 @@ public async Task CanStepPlanWithStepsAndContextAsync() // Act cv.Set("variables", "bar"); + cv.Update(string.Empty); plan = await kernel.Object.StepAsync(cv, plan); // Assert @@ -454,7 +458,8 @@ public async Task CanExecutePanWithTreeStepsAsync() .Returns(() => Task.FromResult(returnContext)); subPlan.AddSteps(childFunction1.Object, childFunction2.Object, childFunction3.Object); - plan.AddSteps(subPlan, nodeFunction1.Object); + plan.AddSteps(subPlan); + plan.AddSteps(nodeFunction1.Object); // Act while (plan.HasNextStep) @@ -464,7 +469,7 @@ public async Task CanExecutePanWithTreeStepsAsync() // Assert Assert.NotNull(plan); - Assert.Equal($"Child 3 heard Child 2 is happy about Child 1 output! - this just happened.", plan.State.ToString()); + Assert.Equal($"Child 3 heard Child 2 is happy about Child 1 output!Write a poem or joke - this just happened.", plan.State.ToString()); nodeFunction1.Verify(x => x.InvokeAsync(It.IsAny(), null, null, null), Times.Once); childFunction1.Verify(x => x.InvokeAsync(It.IsAny(), null, null, null), Times.Once); childFunction2.Verify(x => x.InvokeAsync(It.IsAny(), null, null, null), Times.Once); diff --git a/dotnet/src/SemanticKernel.UnitTests/Orchestration/PlanVariableExpansionTests.cs b/dotnet/src/SemanticKernel.UnitTests/Orchestration/PlanVariableExpansionTests.cs new file mode 100644 index 000000000000..a8f398ff4c0a --- /dev/null +++ b/dotnet/src/SemanticKernel.UnitTests/Orchestration/PlanVariableExpansionTests.cs @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.SemanticKernel.Orchestration; +using Xunit; + +namespace SemanticKernel.UnitTests.Orchestration; + +public sealed class PlanVariableExpansionTests +{ + [Fact] + public void ExpandFromVariablesWithNoVariablesReturnsInput() + { + // Arrange + var input = "Hello world!"; + var variables = new ContextVariables(); + var plan = new Plan("This is my goal"); + + // Act + var result = plan.ExpandFromVariables(variables, input); + + // Assert + Assert.Equal(input, result); + } + + [Theory] + [InlineData("Hello $name! $greeting", "Hello Bob! How are you?", "name", "Bob", "greeting", "How are you?")] + [InlineData("$SOMETHING_ELSE;$SOMETHING_ELSE2", "The string;Another string", "SOMETHING_ELSE", "The string", "SOMETHING_ELSE2", "Another string")] + [InlineData("[$FirstName,$LastName,$Age]", "[John,Doe,35]", "FirstName", "John", "LastName", "Doe", "Age", "35")] + [InlineData("$Category ($Count)", "Fruits (3)", "Category", "Fruits", "Count", "3")] + [InlineData("$Animal eats $Food", "Dog eats Bones", "Animal", "Dog", "Food", "Bones")] + [InlineData("$Country is in $Continent", "Canada is in North America", "Country", "Canada", "Continent", "North America")] + [InlineData("Hello $name", "Hello world", "name", "world")] + [InlineData("$VAR1 $VAR2", "value1 value2", "VAR1", "value1", "VAR2", "value2")] + [InlineData("$A-$A-$A", "x-x-x", "A", "x")] + [InlineData("$A$B$A", "aba", "A", "a", "B", "b")] + [InlineData("$ABC", "", "A", "", "B", "", "C", "")] + [InlineData("$NO_VAR", "", "A", "a", "B", "b", "C", "c")] + [InlineData("$name$invalid_name", "world", "name", "world")] + public void ExpandFromVariablesWithVariablesReturnsExpandedString(string input, string expected, params string[] variables) + { + // Arrange + var contextVariables = new ContextVariables(); + for (var i = 0; i < variables.Length; i += 2) + { + contextVariables.Set(variables[i], variables[i + 1]); + } + + var plan = new Plan("This is my goal"); + + // Act + var result = plan.ExpandFromVariables(contextVariables, input); + + // Assert + Assert.Equal(expected, result); + } +} diff --git a/dotnet/src/SemanticKernel.UnitTests/Planning/ConditionalFlowHelperTests.cs b/dotnet/src/SemanticKernel.UnitTests/Planning/ConditionalFlowHelperTests.cs deleted file mode 100644 index 91e99b88b98d..000000000000 --- a/dotnet/src/SemanticKernel.UnitTests/Planning/ConditionalFlowHelperTests.cs +++ /dev/null @@ -1,549 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using System.Xml; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.AI.TextCompletion; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.Planning; -using Moq; -using Xunit; -using Xunit.Abstractions; - -namespace SemanticKernel.UnitTests.Planning; - -public class ConditionalFlowHelperTests -{ - private readonly ITestOutputHelper _testOutputHelper; - private const string ValidIfStructure = ""; - private const string ValidWhileStructure = ""; - - public ConditionalFlowHelperTests(ITestOutputHelper testOutputHelper) - { - this._testOutputHelper = testOutputHelper; - this._testOutputHelper.WriteLine("Tests initialized"); - } - - [Fact] - public void ItCanBeInstantiated() - { - // Arrange - var kernel = KernelBuilder.Create(); - _ = kernel.Config.AddOpenAITextCompletionService("test", "test", "test"); - var completionBackendMock = new Mock(); - - // Act - Assert no exception occurs - _ = new ConditionalFlowHelper(kernel); - _ = new ConditionalFlowHelper(kernel, completionBackendMock.Object); - } - - [Fact] - public async Task IfAsyncCanRunIfAsync() - { - // Arrange - var kernel = KernelBuilder.Create(); - _ = kernel.Config.AddOpenAITextCompletionService("test", "test", "test"); - var completionBackendMock = SetupCompletionBackendMock(new Dictionary - { - { ConditionalFlowConstants.IfStructureCheckPrompt[..30], "{\"valid\": true}" }, - { ConditionalFlowConstants.EvaluateConditionPrompt[..30], "{\"valid\": true, \"condition\": true}" }, - }); - - var target = new ConditionalFlowHelper(kernel, completionBackendMock.Object); - - // Act - var resultingBranch = await target.IfAsync(ValidIfStructure, this.CreateSKContext(kernel)); - - // Assert - Assert.NotNull(resultingBranch); - Assert.Equal("", resultingBranch); - } - - [Theory] - [InlineData("{\"valid\": true}")] - [InlineData("{\"valid\": true, \"condition\": null}")] - public async Task IfAsyncInvalidConditionJsonPropertyShouldFailAsync(string llmConditionResult) - { - // Arrange - var kernel = KernelBuilder.Create(); - _ = kernel.Config.AddOpenAITextCompletionService("test", "test", "test"); - var completionBackendMock = SetupCompletionBackendMock(new Dictionary - { - { ConditionalFlowConstants.IfStructureCheckPrompt[..30], "{\"valid\": true}" }, - { ConditionalFlowConstants.EvaluateConditionPrompt[..30], llmConditionResult }, - }); - - var target = new ConditionalFlowHelper(kernel, completionBackendMock.Object); - - // Act - var exception = await Assert.ThrowsAsync(async () => - { - await target.IfAsync(ValidIfStructure, this.CreateSKContext(kernel)); - }); - - // Assert - Assert.NotNull(exception); - Assert.Equal(ConditionException.ErrorCodes.InvalidResponse, exception.ErrorCode); - } - - [Fact] - public async Task IfAsyncInvalidIfStatementWithoutConditionShouldFailAsync() - { - // Arrange - var kernel = KernelBuilder.Create(); - _ = kernel.Config.AddOpenAITextCompletionService("test", "test", "test"); - - // To be able to check the condition need ensure success in the if statement check first - var completionBackendMock = SetupCompletionBackendMock(new Dictionary - { - { ConditionalFlowConstants.IfStructureCheckPrompt[..30], "{\"valid\": true}" }, - }); - - var target = new ConditionalFlowHelper(kernel, completionBackendMock.Object); - - // Act - var exception = await Assert.ThrowsAsync(async () => - { - await target.IfAsync("", this.CreateSKContext(kernel)); - }); - - // Assert - Assert.Equal(ConditionException.ErrorCodes.InvalidCondition, exception.ErrorCode); - } - - [Theory] - [InlineData("")] - [InlineData("Unexpected Result")] - public async Task IfAsyncInvalidJsonIfStatementCheckResponseShouldFailContextAsync(string llmResult) - { - // Arrange - var kernel = KernelBuilder.Create(); - _ = kernel.Config.AddOpenAITextCompletionService("test", "test", "test"); - var completionBackendMock = SetupCompletionBackendMock(new Dictionary - { - { ConditionalFlowConstants.IfStructureCheckPrompt[..30], llmResult } - }); - var target = new ConditionalFlowHelper(kernel, completionBackendMock.Object); - - // Act - var exception = await Assert.ThrowsAsync(async () => - { - await target.IfAsync(ValidIfStructure, this.CreateSKContext(kernel)); - }); - - // Assert - Assert.Equal(ConditionException.ErrorCodes.InvalidResponse, exception.ErrorCode); - } - - [Theory] - [InlineData("TRUE")] - [InlineData("")] - [InlineData("")] - [InlineData("")] - [InlineData("")] - [InlineData("")] - public async Task IfAsyncInvalidIfStatementXmlShouldFailAsync(string ifContentInput) - { - // Arrange - var kernel = KernelBuilder.Create(); - _ = kernel.Config.AddOpenAITextCompletionService("test", "test", "test"); - - // To be able to check the condition need ensure success in the if statement check first - var completionBackendMock = SetupCompletionBackendMock(new Dictionary - { - { ConditionalFlowConstants.IfStructureCheckPrompt[..30], "{\"valid\": true}" }, - }); - - var target = new ConditionalFlowHelper(kernel, completionBackendMock.Object); - - // Act - var exception = await Assert.ThrowsAsync(async () => - { - await target.IfAsync(ifContentInput, this.CreateSKContext(kernel)); - }); - - // Assert - Assert.NotNull(exception); - } - - [Theory] - [InlineData("TRUE")] - [InlineData("")] - [InlineData("")] - [InlineData("")] - [InlineData("")] - [InlineData("")] - public async Task WhileAsyncInvalidStatementXmlShouldFailAsync(string whileContentInput) - { - // Arrange - var kernel = KernelBuilder.Create(); - _ = kernel.Config.AddOpenAITextCompletionService("test", "test", "test"); - - var target = new ConditionalFlowHelper(kernel); - - // Act - var exception = await Assert.ThrowsAsync(async () => - { - await target.WhileAsync(whileContentInput, this.CreateSKContext(kernel)); - }); - - // Assert - Assert.NotNull(exception); - } - - [Theory] - [InlineData("")] - [InlineData("")] - [InlineData("")] - [InlineData("")] - [InlineData("")] - [InlineData("")] - [InlineData("")] - public async Task IfAsyncInvalidIfStatementStructureShouldFailAsync(string ifContentInput) - { - // Arrange - var kernel = KernelBuilder.Create(); - _ = kernel.Config.AddOpenAITextCompletionService("test", "test", "test"); - var target = new ConditionalFlowHelper(kernel); - - // Act - var exception = await Assert.ThrowsAsync(async () => - { - await target.IfAsync(ifContentInput, this.CreateSKContext(kernel)); - }); - - // Assert - Assert.Equal(ConditionException.ErrorCodes.InvalidStatementStructure, exception.ErrorCode); - } - - [Theory] - [InlineData("")] - [InlineData("")] - [InlineData("")] - [InlineData("")] - [InlineData("")] - public async Task WhileAsyncInvalidWhileStatementStructureShouldFailAsync(string whileContentInput) - { - // Arrange - var kernel = KernelBuilder.Create(); - _ = kernel.Config.AddOpenAITextCompletionService("test", "test", "test"); - var target = new ConditionalFlowHelper(kernel); - - // Act - var exception = await Assert.ThrowsAsync(async () => - { - await target.WhileAsync(whileContentInput, this.CreateSKContext(kernel)); - }); - - // Assert - Assert.Equal(ConditionException.ErrorCodes.InvalidStatementStructure, exception.ErrorCode); - } - - [Theory(Skip = "LLM IfStatementCheck is disabled")] - [InlineData("{\"valid\": false, \"reason\": null}", ConditionalFlowHelper.NoReasonMessage)] - [InlineData("{\"valid\": false, \"reason\": \"\"}", ConditionalFlowHelper.NoReasonMessage)] - [InlineData("{\"valid\": false, \"reason\": \"Something1 Error\"}", "Something1 Error")] - [InlineData("{\"valid\": false, \n\"reason\": \"Something2 Error\"}", "Something2 Error")] - [InlineData("{\"valid\": false, \n\n\"reason\": \"Something3 Error\"}", "Something3 Error")] - public async Task IfAsyncInvalidIfStatementCheckResponseShouldFailContextWithReasonMessageAsync(string llmResult, string expectedReason) - { - // Arrange - var kernel = KernelBuilder.Create(); - _ = kernel.Config.AddOpenAITextCompletionService("test", "test", "test"); - var completionBackendMock = SetupCompletionBackendMock(new Dictionary - { - { ConditionalFlowConstants.IfStructureCheckPrompt[..30], llmResult }, - }); - var target = new ConditionalFlowHelper(kernel, completionBackendMock.Object); - - // Act - var exception = await Assert.ThrowsAsync(async () => - { - await target.IfAsync(ValidIfStructure, this.CreateSKContext(kernel)); - }); - - // Assert - Assert.Equal(ConditionException.ErrorCodes.InvalidStatementStructure, exception.ErrorCode); - Assert.Equal($"{nameof(ConditionException.ErrorCodes.InvalidStatementStructure)}: {expectedReason}", exception.Message); - } - - [Theory] - [InlineData("{\"valid\": false }", ConditionalFlowHelper.NoReasonMessage)] - [InlineData("{\"valid\": false \n}", ConditionalFlowHelper.NoReasonMessage)] - [InlineData("{\"valid\": false, \n \"reason\":\"\" }", ConditionalFlowHelper.NoReasonMessage)] - [InlineData("{\"valid\": false, \n\n\"reason\": \"Something1 Error\"}", "Something1 Error")] - [InlineData("{\"valid\": false, \n\n\"reason\": \"Something2 Error\"}", "Something2 Error")] - [InlineData("{\"valid\": false, \n\n\"reason\": \"Something3 Error\"}", "Something3 Error")] - public async Task IfAsyncInvalidEvaluateConditionResponseShouldFailContextWithReasonMessageAsync(string llmResult, string expectedReason) - { - // Arrange - var kernel = KernelBuilder.Create(); - _ = kernel.Config.AddOpenAITextCompletionService("test", "test", "test"); - var ifContent = ValidIfStructure; - var completionBackendMock = SetupCompletionBackendMock(new Dictionary - { - { ConditionalFlowConstants.IfStructureCheckPrompt[..30], "{\"valid\": true}" }, - { ConditionalFlowConstants.EvaluateConditionPrompt[..30], llmResult }, - }); - - var target = new ConditionalFlowHelper(kernel, completionBackendMock.Object); - - // Act - var exception = await Assert.ThrowsAsync(async () => - { - await target.IfAsync(ifContent, this.CreateSKContext(kernel)); - }); - - // Assert - Assert.Equal(ConditionException.ErrorCodes.InvalidCondition, exception.ErrorCode); - Assert.Equal($"{nameof(ConditionException.ErrorCodes.InvalidCondition)}: {expectedReason}", exception.Message); - } - - [Theory] - [InlineData("{\"valid\": true, \"condition\": true}", "", "")] - [InlineData("{\"valid\": true, \"condition\": false}", "", "")] - [InlineData("{\"valid\": true, \"condition\": true}", "", "")] - [InlineData("{\"valid\": true, \"condition\": false}", "", "")] - public async Task IfAsyncValidEvaluateConditionResponseShouldReturnAsync(string llmResult, string inputIfStructure, string expectedResult) - { - // Arrange - var kernel = KernelBuilder.Create(); - _ = kernel.Config.AddOpenAITextCompletionService("test", "test", "test"); - var completionBackendMock = SetupCompletionBackendMock(new Dictionary - { - { ConditionalFlowConstants.IfStructureCheckPrompt[..30], "{\"valid\": true}" }, - { ConditionalFlowConstants.EvaluateConditionPrompt[..30], llmResult }, - }); - - var target = new ConditionalFlowHelper(kernel, completionBackendMock.Object); - - // Act - var result = await target.IfAsync(inputIfStructure, this.CreateSKContext(kernel)); - - // Assert - Assert.NotNull(result); - Assert.Equal(expectedResult, result); - } - - [Theory] - [InlineData("Variable1,Variable2", "Variable1")] - [InlineData("Variable1,Variable2", "a")] - [InlineData("Variable1,Variable2,Variable3", "Variable2")] - public async Task IfAsyncEvaluateShouldUseExistingConditionVariablesOnlyAsync(string existingVariables, string conditionVariables) - { - // Arrange - var kernel = KernelBuilder.Create(); - _ = kernel.Config.AddOpenAITextCompletionService("test", "test", "test"); - var fakeConditionVariables = string.Join(" AND ", conditionVariables.Split(",").Select(x => $"${x} equals 0")); - var ifContent = ValidIfStructure.Replace("$a equals 0", fakeConditionVariables, StringComparison.OrdinalIgnoreCase); - - var completionBackendMock = SetupCompletionBackendMock(new Dictionary - { - { - ConditionalFlowConstants.IfStructureCheckPrompt[..30], - $"{{\"valid\": true, \"variables\": [\"{string.Join("\",\"", conditionVariables.Split(','))}\"]}}" - }, - { ConditionalFlowConstants.EvaluateConditionPrompt[..30], "{ \"valid\": true, \"condition\": true }" }, - }); - - var target = new ConditionalFlowHelper(kernel, completionBackendMock.Object); - ContextVariables contextVariables = CreateContextVariablesForTest(existingVariables, ifContent); - IEnumerable notExpectedVariablesInPrompt = GetNotExpectedVariablesInPromptForTest(existingVariables, conditionVariables); - - // Act - var context = this.CreateSKContext(kernel, contextVariables); - _ = await target.IfAsync(ifContent, context); - - // Assert - Assert.NotNull(context); - Assert.True(context.Variables.ContainsKey("ConditionalVariables")); - if (!string.IsNullOrEmpty(conditionVariables)) - { - foreach (var variableName in conditionVariables.Split(',')) - { - Assert.Contains(variableName, context.Variables["ConditionalVariables"], StringComparison.Ordinal); - } - } - - foreach (var variableName in notExpectedVariablesInPrompt) - { - Assert.DoesNotContain(variableName, context.Variables["ConditionalVariables"], StringComparison.Ordinal); - } - } - - [Theory] - [InlineData("Variable1,Variable2", "Variable4")] - [InlineData("Variable1,Variable2,Variable3", "Variable5,Variable2")] - public async Task IfAsyncEvaluateAsUndefinedWhenConditionVariablesDontExistsAsync(string existingVariables, string conditionVariables) - { - // Arrange - var kernel = KernelBuilder.Create(); - _ = kernel.Config.AddOpenAITextCompletionService("test", "test", "test"); - var fakeConditionVariables = string.Join(" AND ", conditionVariables.Split(",").Select(x => $"${x} equals 0")); - var ifContent = ValidIfStructure.Replace("$a equals 0", fakeConditionVariables, StringComparison.OrdinalIgnoreCase); - - var completionBackendMock = SetupCompletionBackendMock(new Dictionary - { - { - ConditionalFlowConstants.IfStructureCheckPrompt[..30], - $"{{\"valid\": true, \"variables\": [\"{string.Join("\",\"", conditionVariables.Split(','))}\"]}}" - }, - { ConditionalFlowConstants.EvaluateConditionPrompt[..30], "{ \"valid\": true, \"condition\": true }" }, - }); - - var target = new ConditionalFlowHelper(kernel, completionBackendMock.Object); - - ContextVariables contextVariables = CreateContextVariablesForTest(existingVariables, ifContent); - IEnumerable expectedUndefined = GetExpectedUndefinedForTest(existingVariables, conditionVariables); - SKContext context = this.CreateSKContext(kernel, contextVariables); - - // Act - await target.IfAsync(ifContent, context); - - // Assert - Assert.NotNull(context); - Assert.True(context.Variables.ContainsKey("ConditionalVariables")); - foreach (var variableName in expectedUndefined) - { - Assert.Contains($"{variableName} = undefined", context.Variables["ConditionalVariables"], StringComparison.Ordinal); - } - } - - [Theory] - [InlineData("Variable1,Variable2", "Variable1")] - [InlineData("Variable1,Variable2", "a")] - [InlineData("Variable1,Variable2,Variable3", "Variable2")] - public async Task WhileAsyncEvaluateShouldUseExistingConditionVariablesOnlyAsync(string existingVariables, string conditionVariables) - { - // Arrange - var kernel = KernelBuilder.Create(); - _ = kernel.Config.AddOpenAITextCompletionService("test", "test", "test"); - var fakeConditionVariables = string.Join(" AND ", conditionVariables.Split(",").Select(x => $"${x} equals 0")); - var ifContent = ValidWhileStructure.Replace("$a equals 0", fakeConditionVariables, StringComparison.OrdinalIgnoreCase); - - var completionBackendMock = SetupCompletionBackendMock(new Dictionary - { - { ConditionalFlowConstants.EvaluateConditionPrompt[..30], "{ \"valid\": true, \"condition\": true }" }, - }); - - var target = new ConditionalFlowHelper(kernel, completionBackendMock.Object); - ContextVariables contextVariables = CreateContextVariablesForTest(existingVariables, ifContent); - IEnumerable notExpectedVariablesInPrompt = GetNotExpectedVariablesInPromptForTest(existingVariables, conditionVariables); - - // Act - var context = this.CreateSKContext(kernel, contextVariables); - _ = await target.WhileAsync(ifContent, context); - - // Assert - Assert.NotNull(context); - Assert.True(context.Variables.ContainsKey("ConditionalVariables")); - if (!string.IsNullOrEmpty(conditionVariables)) - { - foreach (var variableName in conditionVariables.Split(',')) - { - Assert.Contains(variableName, context.Variables["ConditionalVariables"], StringComparison.Ordinal); - } - } - - foreach (var variableName in notExpectedVariablesInPrompt) - { - Assert.DoesNotContain(variableName, context.Variables["ConditionalVariables"], StringComparison.Ordinal); - } - } - - [Theory] - [InlineData("Variable1,Variable2", "Variable4")] - [InlineData("Variable1,Variable2,Variable3", "Variable5,Variable2")] - public async Task WhileAsyncEvaluateAsUndefinedWhenConditionVariablesDontExistsAsync(string existingVariables, string conditionVariables) - { - // Arrange - var kernel = KernelBuilder.Create(); - _ = kernel.Config.AddOpenAITextCompletionService("test", "test", "test"); - var fakeConditionVariables = string.Join(" AND ", conditionVariables.Split(",").Select(x => $"${x} equals 0")); - var ifContent = ValidWhileStructure.Replace("$a equals 0", fakeConditionVariables, StringComparison.OrdinalIgnoreCase); - - var completionBackendMock = SetupCompletionBackendMock(new Dictionary - { - { ConditionalFlowConstants.EvaluateConditionPrompt[..30], "{ \"valid\": true, \"condition\": true }" }, - }); - - var target = new ConditionalFlowHelper(kernel, completionBackendMock.Object); - - ContextVariables contextVariables = CreateContextVariablesForTest(existingVariables, ifContent); - IEnumerable expectedUndefined = GetExpectedUndefinedForTest(existingVariables, conditionVariables); - SKContext context = this.CreateSKContext(kernel, contextVariables); - - // Act - await target.WhileAsync(ifContent, context); - - // Assert - Assert.NotNull(context); - Assert.True(context.Variables.ContainsKey("ConditionalVariables")); - foreach (var variableName in expectedUndefined) - { - Assert.Contains($"{variableName} = undefined", context.Variables["ConditionalVariables"], StringComparison.Ordinal); - } - } - - private static IEnumerable GetExpectedUndefinedForTest(string existingVariables, string conditionVariables) - { - IEnumerable expectedUndefined = Enumerable.Empty(); - if (!string.IsNullOrEmpty(conditionVariables)) - { - expectedUndefined = conditionVariables.Split(',').Except(existingVariables?.Split(",") ?? Array.Empty()); - } - - return expectedUndefined; - } - - private SKContext CreateSKContext(IKernel kernel, ContextVariables? variables = null, CancellationToken cancellationToken = default) - { - return new SKContext(variables ?? new ContextVariables(), kernel.Memory, kernel.Skills, kernel.Log, cancellationToken); - } - - private static Mock SetupCompletionBackendMock(Dictionary promptsAndResponses) - { - var completionBackendMock = new Mock(); - - // For each prompt and response pair, setup the mock to return the response when the prompt is passed as an argument - foreach (var pair in promptsAndResponses) - { - completionBackendMock.Setup(a => a.CompleteAsync( - It.Is(prompt => prompt.Contains(pair.Key)), // Match the prompt by checking if it contains the expected substring - It.IsAny(), // Ignore the settings parameter - It.IsAny() // Ignore the cancellation token parameter - )).ReturnsAsync(pair.Value); // Return the expected response - } - - return completionBackendMock; - } - - private static IEnumerable GetNotExpectedVariablesInPromptForTest(string existingVariables, string conditionVariables) - { - IEnumerable notExpectedVariablesInPrompt = existingVariables.Split(','); - if (!string.IsNullOrEmpty(conditionVariables)) - { - notExpectedVariablesInPrompt = notExpectedVariablesInPrompt.Except(conditionVariables.Split(',')); - } - - return notExpectedVariablesInPrompt; - } - - private static ContextVariables CreateContextVariablesForTest(string existingVariables, string ifContent) - { - var contextVariables = new ContextVariables(ifContent); - if (!string.IsNullOrEmpty(existingVariables)) - { - foreach (var variableName in existingVariables.Split(',')) - { - contextVariables.Set(variableName, "x"); - } - } - - return contextVariables; - } -} diff --git a/dotnet/src/SemanticKernel.UnitTests/Planning/FunctionFlowRunnerTests.cs b/dotnet/src/SemanticKernel.UnitTests/Planning/FunctionFlowRunnerTests.cs deleted file mode 100644 index 5eabe427f9e1..000000000000 --- a/dotnet/src/SemanticKernel.UnitTests/Planning/FunctionFlowRunnerTests.cs +++ /dev/null @@ -1,490 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Text.RegularExpressions; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.AI.TextCompletion; -using Microsoft.SemanticKernel.Memory; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.Planning; -using Microsoft.SemanticKernel.SemanticFunctions; -using Microsoft.SemanticKernel.SkillDefinition; -using Moq; -using Xunit; -using Xunit.Abstractions; - -namespace SemanticKernel.UnitTests.Planning; - -public class FunctionFlowRunnerTests -{ - private readonly ITestOutputHelper _testOutputHelper; - - public FunctionFlowRunnerTests(ITestOutputHelper testOutputHelper) - { - this._testOutputHelper = testOutputHelper; - } - - [Fact] - public async Task ItExecuteXmlPlanAsyncValidEmptyPlanAsync() - { - // Arrange - var kernelMock = this.CreateKernelMock(out _, out _, out _); - var kernel = kernelMock.Object; - var target = new FunctionFlowRunner(kernel); - - var emptyPlanSpec = @" -Some goal - - -"; - - // Act - var result = await target.ExecuteXmlPlanAsync(this.CreateSKContext(kernel), emptyPlanSpec); - - // Assert - Assert.NotNull(result); - Assert.NotNull(result.Variables[SkillPlan.PlanKey]); - } - - [Theory] - [InlineData("Some goal")] - public async Task ItExecuteXmlPlanAsyncFailWhenInvalidPlanXmlAsync(string invalidPlanSpec) - { - // Arrange - var kernelMock = this.CreateKernelMock(out _, out _, out _); - var kernel = kernelMock.Object; - var target = new FunctionFlowRunner(kernel); - - // Act - var exception = await Assert.ThrowsAsync(async () => - { - await target.ExecuteXmlPlanAsync(this.CreateSKContext(kernel), invalidPlanSpec); - }); - - // Assert - Assert.NotNull(exception); - Assert.Equal(PlanningException.ErrorCodes.InvalidPlan, exception.ErrorCode); - } - - [Fact] - public async Task ItExecuteXmlPlanAsyncAndFailWhenSkillOrFunctionNotExistsAsync() - { - // Arrange - var kernelMock = this.CreateKernelMock(out _, out _, out _); - var kernel = kernelMock.Object; - var target = new FunctionFlowRunner(kernel); - - SKContext? result = null; - // Act - var exception = await Assert.ThrowsAsync(async () => - { - result = await target.ExecuteXmlPlanAsync(this.CreateSKContext(kernel), @" -Some goal - - -"); - }); - - // Assert - Assert.NotNull(exception); - Assert.Equal(PlanningException.ErrorCodes.InvalidPlan, exception.ErrorCode); - } - - [Fact] - public async Task ItExecuteXmlPlanAsyncFailWhenElseComesWithoutIfAsync() - { - // Arrange - var kernelMock = this.CreateKernelMock(out _, out _, out _); - var kernel = kernelMock.Object; - var target = new FunctionFlowRunner(kernel); - - SKContext? result = null; - // Act - var exception = await Assert.ThrowsAsync(async () => - { - result = await target.ExecuteXmlPlanAsync(this.CreateSKContext(kernel), @" -Some goal - - - - -"); - }); - - // Assert - Assert.NotNull(exception); - Assert.Equal(PlanningException.ErrorCodes.InvalidPlan, exception.ErrorCode); - } - - /// - /// Potential tests scenarios with Functions, Ifs and Else (Nested included) - /// - /// Plan input - /// Expected plan output - /// Condition result - /// Unit test result - private async Task ItExecuteXmlPlanAsyncAndReturnsAsExpectedAsync(string inputPlanSpec, string expectedPlanOutput, bool? conditionResult = null) - { - // Arrange - var kernelMock = this.CreateKernelMock(out _, out var skillMock, out _); - var kernel = kernelMock.Object; - var mockFunction = new Mock(); - - var ifStructureResultContext = this.CreateSKContext(kernel); - ifStructureResultContext.Variables.Update("{\"valid\": true}"); - - var evaluateConditionResultContext = this.CreateSKContext(kernel); - evaluateConditionResultContext.Variables.Update($"{{\"valid\": true, \"condition\": {((conditionResult ?? false) ? "true" : "false")}}}"); - - mockFunction.Setup(f => f.InvokeAsync(It.Is(i => i.StartsWith("(), It.IsAny(), - It.IsAny(), - It.IsAny())) - .ReturnsAsync(ifStructureResultContext); - - mockFunction.Setup(f => f.InvokeAsync(It.Is(i => i.StartsWith("$a equals b")), - It.IsAny(), It.IsAny(), - It.IsAny(), - It.IsAny())) - .ReturnsAsync(evaluateConditionResultContext); - - skillMock.Setup(s => s.HasSemanticFunction(It.IsAny(), It.IsAny())).Returns(true); - skillMock.Setup(s => s.GetSemanticFunction(It.IsAny(), It.IsAny())).Returns(mockFunction.Object); - kernelMock.Setup(k => k.RunAsync(It.IsAny(), It.IsAny())).ReturnsAsync(this.CreateSKContext(kernel)); - kernelMock.Setup(k => k.RegisterSemanticFunction(It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(mockFunction.Object); - - var target = new FunctionFlowRunner(kernel); - - SKContext? result = null; - // Act - result = await target.ExecuteXmlPlanAsync(this.CreateSKContext(kernel), inputPlanSpec); - - // Assert - Assert.NotNull(result); - Assert.False(result.ErrorOccurred); - Assert.True(result.Variables.ContainsKey(SkillPlan.PlanKey)); - Assert.Equal( - NormalizeSpacesBeforeFunctions(expectedPlanOutput), - NormalizeSpacesBeforeFunctions(result.Variables[SkillPlan.PlanKey])); - - // Removes line breaks and spaces before Some goal", - "Some goal")] - [InlineData( - "Some goal", - "Some goal")] - [InlineData( - "Some goal", - "Some goal ")] - public async Task ItExecuteXmlStepPlanAsyncAndReturnsAsExpectedAsync(string inputPlanSpec, string expectedPlanOutput, bool? conditionResult = null) - { - await this.ItExecuteXmlPlanAsyncAndReturnsAsExpectedAsync(inputPlanSpec, expectedPlanOutput, conditionResult); - } - - [Theory] - [InlineData( - @"Some goal - - - - - - -", - @"Some goal - - - - - -")] - [InlineData( - @"Some goal - - - - -", - @"Some goal - - -", true)] - [InlineData( - @"Some goal - - - - -", - @"Some goal - -", false)] - [InlineData( - @"Some goal - - - - - - - -", - @"Some goal - - -", false)] - [InlineData( - @"Some goal - - - - - - - - - - -", - @"Some goal - - - - - -", true)] - [InlineData( - @"Some goal - - - - - - - - - - -", - @"Some goal - - - - - -", false)] - [InlineData( - @"Some goal - - - - - - - - - - -", - @"Some goal - - - - - -", true)] - [InlineData( - @"Some goal - - - - - - - - - - -", - @"Some goal - - -", false)] - public async Task ItExecuteXmlIfPlanAsyncAndReturnsAsExpectedAsync(string inputPlanSpec, string expectedPlanOutput, bool? conditionResult = null) - { - await this.ItExecuteXmlPlanAsyncAndReturnsAsExpectedAsync(inputPlanSpec, expectedPlanOutput, conditionResult); - } - - [Theory] - [InlineData( - @"Some goal - - - - - - -", - @"Some goal - - - - - -")] - [InlineData( - @"Some goal - - - - -", - @"Some goal - - - - - -", true)] - [InlineData( - @"Some goal - - - - -", - @"Some goal - -", false)] - [InlineData( - @"Some goal - - - - - -", - @"Some goal - - -", false)] - [InlineData( - @"Some goal - - - - - - - - -", - @"Some goal - - - - - - - - - -", true)] - [InlineData( - @"Some goal - - - - - - - - -", - @"Some goal - - - - - -", false)] - [InlineData( - @"Some goal - - - - - - - -", - @"Some goal - - - - - - - - - - - -", true)] - [InlineData( - @"Some goal - - - - - - - - -", - @"Some goal - - -", false)] - public async Task ItExecuteXmlWhilePlanAsyncAndReturnsAsExpectedAsync(string inputPlanSpec, string expectedPlanOutput, bool? conditionResult = null) - { - await this.ItExecuteXmlPlanAsyncAndReturnsAsExpectedAsync(inputPlanSpec, expectedPlanOutput, conditionResult); - } - - private SKContext CreateSKContext( - IKernel kernel, - ContextVariables? variables = null, - CancellationToken cancellationToken = default) - { - return new SKContext(variables ?? new ContextVariables(), kernel.Memory, kernel.Skills, kernel.Log, cancellationToken); - } - - private Mock CreateKernelMock( - out Mock semanticMemoryMock, - out Mock mockSkillCollection, - out Mock mockLogger) - { - semanticMemoryMock = new Mock(); - mockSkillCollection = new Mock(); - mockLogger = new Mock(); - - var kernelMock = new Mock(); - kernelMock.SetupGet(k => k.Skills).Returns(mockSkillCollection.Object); - kernelMock.SetupGet(k => k.Log).Returns(mockLogger.Object); - kernelMock.SetupGet(k => k.Memory).Returns(semanticMemoryMock.Object); - - return kernelMock; - } -} diff --git a/dotnet/src/SemanticKernel.UnitTests/Planning/PlanningTests.cs b/dotnet/src/SemanticKernel.UnitTests/Planning/PlanningTests.cs new file mode 100644 index 000000000000..e9bf1fd640bf --- /dev/null +++ b/dotnet/src/SemanticKernel.UnitTests/Planning/PlanningTests.cs @@ -0,0 +1,157 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.AI.TextCompletion; +using Microsoft.SemanticKernel.Memory; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.Planning.Planners; +using Microsoft.SemanticKernel.SemanticFunctions; +using Microsoft.SemanticKernel.SkillDefinition; +using Moq; +using Xunit; + +namespace SemanticKernel.UnitTests.Planning; + +public sealed class PlanningTests +{ + [Theory] + [InlineData("Write a poem or joke and send it in an e-mail to Kai.")] + public async Task ItCanCreatePlanAsync(string goal) + { + // Arrange + var kernel = new Mock(); + kernel.Setup(x => x.Log).Returns(new Mock().Object); + + var memory = new Mock(); + + var input = new List<(string name, string skillName, string description, bool isSemantic)>() + { + ("SendEmail", "email", "Send an e-mail", false), + ("GetEmailAddress", "email", "Get an e-mail address", false), + ("Translate", "WriterSkill", "Translate something", true), + ("Summarize", "SummarizeSkill", "Summarize something", true) + }; + + var functionsView = new FunctionsView(); + var skills = new Mock(); + foreach (var (name, skillName, description, isSemantic) in input) + { + var functionView = new FunctionView(name, skillName, description, new List(), isSemantic, true); + var mockFunction = CreateMockFunction(functionView); + functionsView.AddFunction(functionView); + + mockFunction.Setup(x => + x.InvokeAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((context, settings, log, cancel) => + { + context.Variables.Update("MOCK FUNCTION CALLED"); + return Task.FromResult(context); + }); + + if (isSemantic) + { + skills.Setup(x => x.GetSemanticFunction(It.Is(s => s == skillName), It.Is(s => s == name))) + .Returns(mockFunction.Object); + skills.Setup(x => x.HasSemanticFunction(It.Is(s => s == skillName), It.Is(s => s == name))).Returns(true); + } + else + { + skills.Setup(x => x.GetNativeFunction(It.Is(s => s == skillName), It.Is(s => s == name))) + .Returns(mockFunction.Object); + skills.Setup(x => x.HasNativeFunction(It.Is(s => s == skillName), It.Is(s => s == name))).Returns(true); + } + } + + skills.Setup(x => x.GetFunctionsView(It.IsAny(), It.IsAny())).Returns(functionsView); + + var expectedFunctions = input.Select(x => x.name).ToList(); + var expectedSkills = input.Select(x => x.skillName).ToList(); + + var context = new SKContext( + new ContextVariables(), + memory.Object, + skills.Object, + new Mock().Object + ); + + var returnContext = new SKContext( + new ContextVariables(), + memory.Object, + skills.Object, + new Mock().Object + ); + var planString = + @" + + + + + +"; + + returnContext.Variables.Update(planString); + + var mockFunctionFlowFunction = new Mock(); + mockFunctionFlowFunction.Setup(x => x.InvokeAsync( + It.IsAny(), + null, + null, + null + )).Callback( + (c, s, l, ct) => c.Variables.Update("Hello world!") + ).Returns(() => Task.FromResult(returnContext)); + + // Mock Skills + kernel.Setup(x => x.Skills).Returns(skills.Object); + kernel.Setup(x => x.CreateNewContext()).Returns(context); + + kernel.Setup(x => x.RegisterSemanticFunction( + It.IsAny(), + It.IsAny(), + It.IsAny() + )).Returns(mockFunctionFlowFunction.Object); + + var planner = new SequentialPlanner(kernel.Object); + + // Act + var plan = await planner.CreatePlanAsync(goal); + + // Assert + Assert.Equal(goal, plan.Description); + + Assert.Contains( + plan.Steps, + step => + expectedFunctions.Contains(step.Name) && + expectedSkills.Contains(step.SkillName)); + + foreach (var expectedFunction in expectedFunctions) + { + Assert.Contains( + plan.Steps, + step => step.Name == expectedFunction); + } + + foreach (var expectedSkill in expectedSkills) + { + Assert.Contains( + plan.Steps, + step => step.SkillName == expectedSkill); + } + } + + // Method to create Mock objects + private static Mock CreateMockFunction(FunctionView functionView) + { + var mockFunction = new Mock(); + mockFunction.Setup(x => x.Describe()).Returns(functionView); + mockFunction.Setup(x => x.Name).Returns(functionView.Name); + mockFunction.Setup(x => x.SkillName).Returns(functionView.SkillName); + return mockFunction; + } +} diff --git a/dotnet/src/SemanticKernel.UnitTests/Planning/SKContextExtensionsTests.cs b/dotnet/src/SemanticKernel.UnitTests/Planning/SKContextExtensionsTests.cs index 46785318c6fa..6a39f651ce74 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Planning/SKContextExtensionsTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Planning/SKContextExtensionsTests.cs @@ -6,11 +6,11 @@ using System.Threading.Tasks; using Microsoft.SemanticKernel.Memory; using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.Planning.Planners; using Microsoft.SemanticKernel.SkillDefinition; using Moq; using SemanticKernel.UnitTests.XunitHelpers; using Xunit; -using static Microsoft.SemanticKernel.CoreSkills.PlannerSkill; namespace SemanticKernel.UnitTests.Planning; @@ -38,11 +38,11 @@ public async Task CanCallGetAvailableFunctionsWithNoFunctionsAsync() // Arrange GetAvailableFunctionsAsync parameters var context = new SKContext(variables, memory.Object, skills.ReadOnlySkillCollection, logger, cancellationToken); - var config = new PlannerSkillConfig(); + var config = new PlannerConfig(); var semanticQuery = "test"; // Act - var result = await context.GetAvailableFunctionsAsync(config, semanticQuery).ConfigureAwait(true); + var result = await context.GetAvailableFunctionsAsync(config, semanticQuery).ConfigureAwait(false); // Assert Assert.NotNull(result); @@ -91,11 +91,11 @@ public async Task CanCallGetAvailableFunctionsWithFunctionsAsync() // Arrange GetAvailableFunctionsAsync parameters var context = new SKContext(variables, memory.Object, skills.Object, logger, cancellationToken); - var config = new PlannerSkillConfig(); + var config = new PlannerConfig(); var semanticQuery = "test"; // Act - var result = (await context.GetAvailableFunctionsAsync(config, semanticQuery).ConfigureAwait(true)).ToList(); + var result = (await context.GetAvailableFunctionsAsync(config, semanticQuery).ConfigureAwait(false)).ToList(); // Assert Assert.NotNull(result); @@ -106,7 +106,7 @@ public async Task CanCallGetAvailableFunctionsWithFunctionsAsync() config.IncludedFunctions.UnionWith(new List { "nativeFunctionName" }); // Act - result = (await context.GetAvailableFunctionsAsync(config, semanticQuery).ConfigureAwait(true)).ToList(); + result = (await context.GetAvailableFunctionsAsync(config, semanticQuery).ConfigureAwait(false)).ToList(); // Assert Assert.NotNull(result); @@ -155,11 +155,11 @@ public async Task CanCallGetAvailableFunctionsWithFunctionsWithRelevancyAsync() // Arrange GetAvailableFunctionsAsync parameters var context = new SKContext(variables, memory.Object, skills.Object, logger, cancellationToken); - var config = new PlannerSkillConfig() { RelevancyThreshold = 0.78 }; + var config = new PlannerConfig() { RelevancyThreshold = 0.78 }; var semanticQuery = "test"; // Act - var result = (await context.GetAvailableFunctionsAsync(config, semanticQuery).ConfigureAwait(true)).ToList(); + var result = (await context.GetAvailableFunctionsAsync(config, semanticQuery).ConfigureAwait(false)).ToList(); // Assert Assert.NotNull(result); @@ -170,7 +170,7 @@ public async Task CanCallGetAvailableFunctionsWithFunctionsWithRelevancyAsync() config.IncludedFunctions.UnionWith(new List { "nativeFunctionName" }); // Act - result = (await context.GetAvailableFunctionsAsync(config, semanticQuery).ConfigureAwait(true)).ToList(); + result = (await context.GetAvailableFunctionsAsync(config, semanticQuery).ConfigureAwait(false)).ToList(); // Assert Assert.NotNull(result); @@ -179,159 +179,6 @@ public async Task CanCallGetAvailableFunctionsWithFunctionsWithRelevancyAsync() Assert.Equal(nativeFunctionView, result[1]); } - // Tests for GetPlannerSkillConfig - [Fact] - public void CanCallGetPlannerSkillConfig() - { - // Arrange - var variables = new ContextVariables(); - var logger = ConsoleLogger.Log; - var cancellationToken = default(CancellationToken); - var memory = new Mock(); - var skills = new Mock(); - var expectedDefault = new PlannerSkillConfig(); - - // Act - var context = new SKContext(variables, memory.Object, skills.Object, logger, cancellationToken); - var config = context.GetPlannerSkillConfig(); - - // Assert - Assert.NotNull(config); - Assert.Equal(expectedDefault.RelevancyThreshold, config.RelevancyThreshold); - Assert.Equal(expectedDefault.MaxRelevantFunctions, config.MaxRelevantFunctions); - Assert.Equal(expectedDefault.ExcludedFunctions, config.ExcludedFunctions); - Assert.Equal(expectedDefault.ExcludedSkills, config.ExcludedSkills); - Assert.Equal(expectedDefault.IncludedFunctions, config.IncludedFunctions); - } - - [Fact] - public void CanCallGetPlannerSkillConfigWithExcludedFunctions() - { - // Arrange - var variables = new ContextVariables(); - var logger = ConsoleLogger.Log; - var cancellationToken = default(CancellationToken); - var memory = new Mock(); - var skills = new Mock(); - var expectedDefault = new PlannerSkillConfig(); - var excludedFunctions = "test1,test2,test3"; - - // Act - variables.Set(Parameters.ExcludedFunctions, excludedFunctions); - var context = new SKContext(variables, memory.Object, skills.Object, logger, cancellationToken); - var config = context.GetPlannerSkillConfig(); - - // Assert - Assert.NotNull(config); - Assert.Equal(expectedDefault.RelevancyThreshold, config.RelevancyThreshold); - Assert.Equal(expectedDefault.MaxRelevantFunctions, config.MaxRelevantFunctions); - Assert.Equal(expectedDefault.ExcludedSkills, config.ExcludedSkills); - Assert.Equal(expectedDefault.IncludedFunctions, config.IncludedFunctions); - Assert.Equal(expectedDefault.ExcludedFunctions.Union(new HashSet { "test1", "test2", "test3" }), config.ExcludedFunctions); - } - - [Fact] - public void CanCallGetPlannerSkillConfigWithIncludedFunctions() - { - // Arrange - var variables = new ContextVariables(); - var logger = ConsoleLogger.Log; - var cancellationToken = default(CancellationToken); - var memory = new Mock(); - var skills = new Mock(); - var expectedDefault = new PlannerSkillConfig(); - var includedFunctions = "test1,CreatePlan"; - - // Act - variables.Set(Parameters.IncludedFunctions, includedFunctions); - var context = new SKContext(variables, memory.Object, skills.Object, logger, cancellationToken); - var config = context.GetPlannerSkillConfig(); - - // Assert - Assert.NotNull(config); - Assert.Equal(expectedDefault.RelevancyThreshold, config.RelevancyThreshold); - Assert.Equal(expectedDefault.MaxRelevantFunctions, config.MaxRelevantFunctions); - Assert.Equal(expectedDefault.ExcludedSkills, config.ExcludedSkills); - Assert.Equal(expectedDefault.ExcludedFunctions, config.ExcludedFunctions); - Assert.Equal(expectedDefault.IncludedFunctions.Union(new HashSet { "test1" }), config.IncludedFunctions); - } - - [Fact] - public void CanCallGetPlannerSkillConfigWithRelevancyThreshold() - { - // Arrange - var variables = new ContextVariables(); - var logger = ConsoleLogger.Log; - var cancellationToken = default(CancellationToken); - var memory = new Mock(); - var skills = new Mock(); - var expectedDefault = new PlannerSkillConfig(); - - // Act - variables.Set(Parameters.RelevancyThreshold, "0.78"); - var context = new SKContext(variables, memory.Object, skills.Object, logger, cancellationToken); - var config = context.GetPlannerSkillConfig(); - - // Assert - Assert.NotNull(config); - Assert.Equal(0.78, config.RelevancyThreshold); - Assert.Equal(expectedDefault.MaxRelevantFunctions, config.MaxRelevantFunctions); - Assert.Equal(expectedDefault.ExcludedSkills, config.ExcludedSkills); - Assert.Equal(expectedDefault.ExcludedFunctions, config.ExcludedFunctions); - Assert.Equal(expectedDefault.IncludedFunctions, config.IncludedFunctions); - } - - [Fact] - public void CanCallGetPlannerSkillConfigWithMaxRelevantFunctions() - { - // Arrange - var variables = new ContextVariables(); - var logger = ConsoleLogger.Log; - var cancellationToken = default(CancellationToken); - var memory = new Mock(); - var skills = new Mock(); - var expectedDefault = new PlannerSkillConfig(); - - // Act - variables.Set(Parameters.MaxRelevantFunctions, "5"); - var context = new SKContext(variables, memory.Object, skills.Object, logger, cancellationToken); - var config = context.GetPlannerSkillConfig(); - - // Assert - Assert.NotNull(config); - Assert.Equal(expectedDefault.RelevancyThreshold, config.RelevancyThreshold); - Assert.Equal(5, config.MaxRelevantFunctions); - Assert.Equal(expectedDefault.ExcludedSkills, config.ExcludedSkills); - Assert.Equal(expectedDefault.ExcludedFunctions, config.ExcludedFunctions); - Assert.Equal(expectedDefault.IncludedFunctions, config.IncludedFunctions); - } - - [Fact] - public void CanCallGetPlannerSkillConfigWithExcludedSkills() - { - // Arrange - var variables = new ContextVariables(); - var logger = ConsoleLogger.Log; - var cancellationToken = default(CancellationToken); - var memory = new Mock(); - var skills = new Mock(); - var expectedDefault = new PlannerSkillConfig(); - var excludedSkills = "test1,test2,test3"; - - // Act - variables.Set(Parameters.ExcludedSkills, excludedSkills); - var context = new SKContext(variables, memory.Object, skills.Object, logger, cancellationToken); - var config = context.GetPlannerSkillConfig(); - - // Assert - Assert.NotNull(config); - Assert.Equal(expectedDefault.RelevancyThreshold, config.RelevancyThreshold); - Assert.Equal(expectedDefault.MaxRelevantFunctions, config.MaxRelevantFunctions); - Assert.Equal(expectedDefault.ExcludedFunctions, config.ExcludedFunctions); - Assert.Equal(expectedDefault.IncludedFunctions, config.IncludedFunctions); - Assert.Equal(expectedDefault.ExcludedSkills.Union(new HashSet { "test1", "test2", "test3" }), config.ExcludedSkills); - } - [Fact] public async Task CanCallGetAvailableFunctionsAsyncWithDefaultRelevancyAsync() { @@ -355,11 +202,11 @@ public async Task CanCallGetAvailableFunctionsAsyncWithDefaultRelevancyAsync() // Arrange GetAvailableFunctionsAsync parameters var context = new SKContext(variables, memory.Object, skills.ReadOnlySkillCollection, logger, cancellationToken); - var config = new PlannerSkillConfig() { RelevancyThreshold = 0.78 }; + var config = new PlannerConfig() { RelevancyThreshold = 0.78 }; var semanticQuery = "test"; // Act - var result = await context.GetAvailableFunctionsAsync(config, semanticQuery).ConfigureAwait(true); + var result = await context.GetAvailableFunctionsAsync(config, semanticQuery).ConfigureAwait(false); // Assert Assert.NotNull(result); diff --git a/dotnet/src/SemanticKernel.UnitTests/Planning/SequentialPlanParserTests.cs b/dotnet/src/SemanticKernel.UnitTests/Planning/SequentialPlanParserTests.cs new file mode 100644 index 000000000000..8500d7707a44 --- /dev/null +++ b/dotnet/src/SemanticKernel.UnitTests/Planning/SequentialPlanParserTests.cs @@ -0,0 +1,245 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Threading; +using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Memory; +using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.Planning; +using Microsoft.SemanticKernel.SemanticFunctions; +using Microsoft.SemanticKernel.SkillDefinition; +using Moq; +using Xunit; +using Xunit.Abstractions; + +namespace SemanticKernel.UnitTests.Planning; + +public class SequentialPlanParserTests +{ + private readonly ITestOutputHelper _testOutputHelper; + + public SequentialPlanParserTests(ITestOutputHelper testOutputHelper) + { + this._testOutputHelper = testOutputHelper; + } + + private Mock CreateKernelMock( + out Mock semanticMemoryMock, + out Mock mockSkillCollection, + out Mock mockLogger) + { + semanticMemoryMock = new Mock(); + mockSkillCollection = new Mock(); + mockLogger = new Mock(); + + var kernelMock = new Mock(); + kernelMock.SetupGet(k => k.Skills).Returns(mockSkillCollection.Object); + kernelMock.SetupGet(k => k.Log).Returns(mockLogger.Object); + kernelMock.SetupGet(k => k.Memory).Returns(semanticMemoryMock.Object); + + return kernelMock; + } + + private SKContext CreateSKContext( + IKernel kernel, + ContextVariables? variables = null, + CancellationToken cancellationToken = default) + { + return new SKContext(variables ?? new ContextVariables(), kernel.Memory, kernel.Skills, kernel.Log, cancellationToken); + } + + private static Mock CreateMockFunction(FunctionView functionView, string result = "") + { + var mockFunction = new Mock(); + mockFunction.Setup(x => x.Describe()).Returns(functionView); + mockFunction.Setup(x => x.Name).Returns(functionView.Name); + mockFunction.Setup(x => x.SkillName).Returns(functionView.SkillName); + return mockFunction; + } + + private void CreateKernelAndFunctionCreateMocks(List<(string name, string skillName, string description, bool isSemantic, string result)> functions, + out IKernel kernel) + { + var kernelMock = this.CreateKernelMock(out _, out var skills, out _); + kernel = kernelMock.Object; + + // For Create + kernelMock.Setup(k => k.CreateNewContext()).Returns(this.CreateSKContext(kernel)); + + var functionsView = new FunctionsView(); + foreach (var (name, skillName, description, isSemantic, resultString) in functions) + { + var functionView = new FunctionView(name, skillName, description, new List(), isSemantic, true); + var mockFunction = CreateMockFunction(functionView); + functionsView.AddFunction(functionView); + + var result = this.CreateSKContext(kernel); + result.Variables.Update(resultString); + mockFunction.Setup(x => x.InvokeAsync(It.IsAny(), null, null, null)) + .ReturnsAsync(result); + + if (string.IsNullOrEmpty(name)) + { + kernelMock.Setup(x => x.RegisterSemanticFunction( + It.IsAny(), + It.IsAny(), + It.IsAny() + )).Returns(mockFunction.Object); + } + else + { + if (isSemantic) + { + skills.Setup(x => x.GetSemanticFunction(It.Is(s => s == skillName), It.Is(s => s == name))) + .Returns(mockFunction.Object); + skills.Setup(x => x.HasSemanticFunction(It.Is(s => s == skillName), It.Is(s => s == name))).Returns(true); + } + else + { + skills.Setup(x => x.GetNativeFunction(It.Is(s => s == skillName), It.Is(s => s == name))) + .Returns(mockFunction.Object); + skills.Setup(x => x.HasNativeFunction(It.Is(s => s == skillName), It.Is(s => s == name))).Returns(true); + } + } + } + + skills.Setup(x => x.GetFunctionsView(It.IsAny(), It.IsAny())).Returns(functionsView); + } + + [Fact] + public void CanCallToPlanFromXml() + { + // Arrange + var functions = new List<(string name, string skillName, string description, bool isSemantic, string result)>() + { + ("Summarize", "SummarizeSkill", "Summarize an input", true, "This is the summary."), + ("Translate", "WriterSkill", "Translate to french", true, "Bonjour!"), + ("GetEmailAddressAsync", "email", "Get email address", false, "johndoe@email.com"), + ("SendEmailAsync", "email", "Send email", false, "Email sent."), + }; + this.CreateKernelAndFunctionCreateMocks(functions, out var kernel); + + var planString = + @" +Summarize an input, translate to french, and e-mail to John Doe + + + + + + +"; + + // Act + var plan = planString.ToPlanFromXml(kernel.CreateNewContext()); + + // Assert + Assert.NotNull(plan); + Assert.Equal("Summarize an input, translate to french, and e-mail to John Doe", plan.Description); + + Assert.Equal(4, plan.Steps.Count); + Assert.Collection(plan.Steps, + step => + { + Assert.Equal("SummarizeSkill", step.SkillName); + Assert.Equal("Summarize", step.Name); + }, + step => + { + Assert.Equal("WriterSkill", step.SkillName); + Assert.Equal("Translate", step.Name); + Assert.Equal("French", step.NamedParameters["language"]); + Assert.True(step.NamedOutputs.ContainsKey("TRANSLATED_SUMMARY")); + }, + step => + { + Assert.Equal("email", step.SkillName); + Assert.Equal("GetEmailAddressAsync", step.Name); + Assert.Equal("John Doe", step.NamedParameters["input"]); + Assert.True(step.NamedOutputs.ContainsKey("EMAIL_ADDRESS")); + }, + step => + { + Assert.Equal("email", step.SkillName); + Assert.Equal("SendEmailAsync", step.Name); + Assert.Equal("$TRANSLATED_SUMMARY", step.NamedParameters["input"]); + Assert.Equal("$EMAIL_ADDRESS", step.NamedParameters["email_address"]); + } + ); + } + + private const string GoalText = "Solve the equation x^2 = 2."; + + [Fact] + public void InvalidPlanExecutePlanReturnsInvalidResult() + { + // Arrange + this.CreateKernelAndFunctionCreateMocks(new(), out var kernel); + var planString = "" + GoalText; + + // Act + Assert.Throws(() => planString.ToPlanFromXml(kernel.CreateNewContext())); + } + + // Test that contains a #text node in the plan + [Theory] + [InlineData("Test the functionFlowRunner", @"Test the functionFlowRunner + + + This is some text + ")] + public void CanCreatePlanWithTextNodes(string goalText, string planText) + { + // Arrange + var functions = new List<(string name, string skillName, string description, bool isSemantic, string result)>() + { + ("Echo", "MockSkill", "Echo an input", true, "Mock Echo Result"), + }; + this.CreateKernelAndFunctionCreateMocks(functions, out var kernel); + + // Act + var plan = planText.ToPlanFromXml(kernel.CreateNewContext()); + + // Assert + Assert.NotNull(plan); + Assert.Equal(goalText, plan.Description); + Assert.Equal(2, plan.Steps.Count); + Assert.Equal("MockSkill", plan.Steps[0].SkillName); + Assert.Equal("Echo", plan.Steps[0].Name); + Assert.Equal("This is some text", plan.Steps[1].Description); + Assert.Equal(0, plan.Steps[1].Steps.Count); + } + + // test that a that is not will just get skipped + [Theory] + [InlineData("Test the functionFlowRunner", @"Test the functionFlowRunner + + + Some other tag + + ")] + public void CanCreatePlanWithIgnoredNodes(string goalText, string planText) + { + // Arrange + var functions = new List<(string name, string skillName, string description, bool isSemantic, string result)>() + { + ("Echo", "MockSkill", "Echo an input", true, "Mock Echo Result"), + }; + this.CreateKernelAndFunctionCreateMocks(functions, out var kernel); + + // Act + var plan = planText.ToPlanFromXml(kernel.CreateNewContext()); + + // Assert + Assert.NotNull(plan); + Assert.Equal(goalText, plan.Description); + Assert.Equal(3, plan.Steps.Count); + Assert.Equal("MockSkill", plan.Steps[0].SkillName); + Assert.Equal("Echo", plan.Steps[0].Name); + Assert.Equal("Some other tag", plan.Steps[1].Description); + Assert.Equal(0, plan.Steps[1].Steps.Count); + Assert.Equal("MockSkill", plan.Steps[2].SkillName); + Assert.Equal("Echo", plan.Steps[2].Name); + } +} diff --git a/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs b/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs deleted file mode 100644 index 6c3fc2247962..000000000000 --- a/dotnet/src/SemanticKernel/CoreSkills/PlannerSkill.cs +++ /dev/null @@ -1,370 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Text.Json; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.Planning; -using Microsoft.SemanticKernel.SkillDefinition; -using Microsoft.SemanticKernel.Text; - -namespace Microsoft.SemanticKernel.CoreSkills; - -/// -/// Semantic skill that creates and executes plans. -/// -/// Usage: -/// var kernel = SemanticKernel.Build(ConsoleLogger.Log); -/// kernel.ImportSkill("planner", new PlannerSkill(kernel)); -/// -/// -public class PlannerSkill -{ - /// - /// Parameter names. - /// - /// - public static class Parameters - { - /// - /// The number of buckets to create - /// - public const string BucketCount = "bucketCount"; - - /// - /// The prefix to use for the bucket labels - /// - public const string BucketLabelPrefix = "bucketLabelPrefix"; - - /// - /// The relevancy threshold when filtering registered functions. - /// - public const string RelevancyThreshold = "relevancyThreshold"; - - /// - /// The maximum number of relevant functions as result of semantic search to include in the plan creation request. - /// - public const string MaxRelevantFunctions = "MaxRelevantFunctions"; - - /// - /// The list of skills to exclude from the plan creation request. - /// - public const string ExcludedSkills = "excludedSkills"; - - /// - /// The list of functions to exclude from the plan creation request. - /// - public const string ExcludedFunctions = "excludedFunctions"; - - /// - /// The list of functions to include in the plan creation request. - /// - public const string IncludedFunctions = "includedFunctions"; - - /// - /// Whether to use conditional capabilities when creating plans. - /// - public const string UseConditionals = "useConditionals"; - } - - internal sealed class PlannerSkillConfig - { - // Depending on the embeddings engine used, the user ask, - // and the functions available, this value may need to be adjusted. - // For default, this is set to null to exhibit previous behavior. - public double? RelevancyThreshold { get; set; } - - // Limits the number of relevant functions as result of semantic - // search included in the plan creation request. - // will be included - // in the plan regardless of this limit. - public int MaxRelevantFunctions { get; set; } = 100; - - // A list of skills to exclude from the plan creation request. - public HashSet ExcludedSkills { get; } = new() { RestrictedSkillName }; - - // A list of functions to exclude from the plan creation request. - public HashSet ExcludedFunctions { get; } = new() { "CreatePlan", "ExecutePlan" }; - - // A list of functions to include in the plan creation request. - public HashSet IncludedFunctions { get; } = new() { "BucketOutputs" }; - - // Whether to use conditional capabilities when creating plans. - public bool UseConditionals { get; set; } = false; - } - - /// - /// The name to use when creating semantic functions that are restricted from the PlannerSkill plans - /// - private const string RestrictedSkillName = "PlannerSkill_Excluded"; - - /// - /// the function flow runner, which executes plans that leverage functions - /// - private readonly FunctionFlowRunner _functionFlowRunner; - - /// - /// the bucket semantic function, which takes a list of items and buckets them into a number of buckets - /// - private readonly ISKFunction _bucketFunction; - - /// - /// the function flow semantic function, which takes a goal and creates an xml plan that can be executed - /// - private readonly ISKFunction _functionFlowFunction; - - /// - /// the conditional function flow semantic function, which takes a goal and creates an xml plan that can be executed - /// - private readonly ISKFunction _conditionalFunctionFlowFunction; - - /// - /// Initializes a new instance of the class. - /// - /// The kernel to use - /// The maximum number of tokens to use for the semantic functions - public PlannerSkill(IKernel kernel, int maxTokens = 1024) - { - this._functionFlowRunner = new(kernel); - - this._bucketFunction = kernel.CreateSemanticFunction( - promptTemplate: SemanticFunctionConstants.BucketFunctionDefinition, - skillName: RestrictedSkillName, - maxTokens: maxTokens, - temperature: 0.0); - - this._functionFlowFunction = kernel.CreateSemanticFunction( - promptTemplate: SemanticFunctionConstants.FunctionFlowFunctionDefinition, - skillName: RestrictedSkillName, - description: "Given a request or command or goal generate a step by step plan to " + - "fulfill the request using functions. This ability is also known as decision making and function flow", - maxTokens: maxTokens, - temperature: 0.0, - stopSequences: new[] { "", " - -[AVAILABLE FUNCTIONS] - - _GLOBAL_FUNCTIONS_.BucketOutputs: - description: When the output of a function is too big, parse the output into a number of buckets. - inputs: - - input: The output from a function that needs to be parse into buckets. - - bucketCount: The number of buckets. - - bucketLabelPrefix: The target label prefix for the resulting buckets. Result will have index appended e.g. bucketLabelPrefix='Result' => Result_1, Result_2, Result_3 - - _GLOBAL_FUNCTIONS_.GetEmailAddress: - description: Gets email address for given contact - inputs: - - input: the name to look up - - _GLOBAL_FUNCTIONS_.SendEmail: - description: email the input text to a recipient - inputs: - - input: the text to email - - recipient: the recipient's email address. Multiple addresses may be included if separated by ';'. - - AuthorAbility.Summarize: - description: summarizes the input text - inputs: - - input: the text to summarize - - Magician.TranslateTo: - description: translate the input to another language - inputs: - - input: the text to translate - - translate_to_language: the language to translate to - -[END AVAILABLE FUNCTIONS] - -Summarize an input, translate to french, and e-mail to John Doe - - - - - - - -[AVAILABLE FUNCTIONS] - - _GLOBAL_FUNCTIONS_.BucketOutputs: - description: When the output of a function is too big, parse the output into a number of buckets. - inputs: - - input: The output from a function that needs to be parse into buckets. - - bucketCount: The number of buckets. - - bucketLabelPrefix: The target label prefix for the resulting buckets. Result will have index appended e.g. bucketLabelPrefix='Result' => Result_1, Result_2, Result_3 - - _GLOBAL_FUNCTIONS_.NovelOutline : - description: Outlines the input text as if it were a novel - inputs: - - input: the title of the novel to outline - - chapterCount: the number of chapters to outline - - Emailer.EmailTo: - description: email the input text to a recipient - inputs: - - input: the text to email - - recipient: the recipient's email address. Multiple addresses may be included if separated by ';'. - - Everything.Summarize: - description: summarize input text - inputs: - - input: the text to summarize - -[END AVAILABLE FUNCTIONS] - -Create an outline for a children's book with 3 chapters about a group of kids in a club and then summarize it. - - - - - -[END EXAMPLES] - -[AVAILABLE FUNCTIONS] - -{{$available_functions}} - -[END AVAILABLE FUNCTIONS] - -{{$input}} -"; - - internal const string ConditionalFunctionFlowFunctionDefinition = - @"[PLAN 1] - AuthorAbility.Summarize: - description: summarizes the input text - inputs: - - input: the text to summarize - Magician.TranslateTo: - description: translate the input to another language - inputs: - - input: the text to translate - - translate_to_language: the language to translate to - _GLOBAL_FUNCTIONS_.GetEmailAddress: - description: Gets email address for given contact - inputs: - - input: the name to look up - _GLOBAL_FUNCTIONS_.SendEmail: - description: email the input text to a recipient - inputs: - - input: the text to email - - recipient: the recipient's email address. Multiple addresses may be included if separated by ';'. - -Summarize an input, translate to french, and e-mail to John Doe - - - - - - - -[PLAN 2] - Everything.Summarize: - description: summarize input text - inputs: - - input: the text to summarize - _GLOBAL_FUNCTIONS_.NovelOutline : - description: Outlines the input text as if it were a novel - inputs: - - input: the title of the novel to outline - - chapterCount: the number of chapters to outline - LanguageHelpers.TranslateTo: - description: translate the input to another language - inputs: - - input: the text to translate - - translate_to_language: the language to translate to - EmailConnector.LookupContactEmail: - description: looks up the a contact and retrieves their email address - inputs: - - input: the name to look up - EmailConnector.EmailTo: - description: email the input text to a recipient - inputs: - - input: the text to email - - recipient: the recipient's email address. Multiple addresses may be included if separated by ';'. - _GLOBAL_FUNCTIONS_.Length: - description: Get the length of a string. - inputs: - - input: Input string - _GLOBAL_FUNCTIONS_.Hour: - description: Get the current clock hour - inputs: - -If its afternoon please Summarize the input, if the input length > 10 and contains ""book"" then Create an outline for a children's book with 3 chapters about a group of kids in a club and summarize it otherwise translate to japanese and email it to Martin. - - - - - - - - - - (dont use elseif) - - - - - - - -[PLAN 3] -{{$available_functions}} - -Plan Rules: -Create an XML plan step by step, to satisfy the goal given. -To create a plan, follow these steps: -1. From a create a as a series of . -2. Only use functions that are required for the given goal. -3. A function has an 'input' and an 'output'. -4. The 'output' from each function is automatically passed as 'input' to the subsequent . -5. 'input' does not need to be specified if it consumes the 'output' of the previous function. -6. To save an 'output' from a , to pass into a future , use ""/> -7. To save an 'output' from a , to return as part of a plan result, use ""/> -8. Only use ""if"", ""else"" or ""while"" tags when needed -9. ""if"", ""else"" and ""while"" tags must be closed -10. Do not use . For such a condition, use an additional ... block instead of . -11. Comparison operators must be literals. -12. Append an ""END"" XML comment at the end of the plan. -13. Dont use arrays or objects for variables -14. Only use variables that where assigned before with setContextVariables -15. Use only the AVAILABLE FUNCTIONS in the deck - -{{$input}} -"; - - internal const string BucketFunctionDefinition = - @"1. Given an output of a function, bucket the output into a list of results. - -Examples: -[CONTENT] -Result 1 -This is the first result. -Result 2 -This is the second result. It's doubled! -Result 3 -This is the third and final result. Truly astonishing. -[END CONTENT] - -EXPECTED BUCKETS: - -Result: -{""buckets"": [""Result 1 -This is the first result."", ""Result 2 -This is the second result. It's doubled!"", ""Result 3 -This is the third and final result. Truly astonishing.""]} - -End examples. - -[CONTENT] -{{$input}} -[END CONTENT] - -EXPECTED BUCKETS: {{$bucketCount}} - -Result: -"; - - internal const string ProblemSolverFunctionDefinition = - @"Create a plan as a list, step by step, to answer the request or goal given. -To create a plan, follow these steps: -1. Identify the request or goal to be achieved. -2. Break down the request into smaller tasks and steps. -3. If the goal has a ""use"" parameter, use those functions with the exact name given. -4. Use experience and logic to determine the steps and tasks needed. -5. Provide a detailed decision-making process for each step. -6. Avoid generic instructions and add as many details as possible. -7. Present the solution using precise XML syntax with ""solution"" and ""step"" tags. -8. Ensure that all tags are closed. -9. Append an ""END"" comment at the end of the plan. -10. If you don't know how to fulfill a request, use the ""noSolution"" tag. -11. Use a computer, browser, apps, and external services to fulfill the goal. -12. Ensure that all goals are fulfilled from a computer. - -Here some good examples: - - -what time is it? - - - - Get current location - Find the time zone for the location in the variables - Get the current time for the time zone in the variables - - - -what time is it? - - - - - Get the current time for time zone in the variables - - - -what time is it? - - - - - - Get the current time from the variables - - - -how long till Christmas? - - - - - - - Get the current date from the variables - Calculate days from ""current date"" to ""December 25"" - - - -Get user's location - - - - Search for the user location in variables - If the user location is unknown ask the user: What is your location? - - - -Get user's location - - - - - Get the location from the variables - If the user location is unknown ask the user to teach you how to find the value - - - -Find my time zone - - - - - Get the location from the variables - If the user location is unknown ask the user: What is your location? - Find the timezone for given location - If the user timezone is unknown ask the user to teach you how to find the value - - - -summarize last week emails - - - - Find the current time and date - Get all emails from given time to time minus 7 days - Summarize the email in variables - - - -Get the current date and time - - - - Find the current date and time - Get date and time from the variables - - - -Get the current date and time - - - - - - - - Get date and time from the variables - - - -how long until my wife's birthday? - - - - - - - - Search for wife's birthday in memory - If the previous step is empty ask the user: when is your wife's birthday? - - - -Search for wife's birthday in memory - - - - Find name of wife in variables - If the wife name is unknown ask the user - Search for wife's birthday in Facebook using the name in memory - Search for wife's birthday in Teams conversations filtering messages by name and using the name in memory - Search for wife's birthday in Emails filtering messages by name and using the name in memory - If the birthday cannot be found tell the user, ask the user to teach you how to find the value - - - -Search for gift ideas - - - - Find topics of interest from personal conversations - Find topics of interest from personal emails - Search Amazon for gifts including topics in the variables - - - -Count from 1 to 5 - - - - Create a counter variable in memory with value 1 - Show the value of the counter variable - If the counter variable is 5 stop - Increment the counter variable - - - -foo bar - - - - Sorry I don't know how to help with that - - -The following is an incorrect example, because the solution uses a skill not listed in the 'use' attribute. - - -do something - - - - - - -End of examples. - - -{{$SKILLS_MANUAL}} - - - -{{$INPUT}} - -"; - - internal const string SolveNextStepFunctionDefinition = - @"{{$INPUT}} - -Update the plan above: -* If there are steps in the solution, then: - ** use the variables to execute the first step - ** if the variables contains a result, replace it with the result of the first step, otherwise store the result in the variables - ** Remove the first step. -* Keep the XML syntax correct, with a new line after the goal. -* Emit only XML. -* If the list of steps is empty, answer the goal using information in the variables, putting the solution inside the solution tag. -* Append at the end. -END OF INSTRUCTIONS. - -Possible updated plan: -"; - internal const string SummarizeConversationDefinition = @"BEGIN CONTENT TO SUMMARIZE: {{$INPUT}} diff --git a/dotnet/src/SemanticKernel/Orchestration/ContextVariablesConverter.cs b/dotnet/src/SemanticKernel/Orchestration/ContextVariablesConverter.cs new file mode 100644 index 000000000000..f60e58abae11 --- /dev/null +++ b/dotnet/src/SemanticKernel/Orchestration/ContextVariablesConverter.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Orchestration; + +/// +/// Converter for to/from JSON. +/// +public class ContextVariablesConverter : JsonConverter +{ + /// + /// Read the JSON and convert to ContextVariables + /// + /// The JSON reader. + /// The type to convert from. + /// The JSON serializer options. + /// The deserialized . + public override ContextVariables Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + // Needs Test + var keyValuePairs = JsonSerializer.Deserialize>>(ref reader, options); + var context = new ContextVariables(); + + foreach (var kvp in keyValuePairs!) + { + context.Set(kvp.Key, kvp.Value); + } + + return context; + } + + public override void Write(Utf8JsonWriter writer, ContextVariables value, JsonSerializerOptions options) + { + // Needs Test + JsonSerializer.Serialize(writer, value, options); + } +} diff --git a/dotnet/src/SemanticKernel/Orchestration/Extensions/ContextVariablesExtensions.cs b/dotnet/src/SemanticKernel/Orchestration/Extensions/ContextVariablesExtensions.cs index 2ac1013a1233..c90a44e5a174 100644 --- a/dotnet/src/SemanticKernel/Orchestration/Extensions/ContextVariablesExtensions.cs +++ b/dotnet/src/SemanticKernel/Orchestration/Extensions/ContextVariablesExtensions.cs @@ -1,11 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. -using System; -using System.Text.Json; -using Microsoft.SemanticKernel.Planning; - +#pragma warning disable IDE0130 // ReSharper disable once CheckNamespace // Extension methods namespace Microsoft.SemanticKernel.Orchestration; +#pragma warning restore IDE0130 /// /// Class that holds extension methods for ContextVariables. @@ -21,80 +19,4 @@ public static ContextVariables ToContextVariables(this string text) { return new ContextVariables(text); } - - /// - /// Simple extension method to update a instance with a Plan instance. - /// - /// The variables to update - /// The Plan to update the with - /// The updated - public static ContextVariables UpdateWithPlanEntry(this ContextVariables vars, SkillPlan plan) - { - vars.Update(plan.ToJson()); - vars.Set(SkillPlan.IdKey, plan.Id); - vars.Set(SkillPlan.GoalKey, plan.Goal); - vars.Set(SkillPlan.PlanKey, plan.PlanString); - vars.Set(SkillPlan.IsCompleteKey, plan.IsComplete.ToString()); - vars.Set(SkillPlan.IsSuccessfulKey, plan.IsSuccessful.ToString()); - vars.Set(SkillPlan.ResultKey, plan.Result); - - return vars; - } - - /// - /// Simple extension method to clear the PlanCreation entries from a instance. - /// - /// The to update - public static ContextVariables ClearPlan(this ContextVariables vars) - { - vars.Set(SkillPlan.IdKey, null); - vars.Set(SkillPlan.GoalKey, null); - vars.Set(SkillPlan.PlanKey, null); - vars.Set(SkillPlan.IsCompleteKey, null); - vars.Set(SkillPlan.IsSuccessfulKey, null); - vars.Set(SkillPlan.ResultKey, null); - return vars; - } - - /// - /// Simple extension method to parse a Plan instance from a instance. - /// - /// The to read - /// An instance of Plan - public static SkillPlan ToPlan(this ContextVariables vars) - { - if (vars.Get(SkillPlan.PlanKey, out string plan)) - { - vars.Get(SkillPlan.IdKey, out string id); - vars.Get(SkillPlan.GoalKey, out string goal); - vars.Get(SkillPlan.IsCompleteKey, out string isComplete); - vars.Get(SkillPlan.IsSuccessfulKey, out string isSuccessful); - vars.Get(SkillPlan.ResultKey, out string result); - - return new SkillPlan() - { - Id = id, - Goal = goal, - PlanString = plan, - IsComplete = !string.IsNullOrEmpty(isComplete) && bool.Parse(isComplete), - IsSuccessful = !string.IsNullOrEmpty(isSuccessful) && bool.Parse(isSuccessful), - Result = result - }; - } - - try - { - return SkillPlan.FromJson(vars.ToString()); - } - catch (ArgumentNullException) - { - } - catch (JsonException) - { - } - - // If SkillPlan.FromJson fails, return a Plan with the current as the plan. - // Validation of that `plan` will be done separately. - return new SkillPlan() { PlanString = vars.ToString() }; - } } diff --git a/dotnet/src/SemanticKernel/Orchestration/Plan.cs b/dotnet/src/SemanticKernel/Orchestration/Plan.cs index b2f05dd04c5d..22683fb4a5e4 100644 --- a/dotnet/src/SemanticKernel/Orchestration/Plan.cs +++ b/dotnet/src/SemanticKernel/Orchestration/Plan.cs @@ -2,12 +2,16 @@ using System; using System.Collections.Generic; +using System.Linq; +using System.Text.Json; using System.Text.Json.Serialization; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.SemanticKernel.AI.TextCompletion; +using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.SkillDefinition; namespace Microsoft.SemanticKernel.Orchestration; @@ -16,29 +20,46 @@ namespace Microsoft.SemanticKernel.Orchestration; /// Standard Semantic Kernel callable plan. /// Plan is used to create trees of s. /// -public class Plan : ISKFunction +public sealed class Plan : ISKFunction { /// /// State of the plan /// [JsonPropertyName("state")] + [JsonConverter(typeof(ContextVariablesConverter))] public ContextVariables State { get; } = new(); /// /// Steps of the plan /// [JsonPropertyName("steps")] - internal IReadOnlyList Steps => this._steps.AsReadOnly(); + public IReadOnlyList Steps => this._steps.AsReadOnly(); /// /// Named parameters for the function /// [JsonPropertyName("named_parameters")] + [JsonConverter(typeof(ContextVariablesConverter))] public ContextVariables NamedParameters { get; set; } = new(); + /// + /// Named outputs for the function + /// + [JsonPropertyName("named_outputs")] + [JsonConverter(typeof(ContextVariablesConverter))] + public ContextVariables NamedOutputs { get; set; } = new(); + + /// + /// Gets whether the plan has a next step. + /// + [JsonIgnore] public bool HasNextStep => this.NextStepIndex < this.Steps.Count; - protected int NextStepIndex { get; set; } = 0; + /// + /// Gets the next step index. + /// + [JsonPropertyName("next_step_index")] + public int NextStepIndex { get; internal set; } = 0; #region ISKFunction implementation @@ -55,11 +76,11 @@ public class Plan : ISKFunction public string Description { get; set; } = string.Empty; /// - [JsonPropertyName("is_semantic")] + [JsonIgnore] public bool IsSemantic { get; internal set; } = false; /// - [JsonPropertyName("request_settings")] + [JsonIgnore] public CompleteRequestSettings RequestSettings { get; internal set; } = new(); #endregion ISKFunction implementation @@ -80,12 +101,21 @@ public Plan(string goal) /// /// The goal of the plan used as description. /// The steps to add. - [JsonConstructor] public Plan(string goal, params ISKFunction[] steps) : this(goal) { this.AddSteps(steps); } + /// + /// Initializes a new instance of the class with a goal description and steps. + /// + /// The goal of the plan used as description. + /// The steps to add. + public Plan(string goal, params Plan[] steps) : this(goal) + { + this.AddSteps(steps); + } + /// /// Initializes a new instance of the class with a function. /// @@ -95,6 +125,70 @@ public Plan(ISKFunction function) this.SetFunction(function); } + /// + /// Initializes a new instance of the class with a function and steps. + /// + /// The name of the plan. + /// The name of the skill. + /// The description of the plan. + /// The index of the next step. + /// The state of the plan. + /// The named parameters of the plan. + /// The named outputs of the plan. + /// The steps of the plan. + [JsonConstructor] + public Plan(string name, string skillName, string description, int nextStepIndex, ContextVariables state, ContextVariables namedParameters, + ContextVariables namedOutputs, + IReadOnlyList steps) + { + this.Name = name; + this.SkillName = skillName; + this.Description = description; + this.NextStepIndex = nextStepIndex; + this.State = state; + this.NamedParameters = namedParameters; + this.NamedOutputs = namedOutputs; + this._steps.Clear(); + this.AddSteps(steps.ToArray()); + } + + /// + /// Deserialize a JSON string into a Plan object. + /// + /// JSON string representation of a Plan + /// The context to use for function registrations. + /// An instance of a Plan object. + public static Plan FromJson(string json, SKContext context) + { + // Needs Test + var plan = JsonSerializer.Deserialize(json, new JsonSerializerOptions() { IncludeFields = true }) ?? new Plan(string.Empty); + + plan = SetRegisteredFunctions(plan, context); + + return plan; + } + + /// + /// Get JSON representation of the plan. + /// + public string ToJson() + { + // Needs Test + return JsonSerializer.Serialize(this); + } + + /// + /// Adds one or more existing plans to the end of the current plan as steps. + /// + /// The plans to add as steps to the current plan. + /// + /// When you add a plan as a step to the current plan, the steps of the added plan are executed after the steps of the current plan have completed. + /// + public void AddSteps(params Plan[] steps) + { + this._steps.AddRange(steps); + } + /// /// Adds one or more new steps to the end of the current plan. /// @@ -104,10 +198,7 @@ public Plan(ISKFunction function) /// public void AddSteps(params ISKFunction[] steps) { - foreach (var step in steps) - { - this._steps.Add(new Plan(step)); - } + this._steps.AddRange(steps.Select(step => new Plan(step))); } /// @@ -131,13 +222,63 @@ public Task RunNextStepAsync(IKernel kernel, ContextVariables variables, C return this.InvokeNextStepAsync(context); } + /// + /// Invoke the next step of the plan + /// + /// Context to use + /// The updated plan + /// If an error occurs while running the plan + public async Task InvokeNextStepAsync(SKContext context) + { + if (this.HasNextStep) + { + var step = this.Steps[this.NextStepIndex]; + + // Merge the state with the current context variables for step execution + var functionVariables = this.GetNextStepVariables(context.Variables, step); + + // Execute the step + var functionContext = new SKContext(functionVariables, context.Memory, context.Skills, context.Log, context.CancellationToken); + var result = await step.InvokeAsync(functionContext); + + if (result.ErrorOccurred) + { + throw new KernelException(KernelException.ErrorCodes.FunctionInvokeError, + $"Error occurred while running plan step: {context.LastErrorDescription}", context.LastException); + } + + #region Update State + + // Update state with result + this.State.Update(result.Result.Trim()); + + // Update state with named outputs (if any) + foreach (var item in step.NamedOutputs) + { + // ignore the input key + if (item.Key.ToUpperInvariant() == "INPUT") + { + continue; + } + + this.State.Set(item.Key, result.Result.Trim()); + } + + #endregion Update State + + this.NextStepIndex++; + } + + return this; + } + #region ISKFunction implementation /// public FunctionView Describe() { // TODO - Eventually, we should be able to describe a plan and it's expected inputs/outputs - return this.Function?.Describe() ?? throw new NotImplementedException(); + return this.Function?.Describe() ?? new(); } /// @@ -174,7 +315,6 @@ public async Task InvokeAsync(SKContext? context = null, CompleteRequ while (this.HasNextStep) { var functionContext = context; - // Loop through State and add anything missing to functionContext AddVariablesToContext(this.State, functionContext); @@ -214,33 +354,126 @@ public ISKFunction SetAIConfiguration(CompleteRequestSettings settings) #endregion ISKFunction implementation /// - /// Invoke the next step of the plan + /// Expand variables in the input string. /// - /// Context to use - /// The updated plan - /// If an error occurs while running the plan - public async Task InvokeNextStepAsync(SKContext context) + /// Variables to use for expansion. + /// Input string to expand. + /// Expanded string. + internal string ExpandFromVariables(ContextVariables variables, string input) { - if (this.HasNextStep) + var result = input; + var matches = Regex.Matches(input, @"\$(?\w+)"); + var orderedMatches = matches.Cast().Select(m => m.Groups["var"].Value).OrderByDescending(m => m.Length); + + foreach (var varName in orderedMatches) { - var step = this.Steps[this.NextStepIndex]; + result = variables.Get(varName, out var value) + ? result.Replace($"${varName}", value) + : this.State.Get(varName, out value) + ? result.Replace($"${varName}", value) + : result.Replace($"${varName}", string.Empty); + } - context = await step.InvokeAsync(context); + return result; + } - if (context.ErrorOccurred) + /// + /// Set functions for a plan and its steps. + /// + /// Plan to set functions for. + /// Context to use. + /// The plan with functions set. + private static Plan SetRegisteredFunctions(Plan plan, SKContext context) + { + if (plan.Steps.Count == 0) + { + if (context.IsFunctionRegistered(plan.SkillName, plan.Name, out var skillFunction)) { - throw new KernelException(KernelException.ErrorCodes.FunctionInvokeError, - $"Error occurred while running plan step: {context.LastErrorDescription}", context.LastException); + Verify.NotNull(skillFunction, nameof(skillFunction)); + plan.SetFunction(skillFunction); } + } + else + { + foreach (var step in plan.Steps) + { + SetRegisteredFunctions(step, context); + } + } - this.NextStepIndex++; - this.State.Update(context.Result.Trim()); + return plan; + } + + /// + /// Add any missing variables from a plan state variables to the context. + /// + private static void AddVariablesToContext(ContextVariables vars, SKContext context) + { + // Loop through vars and add anything missing to context + foreach (var item in vars) + { + if (!context.Variables.ContainsKey(item.Key)) + { + context.Variables.Set(item.Key, item.Value); + } } + } - return this; + /// + /// Get the variables for the next step in the plan. + /// + /// The current context variables. + /// The next step in the plan. + /// The context variables for the next step in the plan. + private ContextVariables GetNextStepVariables(ContextVariables variables, Plan step) + { + // If the current step is passing to another plan, we set the default input to an empty string. + // Otherwise, we use the description from the current plan as the default input. + // We then set the input to the value from the SKContext, or the input from the Plan.State, or the default input. + var defaultInput = step.Steps.Count > 0 ? string.Empty : this.Description ?? string.Empty; + var planInput = string.IsNullOrEmpty(variables.Input) ? this.State.Input : variables.Input; + var stepInput = string.IsNullOrEmpty(planInput) ? defaultInput : planInput; + var stepVariables = new ContextVariables(stepInput); + + // Priority for remaining stepVariables is: + // - NamedParameters (pull from State by a key value) + // - Parameters (from context) + // - Parameters (from State) + var functionParameters = step.Describe(); + foreach (var param in functionParameters.Parameters) + { + if (variables.Get(param.Name, out var value) && !string.IsNullOrEmpty(value)) + { + stepVariables.Set(param.Name, value); + } + else if (this.State.Get(param.Name, out value) && !string.IsNullOrEmpty(value)) + { + // Needs test + stepVariables.Set(param.Name, value); + } + } + + foreach (var item in step.NamedParameters) + { + if (!string.IsNullOrEmpty(item.Value)) + { + var value = this.ExpandFromVariables(variables, item.Value); + stepVariables.Set(item.Key, value); + } + else if (variables.Get(item.Key, out var value) && !string.IsNullOrEmpty(value)) + { + stepVariables.Set(item.Key, value); + } + else if (this.State.Get(item.Key, out value) && !string.IsNullOrEmpty(value)) + { + stepVariables.Set(item.Key, value); + } + } + + return stepVariables; } - protected void SetFunction(ISKFunction function) + private void SetFunction(ISKFunction function) { this.Function = function; this.Name = function.Name; @@ -250,18 +483,7 @@ protected void SetFunction(ISKFunction function) this.RequestSettings = function.RequestSettings; } - protected ISKFunction? Function { get; set; } = null; + private ISKFunction? Function { get; set; } = null; - private List _steps = new(); - - private static void AddVariablesToContext(ContextVariables vars, SKContext context) - { - foreach (var item in vars) - { - if (!context.Variables.ContainsKey(item.Key)) - { - context.Variables.Set(item.Key, item.Value); - } - } - } + private readonly List _steps = new(); } diff --git a/dotnet/src/SemanticKernel/Planning/ConditionException.cs b/dotnet/src/SemanticKernel/Planning/ConditionException.cs deleted file mode 100644 index 6581300fb619..000000000000 --- a/dotnet/src/SemanticKernel/Planning/ConditionException.cs +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.SemanticKernel.Diagnostics; - -namespace Microsoft.SemanticKernel.Planning; - -public class ConditionException : Exception -{ - /// - /// Error codes for . - /// - public enum ErrorCodes - { - /// - /// Unknown error. - /// - UnknownError = -1, - - /// - /// Invalid condition structure. - /// - InvalidCondition = 0, - - /// - /// Invalid statement structure. - /// - InvalidStatementStructure = 1, - - /// - /// Json Response was not present in the output - /// - InvalidResponse = 2, - - /// - /// Required context variables are not present in the context - /// - ContextVariablesNotFound = 3, - } - - /// - /// Gets the error code of the exception. - /// - public ErrorCodes ErrorCode { get; } - - /// - /// Initializes a new instance of the class. - /// - /// The error code. - /// The message. - public ConditionException(ErrorCodes errCode, string? message = null) : base(errCode, message) - { - this.ErrorCode = errCode; - } - - /// - /// Initializes a new instance of the class. - /// - /// The error code. - /// The message. - /// The inner exception. - public ConditionException(ErrorCodes errCode, string message, Exception? e) : base(errCode, message, e) - { - this.ErrorCode = errCode; - } - - #region private ================================================================================ - - private ConditionException() - { - // Not allowed, error code is required - } - - private ConditionException(string message) : base(message) - { - // Not allowed, error code is required - } - - private ConditionException(string message, Exception innerException) : base(message, innerException) - { - // Not allowed, error code is required - } - - #endregion -} diff --git a/dotnet/src/SemanticKernel/Planning/ConditionalFlowConstants.cs b/dotnet/src/SemanticKernel/Planning/ConditionalFlowConstants.cs deleted file mode 100644 index 2ed79b31fcb6..000000000000 --- a/dotnet/src/SemanticKernel/Planning/ConditionalFlowConstants.cs +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -namespace Microsoft.SemanticKernel.Planning; - -internal class ConditionalFlowConstants -{ - internal const string IfStructureCheckPrompt = - @"Structure: - - - (optional) - - -Rules: -. A condition attribute must exists in the if tag -. ""if"" tag must have one or more children nodes -. ""else"" tag is optional -. If ""else"" is provided must have one or more children nodes -. Return true if Test Structure is valid -. Return false if Test Structure is not valid with a reason with everything that is wrong -. Give a json list of variables used inside the attribute ""condition"" of the first ""if"" only -. All the return should be in Json format. -. Response Json Structure: -{ - ""valid"": bool, - ""reason"": string, (only if invalid) - ""variables"": [string] (only the variables within ""Condition"" attribute) -} - -Test Structure: -{{$IfStatementContent}} - -Response: "; - - internal const string EvaluateConditionPrompt = - @"Rules -. Response using the following json structure: -{ ""valid"": bool, ""condition"": bool, ""reason"": string } - -. A list of variables will be provided for the condition evaluation -. If condition has an error, valid will be false and property ""reason"" will have all detail why. -. If condition is valid, update reason with the current evaluation detail. -. Return ""condition"" as true or false depending on the condition evaluation - -Variables Example: -x = 2 -y = ""adfsdasfgsasdddsf"" -z = 100 -w = ""sdf"" - -Given Example: -$x equals 1 and $y contains 'asd' or not ($z greaterthan 10) - -Reason Example: -(x == 1 ∧ (y contains ""asd"") ∨ ¬(z > 10)) -(TRUE ∧ TRUE ∨ ¬ (FALSE)) -(TRUE ∧ TRUE ∨ TRUE) -TRUE - -Variables Example: -24hour = 11 - -Given Example: -$24hour equals 1 and $24hour greaterthan 10 - -Response Example: -{ ""valid"": true, ""condition"": false, ""reason"": ""(24hour == 1 ∧ 24hour > 10) = (FALSE ∧ TRUE) = FALSE"" } - -Variables Example: -24hour = 11 - -Given Example: -Some condition - -Response Example: -{ ""valid"": false, ""reason"": """" } - -Variables Example: -a = 1 -b = undefined -c = ""dome"" - -Given Example: -(a is not undefined) and a greaterthan 100 and a greaterthan 10 or (a equals 1 and a equals 10) or (b is undefined and c is not undefined) - -Response Example: -{ ""valid"": true, ""condition"": true, ""reason"": ""((a is not undefined) ∧ a > 100 ∧ a > 10) ∨ (a == 1 ∧ a == 10) ∨ (b is undefined ∧ c is not undefined) = ((TRUE ∧ FALSE ∧ FALSE) ∨ (TRUE ∧ FALSE) ∨ (TRUE ∧ TRUE)) = (FALSE ∨ FALSE ∨ TRUE) = TRUE"" } - -Variables: -{{$ConditionalVariables}} - -Given: -{{$IfCondition}} - -Response: "; -} diff --git a/dotnet/src/SemanticKernel/Planning/ConditionalFlowHelper.cs b/dotnet/src/SemanticKernel/Planning/ConditionalFlowHelper.cs deleted file mode 100644 index 3da68098dbbe..000000000000 --- a/dotnet/src/SemanticKernel/Planning/ConditionalFlowHelper.cs +++ /dev/null @@ -1,343 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Text.Json; -using System.Text.Json.Nodes; -using System.Text.RegularExpressions; -using System.Threading.Tasks; -using System.Xml; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.AI.TextCompletion; -using Microsoft.SemanticKernel.Orchestration; - -namespace Microsoft.SemanticKernel.Planning; - -/// -/// Semantic skill that evaluates conditional structures -/// -/// Usage: -/// var kernel = SemanticKernel.Build(ConsoleLogger.Log); -/// kernel.ImportSkill("conditional", new ConditionalSkill(kernel)); -/// -/// -public class ConditionalFlowHelper -{ - internal const string NoReasonMessage = "No reason was provided"; - - private readonly ISKFunction _ifStructureCheckFunction; - private readonly ISKFunction _evaluateConditionFunction; - - /// - /// Initializes a new instance of the class. - /// - /// The kernel to use - /// A optional completion backend to run the internal semantic functions - internal ConditionalFlowHelper(IKernel kernel, ITextCompletion? completionBackend = null) - { - this._ifStructureCheckFunction = kernel.CreateSemanticFunction( - ConditionalFlowConstants.IfStructureCheckPrompt, - skillName: "PlannerSkill_Excluded", - description: "Evaluate if an If structure is valid and returns TRUE or FALSE", - maxTokens: 100, - temperature: 0, - topP: 0.5); - - this._evaluateConditionFunction = kernel.CreateSemanticFunction( - ConditionalFlowConstants.EvaluateConditionPrompt, - skillName: "PlannerSkill_Excluded", - description: "Evaluate a condition group and returns TRUE or FALSE", - maxTokens: 100, - temperature: 0, - topP: 0.5); - - if (completionBackend is not null) - { - this._ifStructureCheckFunction.SetAIService(() => completionBackend); - this._evaluateConditionFunction.SetAIService(() => completionBackend); - } - } - - /// - /// Get a planner if statement content and output or contents depending on the conditional evaluation. - /// - /// Full statement content including if and else. - /// The context to use - /// Then or Else contents depending on the conditional evaluation - /// - /// This skill is initially intended to be used only by the Plan Runner. - /// - public async Task IfAsync(string ifFullContent, SKContext context) - { - XmlDocument xmlDoc = new(); - xmlDoc.LoadXml("" + ifFullContent + ""); - - this.EnsureIfStructure(xmlDoc, out var ifNode, out var elseNode); - - var usedVariables = this.GetUsedVariables(ifNode); - - // Temporarily avoiding going to LLM to resolve variable and If structure - // await this.GetVariablesAndEnsureIfStructureIsValidAsync(ifNode.OuterXml, context).ConfigureAwait(false); - - bool conditionEvaluation = await this.EvaluateConditionAsync(ifNode, usedVariables, context).ConfigureAwait(false); - - return conditionEvaluation - ? ifNode.InnerXml - : elseNode?.InnerXml ?? string.Empty; - } - - /// - /// Get a planner while statement content and output or not its contents depending on the conditional evaluation. - /// - /// While content. - /// The context to use - /// None if evaluates to false or the children steps appended of a copy of the while structure - /// - /// This skill is initially intended to be used only by the Plan Runner. - /// - public async Task WhileAsync(string whileContent, SKContext context) - { - XmlDocument xmlDoc = new(); - xmlDoc.LoadXml("" + whileContent + ""); - - this.EnsureWhileStructure(xmlDoc, out var whileNode); - - var usedVariables = this.GetUsedVariables(whileNode); - - bool conditionEvaluation = await this.EvaluateConditionAsync(whileNode, usedVariables, context).ConfigureAwait(false); - - return conditionEvaluation - ? whileNode.InnerXml + whileContent - : string.Empty; - } - - private void EnsureIfStructure(XmlDocument xmlDoc, out XmlNode ifNode, out XmlNode? elseNode) - { - ifNode = - xmlDoc.SelectSingleNode("//if") - ?? throw new ConditionException(ConditionException.ErrorCodes.InvalidStatementStructure, "If is not present"); - - XmlAttribute? conditionContents = ifNode.Attributes?["condition"]; - - if (conditionContents is null) - { - throw new ConditionException(ConditionException.ErrorCodes.InvalidStatementStructure, "Condition attribute is not present"); - } - - if (string.IsNullOrWhiteSpace(conditionContents.Value)) - { - throw new ConditionException(ConditionException.ErrorCodes.InvalidStatementStructure, "Condition attribute value cannot be empty"); - } - - if (!ifNode.HasChildNodes) - { - throw new ConditionException(ConditionException.ErrorCodes.InvalidStatementStructure, "If has no children"); - } - - elseNode = xmlDoc.SelectSingleNode("//else"); - if (elseNode is not null && !elseNode.HasChildNodes) - { - throw new ConditionException(ConditionException.ErrorCodes.InvalidStatementStructure, "Else has no children"); - } - } - - private void EnsureWhileStructure(XmlDocument xmlDoc, out XmlNode whileNode) - { - whileNode = - xmlDoc.SelectSingleNode("//while") - ?? throw new ConditionException(ConditionException.ErrorCodes.InvalidStatementStructure, "While is not present"); - - XmlAttribute? conditionContents = whileNode.Attributes?["condition"]; - - if (conditionContents is null) - { - throw new ConditionException(ConditionException.ErrorCodes.InvalidStatementStructure, "Condition attribute is not present"); - } - - if (string.IsNullOrWhiteSpace(conditionContents.Value)) - { - throw new ConditionException(ConditionException.ErrorCodes.InvalidStatementStructure, "Condition attribute value cannot be empty"); - } - - if (!whileNode.HasChildNodes) - { - throw new ConditionException(ConditionException.ErrorCodes.InvalidStatementStructure, "While has no children"); - } - } - - /// - /// Get the variables used in the If statement and ensure the structure is valid using LLM - /// - /// If structure content - /// Current context - /// List of used variables in the if condition - /// InvalidStatementStructure - /// InvalidResponse - private async Task> GetVariablesAndEnsureIfStructureIsValidAsync(string ifContent, SKContext context) - { - context.Variables.Set("IfStatementContent", ifContent); - var llmRawResponse = (await this._ifStructureCheckFunction.InvokeAsync(ifContent, context).ConfigureAwait(false)).ToString(); - - JsonNode llmJsonResponse = this.GetLlmResponseAsJsonWithProperties(llmRawResponse, "valid"); - var valid = llmJsonResponse["valid"]!.GetValue(); - - if (!valid) - { - var reason = llmJsonResponse?["reason"]?.GetValue(); - - throw new ConditionException(ConditionException.ErrorCodes.InvalidStatementStructure, - !string.IsNullOrWhiteSpace(reason) - ? reason - : NoReasonMessage); - } - - // Get all variables from the json array and remove the $ prefix, return empty list if no variables are found - var usedVariables = llmJsonResponse["variables"]?.Deserialize()? - .Where(v => !string.IsNullOrWhiteSpace(v)) - .Select(v => v.TrimStart('$')) - ?? Enumerable.Empty(); - - return usedVariables; - } - - /// - /// Get the variables used in the If statement condition - /// - /// If Xml Node - /// List of used variables in the if node condition attribute - /// InvalidStatementStructure - /// InvalidCondition - private IEnumerable GetUsedVariables(XmlNode ifNode) - { - var foundVariables = Regex.Matches(ifNode.Attributes!["condition"].Value, "\\$[0-9A-Za-z_]+"); - if (foundVariables.Count == 0) - { - throw new ConditionException(ConditionException.ErrorCodes.InvalidCondition, "No variables found in the condition"); - } - - foreach (Match foundVariable in foundVariables) - { - // Return the variables without the $ - yield return foundVariable.Value.Substring(1); - } - } - - /// - /// Evaluates a condition group and returns TRUE or FALSE - /// - /// If structure content - /// Used variables to send for evaluation - /// Current context - /// Condition result - /// InvalidCondition - /// ContextVariablesNotFound - private async Task EvaluateConditionAsync(XmlNode ifNode, IEnumerable usedVariables, SKContext context) - { - var conditionContent = this.ExtractConditionalContent(ifNode); - - context.Variables.Set("IfCondition", conditionContent); - context.Variables.Set("ConditionalVariables", this.GetConditionalVariablesFromContext(usedVariables, context.Variables)); - - var llmRawResponse = - (await this._evaluateConditionFunction.InvokeAsync(conditionContent, context).ConfigureAwait(false)) - .ToString(); - - JsonNode llmJsonResponse = this.GetLlmResponseAsJsonWithProperties(llmRawResponse, "valid"); - - if (llmJsonResponse is null) - { - throw new ConditionException(ConditionException.ErrorCodes.InvalidResponse, "Response is null"); - } - - var valid = llmJsonResponse["valid"]!.GetValue(); - var reason = llmJsonResponse["reason"]?.GetValue(); - - if (!valid) - { - throw new ConditionException(ConditionException.ErrorCodes.InvalidCondition, - !string.IsNullOrWhiteSpace(reason) - ? reason - : NoReasonMessage); - } - - context.Log.LogDebug("Conditional evaluation: {0}", llmJsonResponse["reason"] ?? NoReasonMessage); - - return llmJsonResponse["condition"]?.GetValue() - ?? throw new ConditionException(ConditionException.ErrorCodes.InvalidResponse, "Condition property null or not found"); - } - - /// - /// Extracts the condition root group content closest the If structure - /// - /// If node to extract condition from - /// Condition group contents - private string ExtractConditionalContent(XmlNode ifNode) - { - var conditionContent = ifNode.Attributes?["condition"] - ?? throw new ConditionException(ConditionException.ErrorCodes.InvalidCondition, " has no condition attribute"); - - return conditionContent.Value; - } - - /// - /// Gets all the variables used in the condition and their values from the context - /// - /// Variables used in the condition - /// Context variables - /// List of variables and its values for prompting - /// ContextVariablesNotFound - private string GetConditionalVariablesFromContext(IEnumerable usedVariables, ContextVariables variables) - { - var checkNotFoundVariables = usedVariables.Where(u => !variables.ContainsKey(u)).ToArray(); - var existingVariables = variables.Where(v => usedVariables.Contains(v.Key)); - - var conditionalVariables = new StringBuilder(); - foreach (var v in existingVariables) - { - // Numeric don't add quotes - var value = Regex.IsMatch(v.Value, "^[0-9.,]+$") ? v.Value : JsonSerializer.Serialize(v.Value); - conditionalVariables.AppendLine($"{v.Key} = {value}"); - } - - foreach (string notFoundVariable in checkNotFoundVariables) - { - conditionalVariables.AppendLine($"{notFoundVariable} = undefined"); - } - - return conditionalVariables.ToString(); - } - - /// - /// Gets a JsonNode traversable structure from the LLM text response - /// - /// String to parse into a JsonNode format - /// If provided ensures if the json object has the properties - /// JsonNode with the parseable json form the llmResponse string - /// Throws if cannot find a Json result or any of the required properties - private JsonNode GetLlmResponseAsJsonWithProperties(string llmResponse, params string[] requiredProperties) - { - var startIndex = llmResponse?.IndexOf('{') ?? -1; - JsonNode? response = null; - - if (startIndex > -1) - { - var jsonResponse = llmResponse!.Substring(startIndex); - response = JsonSerializer.Deserialize(jsonResponse); - - foreach (string requiredProperty in requiredProperties) - { - _ = response?[requiredProperty] - ?? throw new ConditionException(ConditionException.ErrorCodes.InvalidResponse, - $"Response doesn't have the required property: {requiredProperty}"); - } - } - - if (response is null) - { - throw new ConditionException(ConditionException.ErrorCodes.InvalidResponse); - } - - return response; - } -} diff --git a/dotnet/src/SemanticKernel/Planning/FunctionFlowRunner.cs b/dotnet/src/SemanticKernel/Planning/FunctionFlowRunner.cs deleted file mode 100644 index 6301640f162d..000000000000 --- a/dotnet/src/SemanticKernel/Planning/FunctionFlowRunner.cs +++ /dev/null @@ -1,400 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using System.Xml; -using Microsoft.Extensions.Logging; -using Microsoft.SemanticKernel.AI.TextCompletion; -using Microsoft.SemanticKernel.Diagnostics; -using Microsoft.SemanticKernel.Orchestration; -using Microsoft.SemanticKernel.Text; - -namespace Microsoft.SemanticKernel.Planning; - -/// -/// Executes XML plans created by the Function Flow semantic function. -/// -internal class FunctionFlowRunner -{ - /// - /// The tag name used in the plan xml for the user's goal/ask. - /// - internal const string GoalTag = "goal"; - - /// - /// The tag name used in the plan xml for the solution. - /// - internal const string PlanTag = "plan"; - - /// - /// The tag name used in the plan xml for a step that calls a skill function. - /// - internal const string FunctionTag = "function."; - - /// - /// The tag name used in the plan xml for a conditional check - /// - internal const string ConditionIfTag = "if"; - - /// - /// The tag name used in the plan xml for a conditional check - /// - internal const string ConditionElseTag = "else"; - - /// - /// The tag name used in the plan xml for a conditional check - /// - internal const string ConditionWhileTag = "while"; - - /// - /// The attribute tag used in the plan xml for setting the context variable name to set the output of a function to. - /// - internal const string SetContextVariableTag = "setContextVariable"; - - /// - /// The attribute tag used in the plan xml for appending the output of a function to the final result for a plan. - /// - internal const string AppendToResultTag = "appendToResult"; - - private readonly IKernel _kernel; - - private readonly ConditionalFlowHelper _conditionalFlowHelper; - - private static readonly string[] s_functionTagArray = new string[] { FunctionTag }; - - public FunctionFlowRunner(IKernel kernel, ITextCompletion? completionBackend = null) - { - this._kernel = kernel; - this._conditionalFlowHelper = new ConditionalFlowHelper(kernel, completionBackend); - } - - /// - /// Executes the next step of a plan xml. - /// - /// The context to execute the plan in. - /// The plan xml. - /// The resulting plan xml after executing a step in the plan. - /// - /// Brief overview of how it works: - /// 1. The Solution xml is parsed into an XmlDocument. - /// 2. The Goal node is extracted from the plan xml. - /// 3. The Plan node is extracted from the plan xml. - /// 4. The first function node in the plan node is processed. - /// 5. The resulting plan xml is returned. - /// - /// Thrown when the plan xml is invalid. - public async Task ExecuteXmlPlanAsync(SKContext context, string planPayload) - { - try - { - XmlDocument solutionXml = new(); - try - { - solutionXml.LoadXml("" + planPayload + ""); - } - catch (XmlException e) - { - throw new PlanningException(PlanningException.ErrorCodes.InvalidPlan, "Failed to parse plan xml.", e); - } - - // Get the Goal - var (goalTxt, goalXmlString) = GatherGoal(solutionXml); - - // Get the Solution - XmlNodeList planNodes = solutionXml.GetElementsByTagName(PlanTag); - - // Prepare content for the new plan xml - var planContent = new StringBuilder(); - _ = planContent.AppendLine($"<{PlanTag}>"); - - // Use goal as default function {{INPUT}} -- check and see if it's a plan in Input, if so, use goalTxt, otherwise, use the input. - if (!context.Variables.Get("PLAN__INPUT", out var planInput)) - { - // planInput should then be the context.Variables.ToString() only if it's not a plan json - try - { - var plan = SkillPlan.FromJson(context.Variables.ToString()); - planInput = string.IsNullOrEmpty(plan.Goal) ? context.Variables.ToString() : goalTxt; - } - catch (Exception e) when (!e.IsCriticalException()) - { - planInput = context.Variables.ToString(); - } - } - - string functionInput = string.IsNullOrEmpty(planInput) ? goalTxt : planInput; - - // - // Process Solution nodes - // - context.Log.LogDebug("Processing solution"); - - // Process the solution nodes - string stepResults = await this.ProcessNodeListAsync(planNodes, functionInput, context); - // Add the solution and variable updates to the new plan xml - _ = planContent.Append(stepResults) - .AppendLine($""); - // Update the plan xml - var updatedPlan = goalXmlString + planContent.Replace("\r\n", "\n"); - updatedPlan = updatedPlan.Trim(); - - context.Variables.Set(SkillPlan.PlanKey, updatedPlan); - context.Variables.Set("PLAN__INPUT", context.Variables.ToString()); - - return context; - } - catch (Exception e) when (!e.IsCriticalException()) - { - context.Log.LogError(e, "Plan execution failed: {0}", e.Message); - throw; - } - } - - private async Task ProcessNodeListAsync(XmlNodeList nodeList, string functionInput, SKContext context) - { - var stepAndTextResults = new StringBuilder(); - var processFunctions = true; - const string INDENT = " "; - foreach (XmlNode o in nodeList) - { - var parentNodeName = o.Name; - var ignoreElse = false; - context.Log.LogTrace("{0}: found node", parentNodeName); - foreach (XmlNode o2 in o.ChildNodes) - { - if (o2.Name == "#text") - { - context.Log.LogTrace("{0}: appending text node", parentNodeName); - if (o2.Value != null) - { - _ = stepAndTextResults.AppendLine(o2.Value.Trim()); - } - - continue; - } - - if (o2.Name.StartsWith(ConditionElseTag, StringComparison.OrdinalIgnoreCase)) - { - //If else is the first node throws - if (o2.PreviousSibling == null) - { - throw new PlanningException(PlanningException.ErrorCodes.InvalidPlan, "ELSE tag cannot be the first node in the plan."); - } - - if (ignoreElse) - { - ignoreElse = false; - - context.Log.LogTrace("{0}: Skipping processed If's else tag from appending to the plan", parentNodeName); - - //Continue here will avoid adding this else to the next iteration of the plan - continue; - } - } - - if (processFunctions && o2.Name.StartsWith(ConditionWhileTag, StringComparison.OrdinalIgnoreCase)) - { - context.Log.LogTrace("{0}: found WHILE tag node", parentNodeName); - var whileContent = o2.OuterXml; - - var functionVariables = context.Variables.Clone(); - functionVariables.Update(whileContent); - - var branchWhile = await this._conditionalFlowHelper.WhileAsync(whileContent, - new SKContext(functionVariables, this._kernel.Memory, this._kernel.Skills, this._kernel.Log, - context.CancellationToken)); - - _ = stepAndTextResults.Append(INDENT).AppendLine(branchWhile); - - processFunctions = false; - - // We need to continue so we don't ignore any next siblings to while tag - continue; - } - - if (processFunctions && o2.Name.StartsWith(ConditionIfTag, StringComparison.OrdinalIgnoreCase)) - { - context.Log.LogTrace("{0}: found IF tag node", parentNodeName); - // Includes IF + ELSE statement - var ifFullContent = o2.OuterXml; - - //Go for the next node to see if it's an else - if (this.CheckIfNextNodeIsElseAndGetItsContents(o2, out var elseContents)) - { - ifFullContent += elseContents; - - // Ignore the next immediate sibling else tag from this IF to the plan since we already processed it - ignoreElse = true; - } - - var functionVariables = context.Variables.Clone(); - functionVariables.Update(ifFullContent); - - var branchIfOrElse = await this._conditionalFlowHelper.IfAsync(ifFullContent, - new SKContext(functionVariables, this._kernel.Memory, this._kernel.Skills, this._kernel.Log, - context.CancellationToken)); - - _ = stepAndTextResults.Append(INDENT).AppendLine(branchIfOrElse); - - processFunctions = false; - - // We need to continue so we don't ignore any next siblings - continue; - } - - if (o2.Name.StartsWith(FunctionTag, StringComparison.OrdinalIgnoreCase)) - { - var splits = o2.Name.Split(s_functionTagArray, StringSplitOptions.None); - string skillFunctionName = (splits.Length > 1) ? splits[1] : string.Empty; - context.Log.LogTrace("{0}: found skill node {1}", parentNodeName, skillFunctionName); - GetSkillFunctionNames(skillFunctionName, out var skillName, out var functionName); - if (!context.IsFunctionRegistered(skillName, functionName, out var skillFunction)) - { - throw new PlanningException(PlanningException.ErrorCodes.InvalidPlan, - $"Plan is using an unavailable skill: {skillName}.{functionName}"); - } - - if (processFunctions && !string.IsNullOrEmpty(functionName)) - { - Verify.NotNull(functionName, nameof(functionName)); - Verify.NotNull(skillFunction, nameof(skillFunction)); - context.Log.LogTrace("{0}: processing function {1}.{2}", parentNodeName, skillName, functionName); - - var functionVariables = new ContextVariables(functionInput); - var variableTargetName = string.Empty; - var appendToResultName = string.Empty; - if (o2.Attributes is not null) - { - foreach (XmlAttribute attr in o2.Attributes) - { - context.Log.LogTrace("{0}: processing attribute {1}", parentNodeName, attr.ToString()); - bool innerTextStartWithSign = attr.InnerText.StartsWith("$", StringComparison.Ordinal); - - if (attr.Name.Equals(SetContextVariableTag, StringComparison.OrdinalIgnoreCase)) - { - variableTargetName = innerTextStartWithSign - ? attr.InnerText.Substring(1) - : attr.InnerText; - } - else if (innerTextStartWithSign) - { - // Split the attribute value on the comma or ; character - var attrValues = attr.InnerText.Split(new char[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries); - if (attrValues.Length > 0) - { - // If there are multiple values, create a list of the values - var attrValueList = new List(); - foreach (var attrValue in attrValues) - { - if (context.Variables.Get(attrValue.Substring(1), out var variableReplacement)) - { - attrValueList.Add(variableReplacement); - } - } - - if (attrValueList.Count > 0) - { - functionVariables.Set(attr.Name, string.Concat(attrValueList)); - } - } - } - else if (attr.Name.Equals(AppendToResultTag, StringComparison.OrdinalIgnoreCase)) - { - appendToResultName = attr.InnerText; - } - else - { - functionVariables.Set(attr.Name, attr.InnerText); - } - } - } - - // capture current keys before running function - var keysToIgnore = functionVariables.Select(x => x.Key).ToList(); - var result = await this._kernel.RunAsync(functionVariables, skillFunction); - // TODO respect ErrorOccurred - - // copy all values for VariableNames in functionVariables not in keysToIgnore to context.Variables - foreach (var variable in functionVariables) - { - if (!keysToIgnore.Contains(variable.Key, StringComparer.OrdinalIgnoreCase) - && functionVariables.Get(variable.Key, out var value)) - { - context.Variables.Set(variable.Key, value); - } - } - - _ = context.Variables.Update(result.ToString()); - if (!string.IsNullOrEmpty(variableTargetName)) - { - context.Variables.Set(variableTargetName, result.ToString()); - } - - if (!string.IsNullOrEmpty(appendToResultName)) - { - _ = context.Variables.Get(SkillPlan.ResultKey, out var resultsSoFar); - context.Variables.Set(SkillPlan.ResultKey, - string.Join(Environment.NewLine + Environment.NewLine, resultsSoFar, appendToResultName, result.ToString()).Trim()); - } - - processFunctions = false; - } - else - { - context.Log.LogTrace("{0}: appending function node {1}", parentNodeName, skillFunctionName); - _ = stepAndTextResults.Append(INDENT).AppendLine(o2.OuterXml); - } - - continue; - } - - _ = stepAndTextResults.Append(INDENT).AppendLine(o2.OuterXml); - } - } - - return stepAndTextResults.Replace("\r\n", "\n").ToString(); - } - - private bool CheckIfNextNodeIsElseAndGetItsContents(XmlNode ifNode, out string? elseContents) - { - elseContents = null; - if (ifNode.NextSibling is null) - { - return false; - } - - if (!ifNode.NextSibling.Name.Equals("else", StringComparison.OrdinalIgnoreCase)) - { - return false; - } - - elseContents = ifNode.NextSibling.OuterXml; - return true; - } - - private static (string goalTxt, string goalXmlString) GatherGoal(XmlDocument xmlDoc) - { - XmlNodeList goal = xmlDoc.GetElementsByTagName(GoalTag); - if (goal.Count == 0) - { - throw new PlanningException(PlanningException.ErrorCodes.InvalidPlan, "No goal found."); - } - - string goalTxt = goal[0]!.FirstChild!.Value ?? string.Empty; - var goalContent = new StringBuilder(); - _ = goalContent.Append($"<{GoalTag}>") - .Append(goalTxt) - .AppendLine($""); - return (goalTxt.Trim(), goalContent.Replace("\r\n", "\n").ToString().Trim()); - } - - private static void GetSkillFunctionNames(string skillFunctionName, out string skillName, out string functionName) - { - var skillFunctionNameParts = skillFunctionName.Split('.'); - skillName = skillFunctionNameParts?.Length > 0 ? skillFunctionNameParts[0] : string.Empty; - functionName = skillFunctionNameParts?.Length > 1 ? skillFunctionNameParts[1] : skillFunctionName; - } -} diff --git a/dotnet/src/SemanticKernel/Planning/Planners/PlannerConfig.cs b/dotnet/src/SemanticKernel/Planning/Planners/PlannerConfig.cs new file mode 100644 index 000000000000..7300e3385b6e --- /dev/null +++ b/dotnet/src/SemanticKernel/Planning/Planners/PlannerConfig.cs @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; + +namespace Microsoft.SemanticKernel.Planning.Planners; + +/// +/// Common configuration for planner instances. +/// +public sealed class PlannerConfig +{ + /// + /// The minimum relevancy score for a function to be considered + /// + /// + /// Depending on the embeddings engine used, the user ask, the step goal + /// and the functions available, this value may need to be adjusted. + /// For default, this is set to null to exhibit previous behavior. + /// + public double? RelevancyThreshold { get; set; } + + /// + /// The maximum number of relevant functions to include in the plan. + /// + /// + /// Limits the number of relevant functions as result of semantic + /// search included in the plan creation request. + /// will be included + /// in the plan regardless of this limit. + /// + public int MaxRelevantFunctions { get; set; } = 100; + + /// + /// A list of skills to exclude from the plan creation request. + /// + public HashSet ExcludedSkills { get; } = new() { }; + + /// + /// A list of functions to exclude from the plan creation request. + /// + public HashSet ExcludedFunctions { get; } = new() { }; + + /// + /// A list of functions to include in the plan creation request. + /// + public HashSet IncludedFunctions { get; } = new() { "BucketOutputs" }; + + /// + /// The maximum number of tokens to allow in a plan. + /// + public int MaxTokens { get; set; } = 1024; +} diff --git a/dotnet/src/SemanticKernel/Planning/Planners/SemanticFunctionConstants.cs b/dotnet/src/SemanticKernel/Planning/Planners/SemanticFunctionConstants.cs new file mode 100644 index 000000000000..47b381691337 --- /dev/null +++ b/dotnet/src/SemanticKernel/Planning/Planners/SemanticFunctionConstants.cs @@ -0,0 +1,148 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.SemanticKernel.Planning.Planners; + +internal static class SemanticFunctionConstants +{ + internal const string FunctionFlowFunctionDefinition = + @"Create an XML plan step by step, to satisfy the goal given. +To create a plan, follow these steps: +0. The plan should be as short as possible. +1. From a create a as a series of . +2. Before using any function in a plan, check that it is present in the most recent [AVAILABLE FUNCTIONS] list. If it is not, do not use it. Do not assume that any function that was previously defined or used in another plan or in [EXAMPLES] is automatically available or compatible with the current plan. +3. Only use functions that are required for the given goal. +4. A function has an 'input' and an 'output'. +5. The 'output' from each function is automatically passed as 'input' to the subsequent . +6. 'input' does not need to be specified if it consumes the 'output' of the previous function. +7. To save an 'output' from a , to pass into a future , use ""/> +8. To save an 'output' from a , to return as part of a plan result, use ""/> +9. Append an ""END"" XML comment at the end of the plan. + +[EXAMPLES] +[AVAILABLE FUNCTIONS] + + _GLOBAL_FUNCTIONS_.BucketOutputs: + description: When the output of a function is too big, parse the output into a number of buckets. + inputs: + - input: The output from a function that needs to be parse into buckets. + - bucketCount: The number of buckets. + - bucketLabelPrefix: The target label prefix for the resulting buckets. Result will have index appended e.g. bucketLabelPrefix='Result' => Result_1, Result_2, Result_3 + + EmailConnector.LookupContactEmail: + description: looks up the a contact and retrieves their email address + inputs: + - input: the name to look up + + EmailConnector.EmailTo: + description: email the input text to a recipient + inputs: + - input: the text to email + - recipient: the recipient's email address. Multiple addresses may be included if separated by ';'. + + LanguageHelpers.TranslateTo: + description: translate the input to another language + inputs: + - input: the text to translate + - translate_to_language: the language to translate to + + WriterSkill.Summarize: + description: summarize input text + inputs: + - input: the text to summarize + +[END AVAILABLE FUNCTIONS] + +Summarize the input, then translate to japanese and email it to Martin + + + + + + + +[AVAILABLE FUNCTIONS] + + _GLOBAL_FUNCTIONS_.BucketOutputs: + description: When the output of a function is too big, parse the output into a number of buckets. + inputs: + - input: The output from a function that needs to be parse into buckets. + - bucketCount: The number of buckets. + - bucketLabelPrefix: The target label prefix for the resulting buckets. Result will have index appended e.g. bucketLabelPrefix='Result' => Result_1, Result_2, Result_3 + + _GLOBAL_FUNCTIONS_.GetEmailAddress: + description: Gets email address for given contact + inputs: + - input: the name to look up + + _GLOBAL_FUNCTIONS_.SendEmail: + description: email the input text to a recipient + inputs: + - input: the text to email + - recipient: the recipient's email address. Multiple addresses may be included if separated by ';'. + + AuthorAbility.Summarize: + description: summarizes the input text + inputs: + - input: the text to summarize + + Magician.TranslateTo: + description: translate the input to another language + inputs: + - input: the text to translate + - translate_to_language: the language to translate to + +[END AVAILABLE FUNCTIONS] + +Summarize an input, translate to french, and e-mail to John Doe + + + + + + + +[AVAILABLE FUNCTIONS] + + _GLOBAL_FUNCTIONS_.BucketOutputs: + description: When the output of a function is too big, parse the output into a number of buckets. + inputs: + - input: The output from a function that needs to be parse into buckets. + - bucketCount: The number of buckets. + - bucketLabelPrefix: The target label prefix for the resulting buckets. Result will have index appended e.g. bucketLabelPrefix='Result' => Result_1, Result_2, Result_3 + + _GLOBAL_FUNCTIONS_.NovelOutline : + description: Outlines the input text as if it were a novel + inputs: + - input: the title of the novel to outline + - chapterCount: the number of chapters to outline + + Emailer.EmailTo: + description: email the input text to a recipient + inputs: + - input: the text to email + - recipient: the recipient's email address. Multiple addresses may be included if separated by ';'. + + Everything.Summarize: + description: summarize input text + inputs: + - input: the text to summarize + +[END AVAILABLE FUNCTIONS] + +Create an outline for a children's book with 3 chapters about a group of kids in a club and then summarize it. + + + + + +[END EXAMPLES] + +[AVAILABLE FUNCTIONS] + +{{$available_functions}} + +[END AVAILABLE FUNCTIONS] + +{{$input}} +"; +} diff --git a/dotnet/src/SemanticKernel/Planning/Planners/SequentialPlanner.cs b/dotnet/src/SemanticKernel/Planning/Planners/SequentialPlanner.cs new file mode 100644 index 000000000000..2e2764978977 --- /dev/null +++ b/dotnet/src/SemanticKernel/Planning/Planners/SequentialPlanner.cs @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Threading.Tasks; +using Microsoft.SemanticKernel.Diagnostics; +using Microsoft.SemanticKernel.Orchestration; + +namespace Microsoft.SemanticKernel.Planning.Planners; + +/// +/// A planner that uses semantic function to create a sequential plan. +/// +public class SequentialPlanner +{ + /// + /// Initialize a new instance of the class. + /// + /// The semantic kernel instance. + /// The planner configuration. + public SequentialPlanner(IKernel? kernel, PlannerConfig? config = null) + { + Verify.NotNull(kernel, $"{this.GetType().FullName} requires a kernel instance."); + this.Config = config ?? new(); + + this.Config.ExcludedSkills.Add(RestrictedSkillName); + + this._functionFlowFunction = kernel.CreateSemanticFunction( + promptTemplate: SemanticFunctionConstants.FunctionFlowFunctionDefinition, + skillName: RestrictedSkillName, + description: "Given a request or command or goal generate a step by step plan to " + + "fulfill the request using functions. This ability is also known as decision making and function flow", + maxTokens: this.Config.MaxTokens, + temperature: 0.0, + stopSequences: new[] { " <_Parameter1>SemanticKernel.UnitTests diff --git a/samples/apps/auth-api-webapp-react/src/components/TaskButton.tsx b/samples/apps/auth-api-webapp-react/src/components/TaskButton.tsx index 064569b239d3..bce2b62d1d1c 100644 --- a/samples/apps/auth-api-webapp-react/src/components/TaskButton.tsx +++ b/samples/apps/auth-api-webapp-react/src/components/TaskButton.tsx @@ -32,29 +32,19 @@ const TaskButton: FC = ({ taskDescription, onTaskComplete, keyConfig, uri setIsBusy(true); try { - var createPlanResult = await sk.invokeAsync( - keyConfig, - { - value: `${input}\n${taskDescription}`, - skills: skills, - }, - 'plannerskill', - 'createplanasync', - ); + var createPlanResult = await sk.createPlanAsync(keyConfig, { + value: `${input}\n${taskDescription}`, + skills: skills, + }); console.log(createPlanResult); var inputs: IAskInput[] = [...createPlanResult.state]; - var executePlanResult = await sk.invokeAsync( - keyConfig, - { - inputs: inputs, - value: createPlanResult.value, - }, - 'plannerskill', - 'executeplanasync', - ); + var executePlanResult = await sk.executePlanAsync(keyConfig, { + inputs: inputs, + value: createPlanResult.value, + }); onTaskComplete(executePlanResult); } catch (e) { diff --git a/samples/apps/auth-api-webapp-react/src/hooks/SemanticKernel.ts b/samples/apps/auth-api-webapp-react/src/hooks/SemanticKernel.ts index 90d8d7488d83..5ef21d57d005 100644 --- a/samples/apps/auth-api-webapp-react/src/hooks/SemanticKernel.ts +++ b/samples/apps/auth-api-webapp-react/src/hooks/SemanticKernel.ts @@ -41,6 +41,16 @@ export class SemanticKernel { return result; }; + public createPlanAsync = async (keyConfig: IKeyConfig, ask: IAsk): Promise => { + const result = await this.getResponseAsync({ + commandPath: `/api/planner/createPlan`, + method: 'POST', + body: ask, + keyConfig: keyConfig, + }); + return result; + }; + public executePlanAsync = async (keyConfig: IKeyConfig, ask: IAsk, maxSteps: number = 10): Promise => { const result = await this.getResponseAsync({ commandPath: `/api/planner/execute/${maxSteps}`, diff --git a/samples/apps/book-creator-webapp-react/src/components/CreateBookWithPlanner.tsx b/samples/apps/book-creator-webapp-react/src/components/CreateBookWithPlanner.tsx index 034b4b88a968..0d0fe9fb4842 100644 --- a/samples/apps/book-creator-webapp-react/src/components/CreateBookWithPlanner.tsx +++ b/samples/apps/book-creator-webapp-react/src/components/CreateBookWithPlanner.tsx @@ -11,7 +11,7 @@ import { Spinner, Subtitle1, Subtitle2, - Title3 + Title3, } from '@fluentui/react-components'; import { Book24Regular, Code24Regular, Thinking24Regular } from '@fluentui/react-icons'; import { FC, useState } from 'react'; @@ -154,7 +154,7 @@ const CreateBookWithPlanner: FC = ({ uri, title, description, keyConfig, functionName: 'createplan', input: JSON.stringify(ask), timestamp: new Date().toTimeString(), - uri: '/api/skills/plannerskill/invoke/createplanasync', + uri: '/api/planner/createplan', }; setProcessHistory((processHistory) => [...processHistory, historyItem]); }; diff --git a/samples/apps/book-creator-webapp-react/src/hooks/SemanticKernel.ts b/samples/apps/book-creator-webapp-react/src/hooks/SemanticKernel.ts index 90d8d7488d83..d25073fff8b3 100644 --- a/samples/apps/book-creator-webapp-react/src/hooks/SemanticKernel.ts +++ b/samples/apps/book-creator-webapp-react/src/hooks/SemanticKernel.ts @@ -41,6 +41,16 @@ export class SemanticKernel { return result; }; + public createPlanAsync = async (keyConfig: IKeyConfig, ask: IAsk): Promise => { + const result = await this.getResponseAsync({ + commandPath: `/api/planner/createplan`, + method: 'POST', + body: ask, + keyConfig: keyConfig, + }); + return result; + }; + public executePlanAsync = async (keyConfig: IKeyConfig, ask: IAsk, maxSteps: number = 10): Promise => { const result = await this.getResponseAsync({ commandPath: `/api/planner/execute/${maxSteps}`, diff --git a/samples/apps/book-creator-webapp-react/src/hooks/TaskRunner.ts b/samples/apps/book-creator-webapp-react/src/hooks/TaskRunner.ts index 0ebf4df0d708..6b0d74a92915 100644 --- a/samples/apps/book-creator-webapp-react/src/hooks/TaskRunner.ts +++ b/samples/apps/book-creator-webapp-react/src/hooks/TaskRunner.ts @@ -32,7 +32,7 @@ export class TaskRunner { skills: skills, }; - var createPlanResult = await this.sk.invokeAsync(this.keyConfig, createPlanAsk, 'plannerskill', 'createplan'); + var createPlanResult = await this.sk.createPlanAsync(this.keyConfig, createPlanAsk); onPlanCreated?.(createPlanAsk, createPlanResult.value); diff --git a/samples/apps/chat-summary-webapp-react/src/hooks/SemanticKernel.ts b/samples/apps/chat-summary-webapp-react/src/hooks/SemanticKernel.ts index 90d8d7488d83..d25073fff8b3 100644 --- a/samples/apps/chat-summary-webapp-react/src/hooks/SemanticKernel.ts +++ b/samples/apps/chat-summary-webapp-react/src/hooks/SemanticKernel.ts @@ -41,6 +41,16 @@ export class SemanticKernel { return result; }; + public createPlanAsync = async (keyConfig: IKeyConfig, ask: IAsk): Promise => { + const result = await this.getResponseAsync({ + commandPath: `/api/planner/createplan`, + method: 'POST', + body: ask, + keyConfig: keyConfig, + }); + return result; + }; + public executePlanAsync = async (keyConfig: IKeyConfig, ask: IAsk, maxSteps: number = 10): Promise => { const result = await this.getResponseAsync({ commandPath: `/api/planner/execute/${maxSteps}`, diff --git a/samples/apps/github-qna-webapp-react/src/hooks/SemanticKernel.ts b/samples/apps/github-qna-webapp-react/src/hooks/SemanticKernel.ts index 80c5400722ad..c1ca2d64ba91 100644 --- a/samples/apps/github-qna-webapp-react/src/hooks/SemanticKernel.ts +++ b/samples/apps/github-qna-webapp-react/src/hooks/SemanticKernel.ts @@ -3,7 +3,16 @@ import { IAsk } from '../model/Ask'; import { IAskResult } from '../model/AskResult'; import { - IKeyConfig, SK_HTTP_HEADER_COMPLETION_BACKEND, SK_HTTP_HEADER_COMPLETION_ENDPOINT, SK_HTTP_HEADER_COMPLETION_KEY, SK_HTTP_HEADER_COMPLETION_MODEL, SK_HTTP_HEADER_EMBEDDING_BACKEND, SK_HTTP_HEADER_EMBEDDING_ENDPOINT, SK_HTTP_HEADER_EMBEDDING_KEY, SK_HTTP_HEADER_EMBEDDING_MODEL, SK_HTTP_HEADER_MSGRAPH + IKeyConfig, + SK_HTTP_HEADER_COMPLETION_BACKEND, + SK_HTTP_HEADER_COMPLETION_ENDPOINT, + SK_HTTP_HEADER_COMPLETION_KEY, + SK_HTTP_HEADER_COMPLETION_MODEL, + SK_HTTP_HEADER_EMBEDDING_BACKEND, + SK_HTTP_HEADER_EMBEDDING_ENDPOINT, + SK_HTTP_HEADER_EMBEDDING_KEY, + SK_HTTP_HEADER_EMBEDDING_MODEL, + SK_HTTP_HEADER_MSGRAPH, } from '../model/KeyConfig'; interface ServiceRequest { @@ -15,7 +24,7 @@ interface ServiceRequest { export class SemanticKernel { // eslint-disable-next-line @typescript-eslint/space-before-function-paren - constructor(private readonly serviceUrl: string) { } + constructor(private readonly serviceUrl: string) {} public invokeAsync = async ( keyConfig: IKeyConfig, @@ -32,6 +41,16 @@ export class SemanticKernel { return result; }; + public createPlanAsync = async (keyConfig: IKeyConfig, ask: IAsk): Promise => { + const result = await this.getResponseAsync({ + commandPath: `/api/planner/createplan`, + method: 'POST', + body: ask, + keyConfig: keyConfig, + }); + return result; + }; + public executePlanAsync = async (keyConfig: IKeyConfig, ask: IAsk, maxSteps: number = 10): Promise => { const result = await this.getResponseAsync({ commandPath: `/api/planner/execute/${maxSteps}`, @@ -74,18 +93,19 @@ export class SemanticKernel { if (!response.ok) { // eslint-disable-next-line no-throw-literal - throw response.statusText + " => " + await response.text(); + throw response.statusText + ' => ' + (await response.text()); } return (await response.json()) as T; } catch (e) { - var additional_error_msg = '' + var additional_error_msg = ''; if (e instanceof TypeError) { // fetch() will reject with a TypeError when a network error is encountered. - additional_error_msg = '\n\nPlease check you have the function running and that it is accessible by the app' + additional_error_msg = + '\n\nPlease check you have the function running and that it is accessible by the app'; } // eslint-disable-next-line no-throw-literal throw e + additional_error_msg; } }; -} \ No newline at end of file +} diff --git a/samples/dotnet/KernelHttpServer/Extensions.cs b/samples/dotnet/KernelHttpServer/Extensions.cs index eea892606ee4..9484f50b64b2 100644 --- a/samples/dotnet/KernelHttpServer/Extensions.cs +++ b/samples/dotnet/KernelHttpServer/Extensions.cs @@ -134,12 +134,6 @@ internal static void RegisterNativeGraphSkills(this IKernel kernel, string graph } } - internal static void RegisterPlanner(this IKernel kernel) - { - PlannerSkill planner = new(kernel); - _ = kernel.ImportSkill(planner, nameof(PlannerSkill)); - } - internal static void RegisterTextMemory(this IKernel kernel) { _ = kernel.ImportSkill(new TextMemorySkill(), nameof(TextMemorySkill)); @@ -170,7 +164,7 @@ internal static void RegisterNativeSkills(this IKernel kernel, IEnumerable InvokeFunctionAsync( return r; } + [Function("CreatePlan")] + public async Task CreatePlanAsync( + [HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "planner/createplan")] + HttpRequestData req, + FunctionContext executionContext) + { + var ask = await JsonSerializer.DeserializeAsync(req.Body, new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }); + + if (ask == null) + { + return await req.CreateResponseWithMessageAsync(HttpStatusCode.BadRequest, "Invalid request, unable to parse the request payload"); + } + + var kernel = SemanticKernelFactory.CreateForRequest( + req, + executionContext.GetLogger(), + ask.Skills); + + if (kernel == null) + { + return await req.CreateResponseWithMessageAsync(HttpStatusCode.BadRequest, "Missing one or more expected HTTP Headers"); + } + + var planner = new SequentialPlanner(kernel); + var goal = ask.Value; + + var plan = await planner.CreatePlanAsync(goal); + + var r = req.CreateResponse(HttpStatusCode.OK); + await r.WriteAsJsonAsync(new AskResult { Value = plan.ToJson() }); + return r; + } + [Function("ExecutePlan")] public async Task ExecutePlanAsync( [HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "planner/execute/{maxSteps?}")] @@ -87,50 +122,45 @@ public async Task ExecutePlanAsync( var kernel = SemanticKernelFactory.CreateForRequest( req, - executionContext.GetLogger()); + executionContext.GetLogger(), + ask.Skills); if (kernel == null) { return await req.CreateResponseWithMessageAsync(HttpStatusCode.BadRequest, "Missing one or more expected HTTP Headers"); } - var contextVariables = new ContextVariables(ask.Value); - - foreach (var input in ask.Inputs) - { - contextVariables.Set(input.Key, input.Value); - } + var context = kernel.CreateNewContext(); - var planner = kernel.Skills.GetFunction("plannerskill", "executeplan"); - var result = await kernel.RunAsync(contextVariables, planner); + var plan = Plan.FromJson(ask.Value, context); var iterations = 1; - while (!result.Variables.ToPlan().IsComplete && - result.Variables.ToPlan().IsSuccessful && + while (plan.HasNextStep && iterations < maxSteps) { - result = await kernel.RunAsync(result.Variables, planner); - iterations++; - } + try + { + plan = await kernel.StepAsync(context.Variables, plan); + } + catch (KernelException e) + { + context.Fail(e.Message, e); + return await ResponseErrorWithMessageAsync(req, context); + } - if (result.ErrorOccurred) - { - return await ResponseErrorWithMessageAsync(req, result); + iterations++; } var r = req.CreateResponse(HttpStatusCode.OK); - await r.WriteAsJsonAsync(new AskResult { Value = result.Variables.ToPlan().Result }); + await r.WriteAsJsonAsync(new AskResult { Value = plan.State.ToString() }); return r; } private static async Task ResponseErrorWithMessageAsync(HttpRequestData req, SKContext result) { - if (result.LastException is AIException aiException && aiException.Detail is not null) - { - return await req.CreateResponseWithMessageAsync(HttpStatusCode.BadRequest, string.Concat(aiException.Message, " - Detail: " + aiException.Detail)); - } - - return await req.CreateResponseWithMessageAsync(HttpStatusCode.BadRequest, result.LastErrorDescription); + return result.LastException is AIException aiException && aiException.Detail is not null + ? await req.CreateResponseWithMessageAsync(HttpStatusCode.BadRequest, string.Concat(aiException.Message, " - Detail: " + aiException.Detail)) + : await req.CreateResponseWithMessageAsync(HttpStatusCode.BadRequest, result.LastErrorDescription); } } diff --git a/samples/dotnet/KernelHttpServer/SemanticKernelFactory.cs b/samples/dotnet/KernelHttpServer/SemanticKernelFactory.cs index 5ebbfb255d0b..bf927e01c60f 100644 --- a/samples/dotnet/KernelHttpServer/SemanticKernelFactory.cs +++ b/samples/dotnet/KernelHttpServer/SemanticKernelFactory.cs @@ -56,6 +56,8 @@ private static KernelBuilder _ConfigureKernelBuilder(ApiKeyConfig config, Kernel config.CompletionConfig.Endpoint, config.CompletionConfig.Key); break; + default: + break; } if (memoryStore != null && config.EmbeddingConfig.IsValid()) @@ -70,6 +72,8 @@ private static KernelBuilder _ConfigureKernelBuilder(ApiKeyConfig config, Kernel c.AddAzureTextEmbeddingGenerationService(config.EmbeddingConfig.ServiceId, config.EmbeddingConfig.DeploymentOrModelId, config.EmbeddingConfig.Endpoint, config.EmbeddingConfig.Key); break; + default: + break; } builder.WithMemoryStorage(memoryStore); @@ -83,7 +87,6 @@ private static IKernel _CompleteKernelSetup(HttpRequestData req, KernelBuilder b kernel.RegisterSemanticSkills(RepoFiles.SampleSkillsPath(), logger, skillsToLoad); kernel.RegisterNativeSkills(skillsToLoad); - kernel.RegisterPlanner(); if (req.Headers.TryGetValues(SKHttpHeaders.MSGraph, out var graphToken)) { diff --git a/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs b/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs index f9f799bebf73..193a400d549e 100644 --- a/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs +++ b/samples/dotnet/kernel-syntax-examples/Example12_Planning.cs @@ -1,14 +1,13 @@ // Copyright (c) Microsoft. All rights reserved. using System; -using System.Collections.Generic; using System.Diagnostics; using System.Threading.Tasks; using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.CoreSkills; using Microsoft.SemanticKernel.KernelExtensions; using Microsoft.SemanticKernel.Memory; using Microsoft.SemanticKernel.Orchestration; +using Microsoft.SemanticKernel.Planning.Planners; using RepoUtils; using Skills; using TextSkill = Skills.TextSkill; @@ -22,106 +21,33 @@ public static async Task RunAsync() await EmailSamplesAsync(); await BookSamplesAsync(); await MemorySampleAsync(); - await IfConditionalSampleAsync(); - await WhileConditionalSampleAsync(); - } - - private static async Task IfConditionalSampleAsync() - { - Console.WriteLine("======== Planning - If/Else Conditional flow example ========"); - var kernel = InitializeKernelAndPlanner(out var planner); - - // Load additional skills to enable planner to do non-trivial asks. - string folder = RepoFiles.SampleSkillsPath(); - kernel.ImportSemanticSkillFromDirectory(folder, "FunSkill"); - kernel.ImportSemanticSkillFromDirectory(folder, "WriterSkill"); - kernel.ImportSkill(new TimeSkill()); - - var originalPlan = await kernel.RunAsync( - @"If is still morning please give me a joke about coffee otherwise tell me one about afternoon, but if its night give me a poem about the moon", - planner["CreatePlan"]); - - /* - If is still morning please give me a joke about coffee otherwise tell me one about afternoon, but if its night give me a poem about the moon - - - - - - - - - - - - - - - */ - - Console.WriteLine("Original plan:"); - Console.WriteLine(originalPlan.Variables.ToPlan().PlanString); - - await ExecutePlanAsync(kernel, planner, originalPlan, 20); - } - - private static async Task WhileConditionalSampleAsync() - { - Console.WriteLine("======== Planning - While Loop Conditional flow example ========"); - var kernel = InitializeKernelAndPlanner(out var planner); - - // Load additional skills to enable planner to do non-trivial asks. - string folder = RepoFiles.SampleSkillsPath(); - kernel.ImportSemanticSkillFromDirectory(folder, "FunSkill"); - kernel.ImportSkill(new MathSkill()); - kernel.ImportSkill(new TimeSkill()); - - var originalPlan = await kernel.RunAsync( - @"Start with a X number equals to the current minutes of the clock and remove 20 from this number until it becomes 0. After that tell me a math style joke where the input is X number + ""bananas""", - planner["CreatePlan"]); - - /* - - Start with a X number equals to the current minutes of the clock and remove 20 from this number until it becomes 0. After that tell me a math style joke where the input is X number + "bananas" - - - - - - - - - */ - - Console.WriteLine("Original plan:"); - Console.WriteLine(originalPlan.Variables.ToPlan().PlanString); - - await ExecutePlanAsync(kernel, planner, originalPlan, 20); } private static async Task PoetrySamplesAsync() { Console.WriteLine("======== Planning - Create and Execute Poetry Plan ========"); - var kernel = InitializeKernelAndPlanner(out var planner); + var kernel = new KernelBuilder().WithLogger(ConsoleLogger.Log).Build(); + kernel.Config.AddAzureTextCompletionService( + Env.Var("AZURE_OPENAI_SERVICE_ID"), + Env.Var("AZURE_OPENAI_DEPLOYMENT_NAME"), + Env.Var("AZURE_OPENAI_ENDPOINT"), + Env.Var("AZURE_OPENAI_KEY")); - // Load additional skills to enable planner to do non-trivial asks. string folder = RepoFiles.SampleSkillsPath(); kernel.ImportSemanticSkillFromDirectory(folder, "SummarizeSkill"); kernel.ImportSemanticSkillFromDirectory(folder, "WriterSkill"); - var originalPlan = await kernel.RunAsync("Write a poem about John Doe, then translate it into Italian.", planner["CreatePlan"]); - // - // Write a poem about John Doe, then translate it into Italian. - // - // - // - // - // + var planner = new SequentialPlanner(kernel); + + var planObject = await planner.CreatePlanAsync("Write a poem about John Doe, then translate it into Italian."); Console.WriteLine("Original plan:"); - Console.WriteLine(originalPlan.Variables.ToPlan().PlanString); + Console.WriteLine(planObject.ToJson()); + + var result = await kernel.RunAsync(planObject); - await ExecutePlanAsync(kernel, planner, originalPlan, 5); + Console.WriteLine("Result:"); + Console.WriteLine(result.Result); } private static async Task EmailSamplesAsync() @@ -135,7 +61,7 @@ private static async Task EmailSamplesAsync() kernel.ImportSemanticSkillFromDirectory(folder, "SummarizeSkill"); kernel.ImportSemanticSkillFromDirectory(folder, "WriterSkill"); - var originalPlan = await kernel.RunAsync("Summarize an input, translate to french, and e-mail to John Doe", planner["CreatePlan"]); + var originalPlan = await planner.CreatePlanAsync("Summarize an input, translate to french, and e-mail to John Doe"); // // Summarize an input, translate to french, and e-mail to John Doe // @@ -147,10 +73,9 @@ private static async Task EmailSamplesAsync() // Console.WriteLine("Original plan:"); - Console.WriteLine(originalPlan.Variables.ToPlan().PlanString); + Console.WriteLine(originalPlan.ToJson()); - var executionResults = originalPlan; - executionResults.Variables.Update( + var input = "Once upon a time, in a faraway kingdom, there lived a kind and just king named Arjun. " + "He ruled over his kingdom with fairness and compassion, earning him the love and admiration of his people. " + "However, the kingdom was plagued by a terrible dragon that lived in the nearby mountains and terrorized the nearby villages, " + @@ -161,8 +86,8 @@ private static async Task EmailSamplesAsync() "she was able to strike the dragon with a single shot through its heart, killing it instantly. The people rejoiced " + "and the kingdom was at peace once again. The king was so grateful to Mira that he asked her to marry him and she agreed. " + "They ruled the kingdom together, ruling with fairness and compassion, just as Arjun had done before. They lived " + - "happily ever after, with the people of the kingdom remembering Mira as the brave young woman who saved them from the dragon."); - await ExecutePlanAsync(kernel, planner, executionResults, 5); + "happily ever after, with the people of the kingdom remembering Mira as the brave young woman who saved them from the dragon."; + await ExecutePlanAsync(kernel, originalPlan, input, 5); } private static async Task BookSamplesAsync() @@ -174,9 +99,8 @@ private static async Task BookSamplesAsync() string folder = RepoFiles.SampleSkillsPath(); kernel.ImportSemanticSkillFromDirectory(folder, "WriterSkill"); - var originalPlan = await kernel.RunAsync( - "Create a book with 3 chapters about a group of kids in a club called 'The Thinking Caps.'", - planner["CreatePlan"]); + var originalPlan = await planner.CreatePlanAsync("Create a book with 3 chapters about a group of kids in a club called 'The Thinking Caps.'"); + // // Create a book with 3 chapters about a group of kids in a club called 'The Thinking Caps.' // @@ -189,11 +113,11 @@ private static async Task BookSamplesAsync() // Console.WriteLine("Original plan:"); - Console.WriteLine(originalPlan.Variables.ToPlan().PlanString); + Console.WriteLine(originalPlan.ToJson()); Stopwatch sw = new(); sw.Start(); - await ExecutePlanAsync(kernel, planner, originalPlan); + await ExecutePlanAsync(kernel, originalPlan); } private static async Task MemorySampleAsync() @@ -220,9 +144,6 @@ private static async Task MemorySampleAsync() .WithMemoryStorage(new VolatileMemoryStore()) .Build(); - // Load native skill into the kernel skill collection, sharing its functions with prompt templates - var planner = kernel.ImportSkill(new PlannerSkill(kernel), "planning"); - string folder = RepoFiles.SampleSkillsPath(); kernel.ImportSemanticSkillFromDirectory(folder, "SummarizeSkill"); kernel.ImportSemanticSkillFromDirectory(folder, "WriterSkill"); @@ -241,16 +162,17 @@ private static async Task MemorySampleAsync() kernel.ImportSkill(new TextSkill(), "text"); kernel.ImportSkill(new Microsoft.SemanticKernel.CoreSkills.TextSkill(), "coretext"); - var context = new ContextVariables("Create a book with 3 chapters about a group of kids in a club called 'The Thinking Caps.'"); - context.Set(PlannerSkill.Parameters.RelevancyThreshold, "0.78"); + var goal = "Create a book with 3 chapters about a group of kids in a club called 'The Thinking Caps.'"; + + var planner = new SequentialPlanner(kernel, new PlannerConfig() { RelevancyThreshold = 0.78 }); - var executionResults = await kernel.RunAsync(context, planner["CreatePlan"]); + var plan = await planner.CreatePlanAsync(goal); Console.WriteLine("Original plan:"); - Console.WriteLine(executionResults.Variables.ToPlan().PlanString); + Console.WriteLine(plan.ToJson()); } - private static IKernel InitializeKernelAndPlanner(out IDictionary planner, int maxTokens = 1024) + private static IKernel InitializeKernelAndPlanner(out SequentialPlanner planner, int maxTokens = 1024) { var kernel = new KernelBuilder().WithLogger(ConsoleLogger.Log).Build(); kernel.Config.AddAzureTextCompletionService( @@ -259,55 +181,57 @@ private static IKernel InitializeKernelAndPlanner(out IDictionary ExecutePlanAsync( + private static async Task ExecutePlanAsync( IKernel kernel, - IDictionary planner, - SKContext executionResults, + Plan plan, + string input = "", int maxSteps = 10) { Stopwatch sw = new(); sw.Start(); // loop until complete or at most N steps - for (int step = 1; !executionResults.Variables.ToPlan().IsComplete && step < maxSteps; step++) + try { - var results = await kernel.RunAsync(executionResults.Variables, planner["ExecutePlan"]); - if (results.Variables.ToPlan().IsSuccessful) + for (int step = 1; plan.HasNextStep && step < maxSteps; step++) { - Console.WriteLine($"Step {step} - Execution results:"); - Console.WriteLine(results.Variables.ToPlan().PlanString); + if (string.IsNullOrEmpty(input)) + { + await plan.InvokeNextStepAsync(kernel.CreateNewContext()); + // or await kernel.StepAsync(plan); + } + else + { + plan = await kernel.StepAsync(input, plan); + } - if (results.Variables.ToPlan().IsComplete) + Console.WriteLine($"Step {step} - Execution results:"); + Console.WriteLine(plan.ToJson()); + if (!plan.HasNextStep) { Console.WriteLine($"Step {step} - COMPLETE!"); - Console.WriteLine(results.Variables.ToPlan().Result); - - // Console.WriteLine("VARIABLES: "); - // Console.WriteLine(string.Join("\n\n", results.Variables.Select(v => $"{v.Key} = {v.Value}"))); + Console.WriteLine(plan.State.ToString()); break; } - // Console.WriteLine($"Step {step} - Results so far:"); - // Console.WriteLine(results.ToPlan().Result); - } - else - { - Console.WriteLine($"Step {step} - Execution failed:"); - Console.WriteLine(results.Variables.ToPlan().Result); - break; + Console.WriteLine($"Step {step} - Results so far:"); + Console.WriteLine(plan.State.ToString()); } - - executionResults = results; + } + catch (KernelException e) + { + Console.WriteLine($"Step - Execution failed:"); + Console.WriteLine(e.Message); } sw.Stop(); Console.WriteLine($"Execution complete in {sw.ElapsedMilliseconds} ms!"); - return executionResults; + return plan; } }