Skip to content

Commit

Permalink
Fix interface method/property handling
Browse files Browse the repository at this point in the history 
- Add missing interface method/property handling when they had no access
  modifiers. Adding new unit tests
  • Loading branch information
@Martin-Molinero
Martin-Molinero committed Apr 19, 2024
1 parent 4e8b435 commit 31803ff
Show file tree
Hide file tree
Showing 3 changed files with 207 additions and 80 deletions.
93 changes: 93 additions & 0 deletions QuantConnectStubsGenerator.Tests/GeneratorTests.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp;
using NUnit.Framework;
using QuantConnectStubsGenerator.Model;
using System.Collections.Generic;
using System.Linq;

namespace QuantConnectStubsGenerator.Tests
{
[TestFixture]
public class GeneratorTests
{
[TestCase("public", true)]
[TestCase("protected", true)]
[TestCase("", false)]
[TestCase("private", false)]
public void Interfaces(string interfaceModifier, bool expected)
{
var testGenerator = new TestGenerator
{
Files = new()
{
{ "Test.cs", $@"
using System;
namespace QuantConnect.Benchmarks
{{
/// <summary>
/// Specifies how to compute a benchmark for an algorithm
/// </summary>
{interfaceModifier} interface IBenchmark
{{
/// <summary>
/// Evaluates this benchmark at the specified time
/// </summary>
/// <param name=""time"">The time to evaluate the benchmark at</param>
/// <returns>The value of the benchmark at the specified time</returns>
decimal Evaluate(DateTime time);
DateTime TestProperty {{get;}}
}}
}}" }
}
};

var result = testGenerator.GenerateModelsPublic();

var namespaces = result.GetNamespaces().ToList();
Assert.AreEqual(2, namespaces.Count);

var baseNameSpace = namespaces.Single(x => x.Name == "QuantConnect");
var benchmarksNameSpace = namespaces.Single(x => x.Name == "QuantConnect.Benchmarks");

if (!expected)
{
Assert.AreEqual(0, benchmarksNameSpace.GetClasses().Count());
return;
}
var benchmark = benchmarksNameSpace.GetClasses().Single();

Assert.AreEqual("Evaluate", benchmark.Methods.Single().Name);
Assert.IsFalse(string.IsNullOrEmpty(benchmark.Methods.Single().Summary));

Assert.AreEqual("TestProperty", benchmark.Properties.Single().Name);
Assert.IsTrue(string.IsNullOrEmpty(benchmark.Properties.Single().Summary));
}

private class TestGenerator : Generator
{
public Dictionary<string, string> Files { get; set; }
public TestGenerator() : base("/", "/", "/")
{
}

protected override IEnumerable<SyntaxTree> GetSyntaxTrees()
{
foreach (var fileContent in Files)
{
yield return CSharpSyntaxTree.ParseText(fileContent.Value, path: fileContent.Key);
}
}

public ParseContext GenerateModelsPublic()
{
ParseContext context = new();

base.GenerateModels(context);

return context;
}
}
}
}
164 changes: 88 additions & 76 deletions QuantConnectStubsGenerator/Generator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,90 @@ public Generator(string leanPath, string runtimePath, string outputDirectory)
}

public void Run()
{
// Create an empty ParseContext which will be filled with all relevant information during parsing
var context = new ParseContext();

GenerateModels(context);

// Render .pyi files containing stubs for all parsed namespaces
Logger.Info($"Generating .py and .pyi files for {context.GetNamespaces().Count()} namespaces");
foreach (var ns in context.GetNamespaces())
{
var namespacePath = ns.Name.Replace('.', '/');
var basePath = Path.GetFullPath($"{namespacePath}/__init__", _outputDirectory);

RenderNamespace(ns, basePath + ".pyi");
GeneratePyLoader(ns.Name, basePath + ".py");
CreateTypedFileForNamespace(ns.Name);
}

// Generate stubs for the clr module
GenerateClrStubs();

// Generate stubs for https://github.com/QuantConnect/Lean/blob/master/Common/AlgorithmImports.py
GenerateAlgorithmImports();

// Create setup.py
GenerateSetup();
}

protected virtual void GenerateModels(ParseContext context)
{
// Create syntax trees for all C# files
var syntaxTrees = GetSyntaxTrees().ToList();

// Create a compilation containing all syntax trees to retrieve semantic models from
var compilation = CSharpCompilation.Create("").AddSyntaxTrees(syntaxTrees);

// Add all assemblies in current project to compilation to improve semantic models
foreach (var assembly in AppDomain.CurrentDomain.GetAssemblies())
{
if (!assembly.IsDynamic && assembly.Location != "")
{
compilation = compilation.AddReferences(MetadataReference.CreateFromFile(assembly.Location));
}
}

// Parse all syntax trees using all parsers
ParseSyntaxTrees<ClassParser>(context, syntaxTrees, compilation);
ParseSyntaxTrees<PropertyParser>(context, syntaxTrees, compilation);
ParseSyntaxTrees<MethodParser>(context, syntaxTrees, compilation);

// Perform post-processing on all parsed classes
foreach (var ns in context.GetNamespaces())
{

// Remove problematic method "GetMethodInfo" from System.Reflection.RuntimeReflectionExtensions
// KW arg is `del` which python is not a fan of. TODO: Make this post filtering more generic?
if (ns.Name == "System.Reflection")
{
var reflectionClass = ns.GetClasses()
.FirstOrDefault(x => x.Type.Name == "RuntimeReflectionExtensions");
var badMethod = reflectionClass.Methods.FirstOrDefault(x => x.Name == "GetMethodInfo");

reflectionClass.Methods.Remove(badMethod);
}


foreach (var cls in ns.GetClasses())
{
// Remove Python implementations for methods where there is both a Python as well as a C# implementation
// The parsed C# implementation is usually more useful for autocomplete
// To improve it a little bit we move the return type of the Python implementation to the C# implementation
PostProcessClass(cls);

// Mark methods which appear multiple times as overloaded
MarkOverloads(cls);
}
}

// Create empty namespaces to fill gaps in between namespaces like "A.B" and "A.B.C.D"
// This is needed to make import resolution work correctly
CreateEmptyNamespaces(context);
}

protected virtual IEnumerable<SyntaxTree> GetSyntaxTrees()
{
// Lean projects not to generate stubs for
var blacklistedProjects = new[]
Expand All @@ -49,9 +133,9 @@ public void Run()
// 2. Any bin CS files
List<Regex> blacklistedRegex = new()
{
new (".*Lean\\/ADDITIONAL_STUBS\\/.*(?:DataProcessing|tests|DataQueueHandlers|Demonstration|Demostration|Algorithm)", RegexOptions.Compiled),
new (".*Lean\\/ADDITIONAL_STUBS\\/.*(?:DataProcessing|tests|DataQueueHandlers|Demonstration|Demostration|Algorithm)", RegexOptions.Compiled),
new(".*\\/bin\\/", RegexOptions.Compiled),
};
};

// Path prefixes for all blacklisted projects
var blacklistedPrefixes = blacklistedProjects
Expand Down Expand Up @@ -93,82 +177,10 @@ public void Run()

Logger.Info($"Parsing {sourceFiles.Count} C# files");

// Create syntax trees for all C# files
var syntaxTrees = sourceFiles
.Select(file => CSharpSyntaxTree.ParseText(File.ReadAllText(file), path: file))
.ToList();

// Create a compilation containing all syntax trees to retrieve semantic models from
var compilation = CSharpCompilation.Create("").AddSyntaxTrees(syntaxTrees);

// Add all assemblies in current project to compilation to improve semantic models
foreach (var assembly in AppDomain.CurrentDomain.GetAssemblies())
{
if (!assembly.IsDynamic && assembly.Location != "")
{
compilation = compilation.AddReferences(MetadataReference.CreateFromFile(assembly.Location));
}
}

// Create an empty ParseContext which will be filled with all relevant information during parsing
var context = new ParseContext();

// Parse all syntax trees using all parsers
ParseSyntaxTrees<ClassParser>(context, syntaxTrees, compilation);
ParseSyntaxTrees<PropertyParser>(context, syntaxTrees, compilation);
ParseSyntaxTrees<MethodParser>(context, syntaxTrees, compilation);

// Perform post-processing on all parsed classes
foreach (var ns in context.GetNamespaces())
{

// Remove problematic method "GetMethodInfo" from System.Reflection.RuntimeReflectionExtensions
// KW arg is `del` which python is not a fan of. TODO: Make this post filtering more generic?
if (ns.Name == "System.Reflection"){
var reflectionClass = ns.GetClasses()
.FirstOrDefault(x => x.Type.Name == "RuntimeReflectionExtensions");
var badMethod = reflectionClass.Methods.FirstOrDefault(x => x.Name == "GetMethodInfo");

reflectionClass.Methods.Remove(badMethod);
}


foreach (var cls in ns.GetClasses())
{
// Remove Python implementations for methods where there is both a Python as well as a C# implementation
// The parsed C# implementation is usually more useful for autocomplete
// To improve it a little bit we move the return type of the Python implementation to the C# implementation
PostProcessClass(cls);

// Mark methods which appear multiple times as overloaded
MarkOverloads(cls);
}
}

// Create empty namespaces to fill gaps in between namespaces like "A.B" and "A.B.C.D"
// This is needed to make import resolution work correctly
CreateEmptyNamespaces(context);

// Render .pyi files containing stubs for all parsed namespaces
Logger.Info($"Generating .py and .pyi files for {context.GetNamespaces().Count()} namespaces");
foreach (var ns in context.GetNamespaces())
foreach (var file in sourceFiles)
{
var namespacePath = ns.Name.Replace('.', '/');
var basePath = Path.GetFullPath($"{namespacePath}/__init__", _outputDirectory);

RenderNamespace(ns, basePath + ".pyi");
GeneratePyLoader(ns.Name, basePath + ".py");
CreateTypedFileForNamespace(ns.Name);
yield return CSharpSyntaxTree.ParseText(File.ReadAllText(file), path: file);
}

// Generate stubs for the clr module
GenerateClrStubs();

// Generate stubs for https://github.com/QuantConnect/Lean/blob/master/Common/AlgorithmImports.py
GenerateAlgorithmImports();

// Create setup.py
GenerateSetup();
}

/// <summary>
Expand Down
30 changes: 26 additions & 4 deletions QuantConnectStubsGenerator/Parser/BaseParser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,17 +114,39 @@ private void ExitClass()
/// </summary>
protected bool HasModifier(MemberDeclarationSyntax node, string modifier)
{
return node.Modifiers.Any(m => m.Text == modifier);
return HasModifier(node.Modifiers, modifier);
}

/// <summary>
/// Check if a node has a modifier like private or static.
/// </summary>
protected bool HasModifier(SyntaxTokenList modifiers, string modifier)
{
return modifiers.Any(m => m.Text == modifier);
}

/// <summary>
/// We skip internal or private nodes
/// </summary>
protected bool ShouldSkip(MemberDeclarationSyntax node)
{
return HasModifier(node, "private") || HasModifier(node, "internal")
// some classes don't any access modifier set, which means private
|| !HasModifier(node, "public") && !HasModifier(node, "protected");
if (HasModifier(node, "private") || HasModifier(node, "internal"))
{
return true;
}

if (node.Modifiers.Count() == 0 && node.Parent != null && node.Parent.IsKind(SyntaxKind.InterfaceDeclaration))
{
// interfaces properties/methods are public by default, so they depend on the parent really
if (node.Parent is InterfaceDeclarationSyntax interfaceDeclarationSyntax)
{
var modifiers = interfaceDeclarationSyntax.Modifiers;
return !HasModifier(modifiers, "public") && !HasModifier(modifiers, "protected");
}
return true;
}
// some classes don't any access modifier set, which means private
return !HasModifier(node, "public") && !HasModifier(node, "protected");
}

/// <summary>
Expand Down

0 comments on commit 31803ff

Please sign in to comment.