Skip to content

Commit f498e05

Browse files
dbartoldbartol
authored
Merge branch 'main' into dbartol/v1
2 parents 613ccaa + b2c64ea commit f498e05

File tree

43 files changed

+316
-224
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+316
-224
lines changed

‎.bazelrc

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@ common --override_module=semmle_code=%workspace%/misc/bazel/semmle_code_stub
1111
build --repo_env=CC=clang --repo_env=CXX=clang++
1212

1313
build:linux --cxxopt=-std=c++20
14-
build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64
14+
# we currently cannot built the swift extractor for ARM
15+
build:macos --cxxopt=-std=c++20 --copt=-arch --copt=x86_64 --linkopt=-arch --linkopt=x86_64
1516
build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor
1617

1718
# this requires developer mode, but is required to have pack installer functioning

‎csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs

Lines changed: 0 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -544,51 +544,6 @@ public void TestVcVarsAllBatFiles()
544544
Assert.Equal(2, vcvarsfiles.Length);
545545
}
546546

547-
[Fact]
548-
public void TestLinuxBuildlessExtractionSuccess()
549-
{
550-
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
551-
actions.FileExists["csharp.log"] = true;
552-
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
553-
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
554-
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
555-
actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
556-
actions.EnumerateDirectories[@"C:\Project"] = "";
557-
558-
var autobuilder = CreateAutoBuilder(false, buildless: "true");
559-
TestAutobuilderScript(autobuilder, 0, 1);
560-
}
561-
562-
[Fact]
563-
public void TestLinuxBuildlessExtractionFailed()
564-
{
565-
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 10;
566-
actions.FileExists["csharp.log"] = true;
567-
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
568-
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
569-
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
570-
actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
571-
actions.EnumerateDirectories[@"C:\Project"] = "";
572-
573-
var autobuilder = CreateAutoBuilder(false, buildless: "true");
574-
TestAutobuilderScript(autobuilder, 10, 1);
575-
}
576-
577-
[Fact]
578-
public void TestLinuxBuildlessExtractionSolution()
579-
{
580-
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
581-
actions.FileExists["csharp.log"] = true;
582-
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
583-
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
584-
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
585-
actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
586-
actions.EnumerateDirectories[@"C:\Project"] = "";
587-
588-
var autobuilder = CreateAutoBuilder(false, buildless: "true");
589-
TestAutobuilderScript(autobuilder, 0, 1);
590-
}
591-
592547
private void TestAutobuilderScript(CSharpAutobuilder autobuilder, int expectedOutput, int commandsRun)
593548
{
594549
Assert.Equal(expectedOutput, autobuilder.GetBuildScript().Run(actions, StartCallback, EndCallback));
@@ -677,21 +632,6 @@ public void TestWindowsBuildBat()
677632
TestAutobuilderScript(autobuilder, 0, 1);
678633
}
679634

680-
[Fact]
681-
public void TestSkipNugetBuildless()
682-
{
683-
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
684-
actions.FileExists["csharp.log"] = true;
685-
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
686-
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
687-
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
688-
actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
689-
actions.EnumerateDirectories[@"C:\Project"] = "";
690-
691-
var autobuilder = CreateAutoBuilder(false, buildless: "true");
692-
TestAutobuilderScript(autobuilder, 0, 1);
693-
}
694-
695635
[Fact]
696636
public void TestDotnetVersionNotInstalled()
697637
{

‎csharp/autobuilder/Semmle.Autobuild.CSharp/Semmle.Autobuild.CSharp.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
<ItemGroup>
66
<ProjectReference Include="..\..\extractor\Semmle.Util\Semmle.Util.csproj" />
77
<ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp\Semmle.Extraction.CSharp.csproj" />
8+
<ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp.Standalone\Semmle.Extraction.CSharp.Standalone.csproj" />
89
<ProjectReference Include="..\..\extractor\Semmle.Extraction.CSharp.DependencyFetching\Semmle.Extraction.CSharp.DependencyFetching.csproj" />
910
<ProjectReference Include="..\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj" />
1011
</ItemGroup>

‎csharp/autobuilder/Semmle.Autobuild.CSharp/StandaloneBuildRule.cs

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -10,17 +10,7 @@ internal class StandaloneBuildRule : IBuildRule<CSharpAutobuildOptions>
1010
{
1111
public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool auto)
1212
{
13-
if (builder.CodeQLExtractorLangRoot is null
14-
|| builder.CodeQlPlatform is null)
15-
{
16-
return BuildScript.Failure;
17-
}
18-
19-
var standalone = builder.Actions.PathCombine(builder.CodeQLExtractorLangRoot, "tools", builder.CodeQlPlatform, "Semmle.Extraction.CSharp.Standalone");
20-
var cmd = new CommandBuilder(builder.Actions);
21-
cmd.RunCommand(standalone);
22-
23-
return cmd.Script;
13+
return BuildScript.Create(_ => Semmle.Extraction.CSharp.Standalone.Program.Main([]));
2414
}
2515
}
2616
}

‎csharp/autobuilder/Semmle.Autobuild.Shared/Autobuilder.cs

Lines changed: 0 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -73,16 +73,6 @@ public interface IAutobuilder<out TAutobuildOptions> where TAutobuildOptions : A
7373
/// A logger.
7474
/// </summary>
7575
ILogger Logger { get; }
76-
77-
/// <summary>
78-
/// Value of CODEQL_EXTRACTOR_<LANG>_ROOT environment variable.
79-
/// </summary>
80-
string? CodeQLExtractorLangRoot { get; }
81-
82-
/// <summary>
83-
/// Value of CODEQL_PLATFORM environment variable.
84-
/// </summary>
85-
string? CodeQlPlatform { get; }
8676
}
8777

8878
/// <summary>
@@ -197,9 +187,6 @@ protected Autobuilder(IBuildActions actions, TAutobuildOptions options, Diagnost
197187
return ret ?? new List<IProjectOrSolution>();
198188
});
199189

200-
CodeQLExtractorLangRoot = Actions.GetEnvironmentVariable(EnvVars.Root(this.Options.Language));
201-
CodeQlPlatform = Actions.GetEnvironmentVariable(EnvVars.Platform);
202-
203190
TrapDir = RequireEnvironmentVariable(EnvVars.TrapDir(this.Options.Language));
204191
SourceArchiveDir = RequireEnvironmentVariable(EnvVars.SourceArchiveDir(this.Options.Language));
205192
DiagnosticsDir = RequireEnvironmentVariable(EnvVars.DiagnosticDir(this.Options.Language));
@@ -364,15 +351,5 @@ protected virtual void Dispose(bool disposing)
364351
diagnostics.Dispose();
365352
}
366353
}
367-
368-
/// <summary>
369-
/// Value of CODEQL_EXTRACTOR_<LANG>_ROOT environment variable.
370-
/// </summary>
371-
public string? CodeQLExtractorLangRoot { get; }
372-
373-
/// <summary>
374-
/// Value of CODEQL_PLATFORM environment variable.
375-
/// </summary>
376-
public string? CodeQlPlatform { get; }
377354
}
378355
}

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/EnvironmentVariableNames.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,11 @@ internal class EnvironmentVariableNames
6060
/// </summary>
6161
public const string FallbackNugetFeeds = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK";
6262

63+
/// <summary>
64+
/// Controls whether to include NuGet feeds from nuget.config files in the fallback restore logic.
65+
/// </summary>
66+
public const string AddNugetConfigFeedsToFallback = "CODEQL_EXTRACTOR_CSHARP_BUILDLESS_NUGET_FEEDS_FALLBACK_INCLUDE_NUGET_CONFIG_FEEDS";
67+
6368
/// <summary>
6469
/// Specifies the path to the nuget executable to be used for package restoration.
6570
/// </summary>

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs

Lines changed: 24 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -98,12 +98,14 @@ public HashSet<AssemblyLookupLocation> Restore()
9898
logger.LogInfo($"Checking NuGet feed responsiveness: {checkNugetFeedResponsiveness}");
9999
compilationInfoContainer.CompilationInfos.Add(("NuGet feed responsiveness checked", checkNugetFeedResponsiveness ? "1" : "0"));
100100

101+
HashSet<string>? explicitFeeds = null;
102+
101103
try
102104
{
103-
if (checkNugetFeedResponsiveness && !CheckFeeds())
105+
if (checkNugetFeedResponsiveness && !CheckFeeds(out explicitFeeds))
104106
{
105107
// todo: we could also check the reachability of the inherited nuget feeds, but to use those in the fallback we would need to handle authentication too.
106-
var unresponsiveMissingPackageLocation = DownloadMissingPackagesFromSpecificFeeds();
108+
var unresponsiveMissingPackageLocation = DownloadMissingPackagesFromSpecificFeeds(explicitFeeds);
107109
return unresponsiveMissingPackageLocation is null
108110
? []
109111
: [unresponsiveMissingPackageLocation];
@@ -163,7 +165,7 @@ public HashSet<AssemblyLookupLocation> Restore()
163165
LogAllUnusedPackages(dependencies);
164166

165167
var missingPackageLocation = checkNugetFeedResponsiveness
166-
? DownloadMissingPackagesFromSpecificFeeds()
168+
? DownloadMissingPackagesFromSpecificFeeds(explicitFeeds)
167169
: DownloadMissingPackages();
168170

169171
if (missingPackageLocation is not null)
@@ -173,13 +175,24 @@ public HashSet<AssemblyLookupLocation> Restore()
173175
return assemblyLookupLocations;
174176
}
175177

176-
private List<string> GetReachableFallbackNugetFeeds()
178+
private List<string> GetReachableFallbackNugetFeeds(HashSet<string>? feedsFromNugetConfigs)
177179
{
178180
var fallbackFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.FallbackNugetFeeds).ToHashSet();
179181
if (fallbackFeeds.Count == 0)
180182
{
181183
fallbackFeeds.Add(PublicNugetOrgFeed);
182-
logger.LogInfo($"No fallback Nuget feeds specified. Using default feed: {PublicNugetOrgFeed}");
184+
logger.LogInfo($"No fallback Nuget feeds specified. Adding default feed: {PublicNugetOrgFeed}");
185+
186+
var shouldAddNugetConfigFeeds = EnvironmentVariables.GetBooleanOptOut(EnvironmentVariableNames.AddNugetConfigFeedsToFallback);
187+
logger.LogInfo($"Adding feeds from nuget.config to fallback restore: {shouldAddNugetConfigFeeds}");
188+
189+
if (shouldAddNugetConfigFeeds && feedsFromNugetConfigs?.Count > 0)
190+
{
191+
// There are some feeds in `feedsFromNugetConfigs` that have already been checked for reachability, we could skip those.
192+
// But we might use different responsiveness testing settings when we try them in the fallback logic, so checking them again is safer.
193+
fallbackFeeds.UnionWith(feedsFromNugetConfigs);
194+
logger.LogInfo($"Using Nuget feeds from nuget.config files as fallback feeds: {string.Join(", ", feedsFromNugetConfigs.OrderBy(f => f))}");
195+
}
183196
}
184197

185198
logger.LogInfo($"Checking fallback Nuget feed reachability on feeds: {string.Join(", ", fallbackFeeds.OrderBy(f => f))}");
@@ -194,6 +207,8 @@ private List<string> GetReachableFallbackNugetFeeds()
194207
logger.LogInfo($"Reachable fallback Nuget feeds: {string.Join(", ", reachableFallbackFeeds.OrderBy(f => f))}");
195208
}
196209

210+
compilationInfoContainer.CompilationInfos.Add(("Reachable fallback Nuget feed count", reachableFallbackFeeds.Count.ToString()));
211+
197212
return reachableFallbackFeeds;
198213
}
199214

@@ -272,9 +287,9 @@ private void RestoreProjects(IEnumerable<string> projects, out ConcurrentBag<Dep
272287
compilationInfoContainer.CompilationInfos.Add(("Failed project restore with package source error", nugetSourceFailures.ToString()));
273288
}
274289

275-
private AssemblyLookupLocation? DownloadMissingPackagesFromSpecificFeeds()
290+
private AssemblyLookupLocation? DownloadMissingPackagesFromSpecificFeeds(HashSet<string>? feedsFromNugetConfigs)
276291
{
277-
var reachableFallbackFeeds = GetReachableFallbackNugetFeeds();
292+
var reachableFallbackFeeds = GetReachableFallbackNugetFeeds(feedsFromNugetConfigs);
278293
if (reachableFallbackFeeds.Count > 0)
279294
{
280295
return DownloadMissingPackages(fallbackNugetFeeds: reachableFallbackFeeds);
@@ -623,10 +638,10 @@ private bool IsFeedReachable(string feed, int timeoutMilliSeconds, int tryCount,
623638
return (timeoutMilliSeconds, tryCount);
624639
}
625640

626-
private bool CheckFeeds()
641+
private bool CheckFeeds(out HashSet<string> explicitFeeds)
627642
{
628643
logger.LogInfo("Checking Nuget feeds...");
629-
var (explicitFeeds, allFeeds) = GetAllFeeds();
644+
(explicitFeeds, var allFeeds) = GetAllFeeds();
630645

631646
var excludedFeeds = EnvironmentVariables.GetURLs(EnvironmentVariableNames.ExcludedNugetFeedsFromResponsivenessCheck)
632647
.ToHashSet() ?? [];

‎csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilations/Compilation.cs

Lines changed: 13 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -11,41 +11,35 @@ internal class Compilation : CachedEntity<object>
1111
{
1212
internal readonly ConcurrentDictionary<string, int> messageCounts = new();
1313

14-
private static (string Cwd, string[] Args) settings;
15-
private static int hashCode;
16-
17-
public static (string Cwd, string[] Args) Settings
18-
{
19-
get { return settings; }
20-
set
21-
{
22-
settings = value;
23-
hashCode = settings.Cwd.GetHashCode();
24-
for (var i = 0; i < settings.Args.Length; i++)
25-
{
26-
hashCode = HashCode.Combine(hashCode, settings.Args[i].GetHashCode());
27-
}
28-
}
29-
}
14+
private readonly string cwd;
15+
private readonly string[] args;
16+
private readonly int hashCode;
3017

3118
#nullable disable warnings
3219
private Compilation(Context cx) : base(cx, null)
3320
{
21+
cwd = cx.Extractor.Cwd;
22+
args = cx.Extractor.Args;
23+
hashCode = cwd.GetHashCode();
24+
for (var i = 0; i < args.Length; i++)
25+
{
26+
hashCode = HashCode.Combine(hashCode, args[i].GetHashCode());
27+
}
3428
}
3529
#nullable restore warnings
3630

3731
public override void Populate(TextWriter trapFile)
3832
{
3933
var assembly = Assembly.CreateOutputAssembly(Context);
4034

41-
trapFile.compilations(this, FileUtils.ConvertToUnix(Compilation.Settings.Cwd));
35+
trapFile.compilations(this, FileUtils.ConvertToUnix(cwd));
4236
trapFile.compilation_assembly(this, assembly);
4337

4438
// Arguments
4539
var expandedIndex = 0;
46-
for (var i = 0; i < Compilation.Settings.Args.Length; i++)
40+
for (var i = 0; i < args.Length; i++)
4741
{
48-
var arg = Compilation.Settings.Args[i];
42+
var arg = args[i];
4943
trapFile.compilation_args(this, i, arg);
5044

5145
if (CommandLineExtensions.IsFileArgument(arg))

‎csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,8 @@ public static ExitCode Run(string[] args)
9797
stopwatch.Start();
9898

9999
var options = Options.CreateWithEnvironment(args);
100-
Entities.Compilation.Settings = (Directory.GetCurrentDirectory(), options.CompilerArguments.ToArray());
100+
var workingDirectory = Directory.GetCurrentDirectory();
101+
var compilerArgs = options.CompilerArguments.ToArray();
101102

102103
using var logger = MakeLogger(options.Verbosity, options.Console);
103104

@@ -123,15 +124,15 @@ public static ExitCode Run(string[] args)
123124

124125
var compilerArguments = CSharpCommandLineParser.Default.Parse(
125126
compilerVersion.ArgsWithResponse,
126-
Entities.Compilation.Settings.Cwd,
127+
workingDirectory,
127128
compilerVersion.FrameworkPath,
128129
compilerVersion.AdditionalReferenceDirectories
129130
);
130131

131132
if (compilerArguments is null)
132133
{
133134
var sb = new StringBuilder();
134-
sb.Append(" Failed to parse command line: ").AppendList(" ", Entities.Compilation.Settings.Args);
135+
sb.Append(" Failed to parse command line: ").AppendList(" ", compilerArgs);
135136
logger.Log(Severity.Error, sb.ToString());
136137
++analyser.CompilationErrors;
137138
return ExitCode.Failed;
@@ -143,7 +144,7 @@ public static ExitCode Run(string[] args)
143144
return ExitCode.Ok;
144145
}
145146

146-
return AnalyseTracing(analyser, compilerArguments, options, canonicalPathCache, stopwatch);
147+
return AnalyseTracing(workingDirectory, compilerArgs, analyser, compilerArguments, options, canonicalPathCache, stopwatch);
147148
}
148149
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
149150
{
@@ -376,6 +377,8 @@ public static ExitCode Analyse(Stopwatch stopwatch, Analyser analyser, CommonOpt
376377
}
377378

378379
private static ExitCode AnalyseTracing(
380+
string cwd,
381+
string[] args,
379382
TracingAnalyser analyser,
380383
CSharpCommandLineArguments compilerArguments,
381384
Options options,
@@ -420,7 +423,7 @@ private static ExitCode AnalyseTracing(
420423
.WithMetadataImportOptions(MetadataImportOptions.All)
421424
);
422425
},
423-
(compilation, options) => analyser.EndInitialize(compilerArguments, options, compilation),
426+
(compilation, options) => analyser.EndInitialize(compilerArguments, options, compilation, cwd, args),
424427
() => { });
425428
}
426429

‎csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,10 @@ public StandaloneAnalyser(IProgressMonitor pm, ILogger logger, bool addAssemblyT
1616
public void Initialize(string outputPath, IEnumerable<(string, string)> compilationInfos, CSharpCompilation compilationIn, CommonOptions options)
1717
{
1818
compilation = compilationIn;
19-
extractor = new StandaloneExtractor(outputPath, compilationInfos, Logger, PathTransformer, options);
19+
extractor = new StandaloneExtractor(Directory.GetCurrentDirectory(), outputPath, compilationInfos, Logger, PathTransformer, options);
2020
this.options = options;
2121
LogExtractorInfo(Extraction.Extractor.Version);
2222
SetReferencePaths();
23-
24-
Entities.Compilation.Settings = (Directory.GetCurrentDirectory(), Array.Empty<string>());
2523
}
2624

2725
#nullable disable warnings

‎csharp/extractor/Semmle.Extraction.CSharp/Extractor/TracingAnalyser.cs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,13 +38,15 @@ public bool BeginInitialize(IEnumerable<string> roslynArgs)
3838
public void EndInitialize(
3939
CSharpCommandLineArguments commandLineArguments,
4040
CommonOptions options,
41-
CSharpCompilation compilation)
41+
CSharpCompilation compilation,
42+
string cwd,
43+
string[] args)
4244
{
4345
if (!init)
4446
throw new InternalError("EndInitialize called without BeginInitialize returning true");
4547
this.options = options;
4648
this.compilation = compilation;
47-
this.extractor = new TracingExtractor(GetOutputName(compilation, commandLineArguments), Logger, PathTransformer, options);
49+
this.extractor = new TracingExtractor(cwd, args, GetOutputName(compilation, commandLineArguments), Logger, PathTransformer, options);
4850
LogDiagnostics();
4951

5052
SetReferencePaths();

0 commit comments

Comments
 (0)