Move to OctoKit and support access tokens and include wait for rateli…

…mits (#331)

tools/Elastic.CommonSchema.Generator/CodeConfiguration.cs

@@ -2,6 +2,7 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
+using System;
 using System.IO;
 
 namespace Elastic.CommonSchema.Generator;
@@ -10,19 +11,27 @@ public static class CodeConfiguration
 {
 	static CodeConfiguration()
 	{
-		//tools/Elastic.CommonSchema.Generator/bin/Debug/net6.0
-		var directoryInfo = new DirectoryInfo(Directory.GetCurrentDirectory());
-		var rootInfo = new DirectoryInfo(Path.Combine(directoryInfo.FullName, @"../../../../../"));
-		Root = rootInfo.FullName;
+		var rootInfo = new DirectoryInfo(Directory.GetCurrentDirectory());
+		do
+		{
+			var file = new FileInfo(Path.Combine(rootInfo.FullName, "license.txt"));
+			if (file.Exists) break;
+			rootInfo = rootInfo.Parent;
+
+		} while (rootInfo != null && rootInfo != rootInfo.Root);
 
+		if (rootInfo == null)
+			throw new Exception("Can not resolve folder structure for ECS.NET codebase");
+
+		Root = rootInfo.FullName;
 		SourceFolder = Path.Combine(Root, "src");
 		ToolFolder = Path.Combine(Root, "tools");
 		ElasticCommonSchemaGeneratedFolder = Path.Combine(SourceFolder, "Elastic.CommonSchema");
 		SpecificationFolder = Path.Combine(SourceFolder, "Specification");
 		ViewFolder = Path.Combine(ToolFolder, "Elastic.CommonSchema.Generator", "Views");
 	}
 
-	private static string Root { get; }
+	public static string Root { get; }
 
 	private static string SourceFolder { get; }
 	private static string ToolFolder { get; }

tools/Elastic.CommonSchema.Generator/Elastic.CommonSchema.Generator.csproj

@@ -6,16 +6,18 @@
     <IsPackable>False</IsPackable>
     <SignAssembly>False</SignAssembly>
     <PreserveCompilationContext>true</PreserveCompilationContext>
+    <LangVersion>latest</LangVersion>
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Octokit" Version="7.1.0" />
     <PackageReference Include="RazorLight" Version="2.3.0" />
     <PackageReference Include="Cogito.Json.Schema.Validation" Version="1.0.1" />
     <PackageReference Include="CsQuery.Core" Version="2.0.1" />
     <PackageReference Include="JsonDiffPatch" Version="2.0.49" />
     <PackageReference Include="JsonDiffPatch.Net" Version="2.1.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
-    <PackageReference Include="ShellProgressBar" Version="5.0.0" />
+    <PackageReference Include="ShellProgressBar" Version="5.2.0" />
     <PackageReference Include="YamlDotNet" Version="6.0.0" />
   </ItemGroup>
 

tools/Elastic.CommonSchema.Generator/Program.cs

@@ -1,78 +1,85 @@
 using System;
+using System.IO;
 using System.Linq;
+using System.Threading.Tasks;
 using Elastic.CommonSchema.Generator.Projection;
 using Elastic.CommonSchema.Generator.Schema;
 
-namespace Elastic.CommonSchema.Generator
+namespace Elastic.CommonSchema.Generator;
+
+public static class Program
 {
-	public static class Program
+	private const string DefaultDownloadBranch = "v8.9.0";
+
+	// ReSharper disable once UnusedParameter.Local
+	private static async Task Main(string[] args)
 	{
-		private const string DefaultDownloadBranch = "v8.6.0";
+		var token = args.Length > 0 ? args[0] : string.Empty;
 
-		// ReSharper disable once UnusedParameter.Local
-		private static void Main(string[] args)
-		{
-			var redownloadCoreSpecification = true;
-			var downloadBranch = DefaultDownloadBranch;
+		Console.WriteLine($"Running from: {Directory.GetCurrentDirectory()}");
+		Console.WriteLine($"Resolved codebase root to: {CodeConfiguration.Root}");
+		Console.WriteLine();
 
-			var answer = "invalid";
-			while (answer != "y" && answer != "n" && answer != "")
-			{
-				Console.Write("Download online specifications? [Y/N] (default N): ");
-				answer = Console.ReadLine()?.Trim().ToLowerInvariant();
-				redownloadCoreSpecification = answer == "y";
-			}
+		var redownloadCoreSpecification = true;
+		var downloadBranch = DefaultDownloadBranch;
 
-			Console.Write($"Tag to use (default {downloadBranch}): ");
-			var readBranch = Console.ReadLine()?.Trim();
-			if (!string.IsNullOrEmpty(readBranch)) downloadBranch = readBranch;
+		var answer = "invalid";
+		while (answer != "y" && answer != "n" && answer != "")
+		{
+			Console.Write("Download online specifications? [Y/N] (default N): ");
+			answer = Console.ReadLine()?.Trim().ToLowerInvariant();
+			redownloadCoreSpecification = answer == "y";
+		}
 
-			if (string.IsNullOrEmpty(downloadBranch))
-				downloadBranch = DefaultDownloadBranch;
+		Console.Write($"Tag to use (default {downloadBranch}): ");
+		var readBranch = Console.ReadLine()?.Trim();
+		if (!string.IsNullOrEmpty(readBranch)) downloadBranch = readBranch;
 
-			if (redownloadCoreSpecification)
-				SpecificationDownloader.Download(downloadBranch);
+		if (string.IsNullOrEmpty(downloadBranch))
+			downloadBranch = DefaultDownloadBranch;
 
+		if (redownloadCoreSpecification)
+			await SpecificationDownloader.DownloadAsync(downloadBranch, token);
 
-			var ecsSchema = new EcsSchemaParser(downloadBranch).Parse();
-			WarnAboutSchemaValidations(ecsSchema);
 
-			var projection = new TypeProjector(ecsSchema).CreateProjection();
-			WarnAboutProjectionValidations(projection);
+		var ecsSchema = new EcsSchemaParser(downloadBranch).Parse();
+		WarnAboutSchemaValidations(ecsSchema);
 
-			FileGenerator.Generate(projection);
-		}
+		var projection = new TypeProjector(ecsSchema).CreateProjection();
+		WarnAboutProjectionValidations(projection);
 
-		private static void WarnAboutSchemaValidations(EcsSchema ecsSchema)
+		FileGenerator.Generate(projection);
+	}
+
+	private static void WarnAboutSchemaValidations(EcsSchema ecsSchema)
+	{
+		if (ecsSchema.Warnings.Count > 0)
 		{
-			if (ecsSchema.Warnings.Count > 0)
-			{
-				Console.ForegroundColor = ConsoleColor.Red;
-				Console.WriteLine("Validation errors in YAML");
-				foreach (var warning in ecsSchema.Warnings.Distinct().OrderBy(w => w))
-					Console.WriteLine(warning);
-				Console.ResetColor();
-				return;
-			}
-			Console.ForegroundColor = ConsoleColor.Green;
-			Console.WriteLine("No validation errors in YAML");
+			Console.ForegroundColor = ConsoleColor.Red;
+			Console.WriteLine("Validation errors in YAML");
+			foreach (var warning in ecsSchema.Warnings.Distinct().OrderBy(w => w))
+				Console.WriteLine(warning);
 			Console.ResetColor();
+			return;
 		}
+		Console.ForegroundColor = ConsoleColor.Green;
+		Console.WriteLine("No validation errors in YAML");
+		Console.ResetColor();
+	}
 
-		private static void WarnAboutProjectionValidations(CommonSchemaTypesProjection projection)
+	private static void WarnAboutProjectionValidations(CommonSchemaTypesProjection projection)
+	{
+		if (projection.Warnings.Count > 0)
 		{
-			if (projection.Warnings.Count > 0)
-			{
-				Console.ForegroundColor = ConsoleColor.Red;
-				Console.WriteLine("Validation errors in Canonical Model");
-				foreach (var warning in projection.Warnings.Distinct().OrderBy(w => w))
-					Console.WriteLine(warning);
-				Console.ResetColor();
-				return;
-			}
-			Console.ForegroundColor = ConsoleColor.Green;
-			Console.WriteLine("No validation errors in the Canonical Model");
+			Console.ForegroundColor = ConsoleColor.Red;
+			Console.WriteLine("Validation errors in Canonical Model");
+			foreach (var warning in projection.Warnings.Distinct().OrderBy(w => w))
+				Console.WriteLine(warning);
 			Console.ResetColor();
+			return;
 		}
+		Console.ForegroundColor = ConsoleColor.Green;
+		Console.WriteLine("No validation errors in the Canonical Model");
+		Console.ResetColor();
 	}
 }

tools/Elastic.CommonSchema.Generator/SpecificationDownloader.cs

@@ -6,27 +6,31 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using System.Net;
-using CsQuery;
+using System.Threading.Tasks;
+using Octokit;
 using ShellProgressBar;
+using static System.Text.Encoding;
 
 namespace Elastic.CommonSchema.Generator
 {
-	public class SpecificationDownloader
+	public static class SpecificationDownloader
 	{
 		private const string Core = "Core";
 		private const string Legacy = "legacy";
 		private const string Composable = "composable";
 		private const string Components = "component";
 
-		private static readonly ProgressBarOptions MainProgressBarOptions = new() { BackgroundColor = ConsoleColor.DarkGray };
+		private static readonly ProgressBarOptions MainProgressBarOptions = new()
+		{
+			BackgroundColor = ConsoleColor.DarkGray, ForegroundColorError = ConsoleColor.Red
+		};
 
 		private static readonly Dictionary<string, string> OnlineSpecifications = new()
 		{
-			{ Core, "https://github.com/elastic/ecs/tree/{version}/generated/ecs" },
-			{ Legacy, "https://github.com/elastic/ecs/tree/{version}/generated/elasticsearch/legacy" },
-			{ Composable, "https://github.com/elastic/ecs/tree/{version}/generated/elasticsearch/composable" },
-			{ Path.Combine(Composable, Components), "https://github.com/elastic/ecs/tree/{version}/generated/elasticsearch/composable/component" }
+			{ Core, "generated/ecs" },
+			{ Legacy, "generated/elasticsearch/legacy" },
+			{ Composable, "generated/elasticsearch/composable" },
+			{ Path.Combine(Composable, Components), "generated/elasticsearch/composable/component" }
 		};
 
 		private static readonly ProgressBarOptions SubProgressBarOptions = new()
@@ -37,54 +41,80 @@ public class SpecificationDownloader
 			BackgroundColor = ConsoleColor.DarkGray
 		};
 
-		public static void Download(string branch)
+		public static async Task DownloadAsync(string branch, string token)
 		{
-			var specifications =
-				(from kv in OnlineSpecifications
-					let url = kv.Value.Replace("{version}", branch)
-					select new Specification { FolderOnDisk = Path.Combine(branch, kv.Key), Branch = branch, GithubListingUrl = url })
-				.ToList();
+			var client = new GitHubClient(new ProductHeaderValue("ecs-generator"));
+			if (!string.IsNullOrEmpty(token))
+				client.Credentials = new Credentials(token);
+			using var queryProgress = new ProgressBar(OnlineSpecifications.Count, "Listing remote files", MainProgressBarOptions);
+
+			await WaitRateLimit(client, queryProgress);
+			var repo = await client.Repository.Get("elastic", "ecs");
+			var specifications = new List<Specification>();
+			foreach (var (folder, remotePath) in OnlineSpecifications)
+			{
+				await WaitRateLimit(client, queryProgress);
+				var contents = await client.Repository.Content.GetAllContentsByRef(repo.Id, remotePath, branch);
+				var spec = new Specification
+				{
+					FolderOnDisk = Path.Combine(branch, folder),
+					Branch = branch,
+					RemoteFiles = contents.Select(c => new RemoteFile(c.Name, c.Path)).ToArray()
+				};
+				specifications.Add(spec);
+			}
 
 			using var progress = new ProgressBar(specifications.Count, "Downloading specifications", MainProgressBarOptions);
 			foreach (var spec in specifications)
 			{
 				progress.Message = $"Downloading to {spec.FolderOnDisk} for branch {branch}";
-				DownloadDefinitions(spec, progress, ".yml");
-				DownloadDefinitions(spec, progress, ".json");
+				await DownloadDefinitionsAsync(spec, client, progress, ".yml", ".json");
 				progress.Tick($"Downloaded to {spec.FolderOnDisk} for branch {branch}");
 			}
 		}
 
 
-		private static void DownloadDefinitions(Specification spec, IProgressBar progress, string filenameMatch)
+		private static async Task WaitRateLimit(GitHubClient client, ProgressBar progressBar)
 		{
-			//TODO move to HttpClient
-#pragma warning disable SYSLIB0014
-#pragma warning disable CS0618
-			var client = new WebClient();
-#pragma warning restore CS0618
-#pragma warning restore SYSLIB0014
-			var html = client.DownloadString(spec.GithubListingUrl);
-			if (!Directory.Exists(CodeConfiguration.SpecificationFolder))
-				Directory.CreateDirectory(CodeConfiguration.SpecificationFolder);
+			var apiInfo = client.GetLastApiInfo();
+			var rateLimit = apiInfo?.RateLimit ?? (await client.RateLimit.GetRateLimits()).Rate;
+			if (rateLimit.Remaining > 0) return;
+			var options = new ProgressBarOptions
+			{
+				ForegroundColor = ConsoleColor.Yellow,
+				ForegroundColorDone = ConsoleColor.DarkGreen,
+				BackgroundColor = ConsoleColor.DarkGray,
+				BackgroundCharacter = '\u2593'
+			};
+			var waitTime = rateLimit.Reset - DateTimeOffset.UtcNow;
+			using var indeterminate = progressBar.SpawnIndeterminate($"Github rate limit hit, waiting: {waitTime}", options);
+			await Task.Delay(waitTime);
+			indeterminate.Finished();
+		}
 
-			var dom = CQ.Create(html);
 
-			WriteToFolder(spec.FolderOnDisk, "root.html", html);
+		private static async Task DownloadDefinitionsAsync(Specification spec, GitHubClient client, ProgressBar progress,
+			params string[] filenameMatch
+		)
+		{
+			if (!Directory.Exists(CodeConfiguration.SpecificationFolder))
+				Directory.CreateDirectory(CodeConfiguration.SpecificationFolder);
 
-			var endpoints = dom[".js-navigation-open"]
-				.Select(s => s.InnerText)
-				.Where(s => !string.IsNullOrEmpty(s) && s.EndsWith(filenameMatch))
-				.ToList();
+			var endpoints = spec.RemoteFiles.Where(r => filenameMatch.Any(m => r.FileName.EndsWith(m))).ToArray();
 
-			using var subBar = progress.Spawn(endpoints.Count, "fetching individual files", SubProgressBarOptions);
-			endpoints.ForEach(s => {
-				var rawFile = spec.GithubDownloadUrl(s);
-				var fileName = rawFile.Split('/').Last();
-				var contents = client.DownloadString(rawFile);
-				WriteToFolder(spec.FolderOnDisk, fileName, contents);
-				subBar.Tick($"Downloading {fileName}");
-			});
+			if (endpoints.Length == 0)
+			{
+				progress.WriteErrorLine($"No remote files found to download to: {spec.FolderOnDisk}");
+				return;
+			}
+			using var subBar = progress.Spawn(endpoints.Length, "fetching individual files", SubProgressBarOptions);
+			foreach (var endpoint in endpoints)
+			{
+				await WaitRateLimit(client, progress);
+				var bytes = await client.Repository.Content.GetRawContentByRef("elastic", "ecs", endpoint.Path, spec.Branch);
+				WriteToFolder(spec.FolderOnDisk, endpoint.FileName, UTF8.GetString(bytes));
+				subBar.Tick($"Downloading {endpoint.FileName}");
+			}
 		}
 
 		private static void WriteToFolder(string folder, string filename, string contents)
@@ -95,15 +125,24 @@ private static void WriteToFolder(string folder, string filename, string content
 			File.WriteAllText(path, contents);
 		}
 
+		private readonly struct RemoteFile
+		{
+			public readonly string FileName;
+			public readonly string Path;
+
+			public RemoteFile(string fileName, string path)
+			{
+				FileName = fileName;
+				Path = path;
+			}
+		}
+
 		private class Specification
 		{
 			// ReSharper disable once UnusedAutoPropertyAccessor.Local
 			public string Branch { get; set; }
 			public string FolderOnDisk { get; set; }
-			public string GithubListingUrl { get; set; }
-
-			public string GithubDownloadUrl(string file) =>
-				$"{GithubListingUrl.Replace("github.com", "raw.githubusercontent.com").Replace("tree/", "")}/{file}";
+			public RemoteFile[] RemoteFiles { get; set; }
 		}
 	}
 }