From 3b25f698747d3ec1d0466c8fb860363584e3860f Mon Sep 17 00:00:00 2001 From: Oren Novotny Date: Sun, 6 Oct 2019 21:51:53 -0400 Subject: [PATCH] Add globbing support and concurrency. --- README.md | 22 +-- src/SignClient/Globber.cs | 306 ++++++++++++++++++++++++++++++++ src/SignClient/Program.cs | 8 +- src/SignClient/SignCommand.cs | 153 ++++++++++++---- src/SignClient/appsettings.json | 4 +- 5 files changed, 441 insertions(+), 52 deletions(-) create mode 100644 src/SignClient/Globber.cs diff --git a/README.md b/README.md index f3aaf8ae..b5dd93a7 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ You'll need to create an `appsettings.json` similar to the following: } ``` -Then, somewhere in your build, you'll need to call the client tool. I use VSTS and call the following +Then, somewhere in your build, you'll need to call the client tool. I use Azure Pipelines and call the following script to sign my files. @@ -115,21 +115,13 @@ if([string]::IsNullOrEmpty($env:SignClientSecret)){ # Setup Variables we need to pass into the sign client tool $appSettings = "$currentDirectory\appsettings.json" -$nupgks = ls $currentDirectory\..\*.nupkg | Select -ExpandProperty FullName - dotnet tool install --tool-path "$currentDirectory" SignClient -foreach ($nupkg in $nupgks){ - Write-Host "Submitting $nupkg for signing" - - & "$currentDirectory\SignClient" 'sign' -c $appSettings -i $nupkg -r $env:SignClientUser -s $env:SignClientSecret -n 'Zeroconf' -d 'Zeroconf' -u 'https://github.com/onovotny/zeroconf' - if ($LASTEXITCODE -ne 0) { - exit 1 - } - Write-Host "Finished signing $nupkg" +& "$currentDirectory\SignClient" 'sign' -c $appSettings -b $Env:\ArtifactDirectory -i **/*.nupkg -r $env:SignClientUser -s $env:SignClientSecret -n 'Zeroconf' -d 'Zeroconf' -u 'https://github.com/onovotny/zeroconf' +if ($LASTEXITCODE -ne 0) { + exit 1 } -Write-Host "Sign-package complete" ``` The parameters to the signing client are as follows: @@ -148,11 +140,12 @@ After signing contents of the archive, the archive itself is signed if supported (currently `VSIX`). ``` -usage: SignClient sign [-c ] [-i ] [-o ] - [-f ] [-s ] [-n ] [-d ] [-u ] +usage: SignClient sign [-c ] [-i ] [-b ] [-o ] + [-f ] [-s ] [-n ] [-d ] [-u ] [-m ] -c, --config Path to config json file -i, --input Path to input file + -b --baseDirectory Base directory for files to override the working directory -o, --output Path to output file. May be same as input to overwrite -f, --filter Path to file containing paths of @@ -161,6 +154,7 @@ usage: SignClient sign [-c ] [-i ] [-o ] -n, --name Name of project for tracking -d, --description Description -u, --descriptionUrl Description Url + -m, --maxConcurrency Maximum concurrency (default is 4) ``` ## ClickOnce diff --git a/src/SignClient/Globber.cs b/src/SignClient/Globber.cs new file mode 100644 index 00000000..658215d8 --- /dev/null +++ b/src/SignClient/Globber.cs @@ -0,0 +1,306 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using Microsoft.Extensions.FileSystemGlobbing; +using Microsoft.Extensions.FileSystemGlobbing.Abstractions; + +// "Borrowed" from https://github.com/Wyamio/Wyam/blob/c7a26da931477a48006f6ebc574c074458719774/src/core/Wyam.Core/IO/Globbing/Globber.cs +// Copyright (c) 2017 Dave Glick +namespace Wyam.Core.IO.Globbing +{ + /// + /// Helper methods to work with globbing patterns. + /// + public static class Globber + { + static readonly Regex HasBraces = new Regex(@"\{.*\}"); + static readonly Regex NumericSet = new Regex(@"^\{(-?[0-9]+)\.\.(-?[0-9]+)\}"); + + /// + /// Gets files from the specified directory using globbing patterns. + /// + /// The directory to search. + /// The globbing pattern(s) to use. + /// Files that match the globbing pattern(s). + public static IEnumerable GetFiles(DirectoryInfo directory, params string[] patterns) => + GetFiles(directory, (IEnumerable)patterns); + + /// + /// Gets files from the specified directory using globbing patterns. + /// + /// The directory to search. + /// The globbing pattern(s) to use. + /// Files that match the globbing pattern(s). + public static IEnumerable GetFiles(DirectoryInfo directory, IEnumerable patterns) + { + // Initially based on code from Reliak.FileSystemGlobbingExtensions (https://github.com/reliak/Reliak.FileSystemGlobbingExtensions) + + var matcher = new Matcher(StringComparison.OrdinalIgnoreCase); + + // Expand braces + var expandedPatterns = patterns + .SelectMany(ExpandBraces) + .Select(f => f.Replace("\\{", "{").Replace("\\}", "}")); // Unescape braces + + // Add the patterns, any that start with ! are exclusions + foreach (var expandedPattern in expandedPatterns) + { + var isExclude = expandedPattern[0] == '!'; + var finalPattern = isExclude ? expandedPattern.Substring(1) : expandedPattern; + finalPattern = finalPattern + .Replace("\\!", "!") // Unescape negation + .Replace("\\", "/"); // Normalize slashes + + // Add exclude or include pattern to matcher + if (isExclude) + { + matcher.AddExclude(finalPattern); + } + else + { + matcher.AddInclude(finalPattern); + } + } + + DirectoryInfoBase directoryInfo = new DirectoryInfoWrapper(directory); + var result = matcher.Execute(directoryInfo); + return result.Files.Select(match => directoryInfo.GetFile(match.Path)).Select(fb => new FileInfo(fb.FullName)); + } + + /// Expands all brace ranges in a pattern, returning a sequence containing every possible combination. + /// The pattern to expand. + /// The expanded globbing patterns. + public static IEnumerable ExpandBraces(string pattern) + { + // Initially based on code from Minimatch (https://github.com/SLaks/Minimatch/blob/master/Minimatch/Minimatcher.cs) + // Brace expansion: + // a{b,c}d -> abd acd + // a{b,}c -> abc ac + // a{0..3}d -> a0d a1d a2d a3d + // a{b,c{d,e}f}g -> abg acdfg acefg + // a{b,c}d{e,f}g -> abdeg acdeg abdeg abdfg + // + // Invalid sets are not expanded. + // a{2..}b -> a{2..}b + // a{b}c -> a{b}c + + if (!HasBraces.IsMatch(pattern)) + { + // shortcut. no need to expand. + return new[] { pattern }; + } + + var escaping = false; + int i; + + // examples and comments refer to this crazy pattern: + // a{b,c{d,e},{f,g}h}x{y,z} + // expected: + // abxy + // abxz + // acdxy + // acdxz + // acexy + // acexz + // afhxy + // afhxz + // aghxy + // aghxz + + // everything before the first \{ is just a prefix. + // So, we pluck that off, and work with the rest, + // and then prepend it to everything we find. + if (pattern[0] != '{') + { + string prefix = null; + for (i = 0; i < pattern.Length; i++) + { + var c = pattern[i]; + if (c == '\\') + { + escaping = !escaping; + } + else if (c == '{' && !escaping) + { + prefix = pattern.Substring(0, i); + break; + } + else + { + escaping = false; + } + } + + // actually no sets, all { were escaped. + if (prefix == null) + { + // no sets + return new[] { pattern }; + } + + return ExpandBraces(pattern.Substring(i)).Select(t => + { + var neg = string.Empty; + + // Check for negated subpattern + if (t.Length > 0 && t[0] == '!') + { + if (prefix[0] != '!') + { + // Only add a new negation if there isn't already one + neg = "!"; + } + t = t.Substring(1); + } + + // Remove duplicated path separators (can happen when there's an empty expansion like "baz/{foo,}/bar") + if (t.Length > 0 && t[0] == '/' && prefix[prefix.Length - 1] == '/') + { + t = t.Substring(1); + } + + return neg + prefix + t; + }); + } + + // now we have something like: + // {b,c{d,e},{f,g}h}x{y,z} + // walk through the set, expanding each part, until + // the set ends. then, we'll expand the suffix. + // If the set only has a single member, then'll put the {} back + + // first, handle numeric sets, since they're easier + var numset = NumericSet.Match(pattern); + if (numset.Success) + { + // console.error("numset", numset[1], numset[2]) + var suf = ExpandBraces(pattern.Substring(numset.Length)).ToList(); + int start = int.Parse(numset.Groups[1].Value), + end = int.Parse(numset.Groups[2].Value), + inc = start > end ? -1 : 1; + var retVal = new List(); + for (var w = start; w != (end + inc); w += inc) + { + // append all the suffixes + retVal.AddRange(suf.Select(t => w + t)); + } + return retVal; + } + + // ok, walk through the set + // We hope, somewhat optimistically, that there + // will be a } at the end. + // If the closing brace isn't found, then the pattern is + // interpreted as braceExpand("\\" + pattern) so that + // the leading \{ will be interpreted literally. + var depth = 1; + var set = new List(); + var member = string.Empty; + escaping = false; + + for (i = 1 /* skip the \{ */; i < pattern.Length && depth > 0; i++) + { + var c = pattern[i]; + + if (escaping) + { + escaping = false; + member += "\\" + c; + } + else + { + switch (c) + { + case '\\': + escaping = true; + continue; + + case '{': + depth++; + member += "{"; + continue; + + case '}': + depth--; + + // if this closes the actual set, then we're done + if (depth == 0) + { + set.Add(member); + member = string.Empty; + + // pluck off the close-brace + break; + } + else + { + member += c; + continue; + } + + case ',': + if (depth == 1) + { + set.Add(member); + member = string.Empty; + } + else + { + member += c; + } + continue; + + default: + member += c; + continue; + } // switch + } // else + } // for + + // now we've either finished the set, and the suffix is + // pattern.substr(i), or we have *not* closed the set, + // and need to escape the leading brace + if (depth != 0) + { + // didn't close pattern + return ExpandBraces("\\" + pattern); + } + + // ["b", "c{d,e}","{f,g}h"] -> ["b", "cd", "ce", "fh", "gh"] + var addBraces = set.Count == 1; + + set = set.SelectMany(ExpandBraces).ToList(); + + if (addBraces) + { + set = set.Select(s => "{" + s + "}").ToList(); + } + + // now attach the suffixes. + // x{y,z} -> ["xy", "xz"] + // console.error("set", set) + // console.error("suffix", pattern.substr(i)) + return ExpandBraces(pattern.Substring(i)).SelectMany(suf => + { + var negated = false; + if (suf.Length > 0 && suf[0] == '!') + { + negated = true; + suf = suf.Substring(1); + } + return set.Select(s => + { + var neg = string.Empty; + if (negated && (s.Length == 0 || s[0] != '!')) + { + // Only add a new negation if there isn't already one + neg = "!"; + } + return neg + s + suf; + }); + }); + } + } +} diff --git a/src/SignClient/Program.cs b/src/SignClient/Program.cs index d5659be9..72fa3b54 100644 --- a/src/SignClient/Program.cs +++ b/src/SignClient/Program.cs @@ -14,19 +14,21 @@ public static int Main(string[] args) cfg.Description = "Signs a file or set of files"; cfg.HelpOption("-? | -h | --help"); var configFile = cfg.Option("-c | --config", "Path to config json file", CommandOptionType.SingleValue); - var inputFile = cfg.Option("-i | --input", "Path to config input file", CommandOptionType.SingleValue); - var outputFile = cfg.Option("-o | --output", "Path to output file. May be same as input to overwrite", CommandOptionType.SingleValue); + var inputFile = cfg.Option("-i | --input", "Path to input file", CommandOptionType.SingleValue); + var baseDirectory = cfg.Option("-b | --baseDirectory", "Base directory for files to override the working directory", CommandOptionType.SingleValue); + var outputFile = cfg.Option("-o | --output", "Path to output. May be same as input to overwrite", CommandOptionType.SingleValue); var fileList = cfg.Option("-f | --filelist", "Full path to file containing paths of files to sign within an archive", CommandOptionType.SingleValue); var secret = cfg.Option("-s | --secret", "Client Secret", CommandOptionType.SingleValue); var user = cfg.Option("-r | --user", "Username", CommandOptionType.SingleValue); var name = cfg.Option("-n | --name", "Name of project for tracking", CommandOptionType.SingleValue); var description = cfg.Option("-d | --description", "Description", CommandOptionType.SingleValue); var descUrl = cfg.Option("-u | --descriptionUrl", "Description Url", CommandOptionType.SingleValue); + var maxConcurrency = cfg.Option("-m | --maxConcurrency", "Maximum concurrency (default is 4)", CommandOptionType.SingleValue); cfg.OnExecute(() => { var sign = new SignCommand(application); - return sign.SignAsync(configFile, inputFile, outputFile, fileList, secret, user, name, description, descUrl); + return sign.Sign(configFile, inputFile, baseDirectory, outputFile, fileList, secret, user, name, description, descUrl, maxConcurrency); }); }); diff --git a/src/SignClient/SignCommand.cs b/src/SignClient/SignCommand.cs index 1afe9e89..98b40b39 100644 --- a/src/SignClient/SignCommand.cs +++ b/src/SignClient/SignCommand.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Net; using System.Net.Http; using System.Security.Authentication; @@ -10,6 +11,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Identity.Client; using Refit; +using Wyam.Core.IO.Globbing; namespace SignClient { @@ -29,17 +31,19 @@ public SignCommand(CommandLineApplication signCommandLineApplication) this.signCommandLineApplication = signCommandLineApplication; } - public async Task SignAsync + public int Sign ( CommandOption configFile, CommandOption inputFile, + CommandOption baseDirectory, CommandOption outputFile, CommandOption fileList, CommandOption clientSecret, CommandOption username, CommandOption name, CommandOption description, - CommandOption descriptionUrl + CommandOption descriptionUrl, + CommandOption maxConcurrency ) { try @@ -63,12 +67,50 @@ CommandOption descriptionUrl return EXIT_CODES.INVALID_OPTIONS; } - if (!outputFile.HasValue()) + if(!maxConcurrency.HasValue()) { - // use input as the output value - outputFile.Values.Add(inputFile.Value()); + maxConcurrency.Values.Add("4"); // default to 4 } + if(baseDirectory.HasValue()) + { + // Make sure this is rooted + if(!Path.IsPathRooted(baseDirectory.Value())) + { + signCommandLineApplication.Error.WriteLine("--directory parameter must be rooted if specified"); + return EXIT_CODES.INVALID_OPTIONS; + } + } + + if(!baseDirectory.HasValue()) + { + baseDirectory.Values.Add(Environment.CurrentDirectory); + } + + List inputFiles; + // If we're going to glob, we can't be fully rooted currently (fix me later) + + if(inputFile.Value().Contains('*')) + { + if(Path.IsPathRooted(inputFile.Value())) + { + signCommandLineApplication.Error.WriteLine("--input parameter cannot be rooted when using a glob. Use a path relative to the working directory"); + return EXIT_CODES.INVALID_OPTIONS; + } + + inputFiles = Globber.GetFiles(new DirectoryInfo(baseDirectory.Value()), inputFile.Value()) + .ToList(); + } + else + { + inputFiles = new List + { + new FileInfo(ExpandFilePath(inputFile.Value())) + }; + } + + + var builder = new ConfigurationBuilder() .AddJsonFile(ExpandFilePath(configFile.Value())) .AddEnvironmentVariables(); @@ -116,35 +158,78 @@ CommandOption descriptionUrl var client = RestService.For(configuration["SignClient:Service:Url"], settings); client.Client.Timeout = Timeout.InfiniteTimeSpan; // TODO: Make configurable on command line - // Prepare input/output file - var input = new FileInfo(ExpandFilePath(inputFile.Value())); - var output = new FileInfo(ExpandFilePath(outputFile.Value())); - Directory.CreateDirectory(output.DirectoryName); + // var max concurrency + if(!int.TryParse(maxConcurrency.Value(), out var maxC) || maxC < 1) + { + signCommandLineApplication.Error.WriteLine("--maxConcurrency parameter is not valid"); + return EXIT_CODES.INVALID_OPTIONS; + } - // Do action + Parallel.ForEach(inputFiles,new ParallelOptions { MaxDegreeOfParallelism = maxC } , input => + { + FileInfo output; - HttpResponseMessage response; + // Special case if there's only one input file and the output has a value, treat it as a file + if(inputFiles.Count == 1 && outputFile.HasValue()) + { + output = new FileInfo(ExpandFilePath(outputFile.Value())); + } + else + { + // if the output is speciied, treat it as a directory, if not, overwrite the current file + if(!outputFile.HasValue()) + { + output = new FileInfo(input.FullName); + } + else + { + var relative = Path.GetRelativePath(baseDirectory.Value(), input.FullName); - response = await client.SignFile(input, - fileList.HasValue() ? new FileInfo(ExpandFilePath(fileList.Value())) : null, - HashMode.Sha256, - name.Value(), - description.Value(), - descriptionUrl.Value()); + var basePath = Path.IsPathRooted(outputFile.Value()) ? + outputFile.Value() : + $"{baseDirectory.Value()}{Path.DirectorySeparatorChar}{outputFile.Value()}"; - // Check response + var fullOutput = Path.Combine(basePath, relative); - if (!response.IsSuccessStatusCode) - { - Console.Error.WriteLine($"Server returned non Ok response: {(int)response.StatusCode} {response.ReasonPhrase}"); - return -1; - } + output = new FileInfo(fullOutput); + } + } + + // Ensure the output directory exists + Directory.CreateDirectory(output.DirectoryName); + + // Do action + + HttpResponseMessage response; + + signCommandLineApplication.Out.WriteLine($"Submitting '{input.FullName}' for signing."); - var str = await response.Content.ReadAsStreamAsync(); + response = client.SignFile(input, + fileList.HasValue() ? new FileInfo(ExpandFilePath(fileList.Value())) : null, + HashMode.Sha256, + name.Value(), + description.Value(), + descriptionUrl.Value()).Result; - // If we're replacing the file, make sure to the existing one first - using var fs = new FileStream(output.FullName, FileMode.Create); - await str.CopyToAsync(fs).ConfigureAwait(false); + // Check response + + if (!response.IsSuccessStatusCode) + { + signCommandLineApplication.Error.WriteLine($"Error signing '{input.FullName}'"); + signCommandLineApplication.Error.WriteLine($"Server returned non Ok response: {(int)response.StatusCode} {response.ReasonPhrase}"); + response.EnsureSuccessStatusCode(); // force the throw to break out of the loop + } + + var str = response.Content.ReadAsStreamAsync().Result; + + // If we're replacing the file, make sure to the existing one first + using var fs = new FileStream(output.FullName, FileMode.Create); + str.CopyTo(fs); + + signCommandLineApplication.Out.WriteLine($"Successfully signed '{input.FullName}'"); + }); + + } catch (AuthenticationException e) { @@ -158,15 +243,17 @@ CommandOption descriptionUrl } return EXIT_CODES.SUCCESS; - } - static string ExpandFilePath(string file) - { - if (!Path.IsPathRooted(file)) + string ExpandFilePath(string file) { - return $"{Environment.CurrentDirectory}{Path.DirectorySeparatorChar}{file}"; + if (!Path.IsPathRooted(file)) + { + return $"{baseDirectory.Value()}{Path.DirectorySeparatorChar}{file}"; + } + return file; } - return file; } + + } } diff --git a/src/SignClient/appsettings.json b/src/SignClient/appsettings.json index 933c6eff..c2f175b1 100644 --- a/src/SignClient/appsettings.json +++ b/src/SignClient/appsettings.json @@ -7,10 +7,10 @@ "TenantId": "71048637-3782-41a3-b6b2-6f4ac8a25ae0" }, "Service": { - //"Url": "https://oren-signservice-dev.azurewebsites.net/", + "Url": "https://oren-signservice-dev.azurewebsites.net/", //"Url": "https://novotny-sign-service.azurewebsites.net/", //"Url": "https://codesign.novotny.org/", - "Url": "https://localhost:44351/", + //"Url": "https://localhost:44351/", //"ResourceId": "https://SignService/0263d4ba-331b-46d1-85e1-bee9898a65a6" // PROD "ResourceId": "https://SignService/5b3c8c6e-f5c3-41ec-8a5f-6b414847a688" }