From 1c8b5db14e491134e0c66da05102a253b73d2b9b Mon Sep 17 00:00:00 2001 From: Vyacheslav Rusakov Date: Thu, 28 Mar 2024 17:20:12 +0700 Subject: [PATCH] add printStats option to print all executed python commands; add debug option (for development); add BasePythonTask.useCustomPython task option to allow custom pythonPath for commands (otherwise, pythonPath selected by checkPython task used); fix caches and stats under configuration cache; all tasks become abstract (would affect all custom tasks) --- CHANGELOG.md | 15 +- .../plugin/python/PythonExtension.groovy | 17 ++ .../gradle/plugin/python/PythonPlugin.groovy | 125 ++++++++----- .../python/cmd/docker/ContainerManager.groovy | 7 + .../python/cmd/docker/DockerFactory.groovy | 7 +- .../plugin/python/cmd/env/Environment.groovy | 24 +++ .../python/cmd/env/GradleEnvironment.groovy | 147 +++++++++++---- .../python/cmd/exec/PythonBinary.groovy | 79 ++++---- .../plugin/python/service/EnvService.groovy | 126 ++++++++++--- .../python/service/stat/PythonStat.groovy | 46 +++++ .../python/service/stat/StatsPrinter.groovy | 83 +++++++++ .../service/value/CacheValueSource.groovy | 29 +++ .../service/value/StatsValueSource.groovy | 28 +++ .../plugin/python/task/BasePythonTask.groovy | 39 +++- .../plugin/python/task/CheckPythonTask.groovy | 30 ++-- .../plugin/python/task/PythonTask.groovy | 4 +- .../plugin/python/task/pip/BasePipTask.groovy | 4 +- .../python/task/pip/PipInstallTask.groovy | 5 +- .../plugin/python/task/pip/PipListTask.groovy | 2 +- .../python/task/pip/PipUpdatesTask.groovy | 2 +- .../plugin/python/AbstractKitTest.groovy | 16 +- .../gradle/plugin/python/AbstractTest.groovy | 12 +- .../ConfigurationCacheSupportKitTest.groovy | 4 +- .../gradle/plugin/python/StatsKitTest.groovy | 168 ++++++++++++++++++ .../UseCustomPythonForTaskKitTest.groovy | 42 +++++ .../python/cmd/AbstractCliMockSupport.groovy | 39 +++- 26 files changed, 908 insertions(+), 192 deletions(-) create mode 100644 src/main/groovy/ru/vyarus/gradle/plugin/python/service/stat/PythonStat.groovy create mode 100644 src/main/groovy/ru/vyarus/gradle/plugin/python/service/stat/StatsPrinter.groovy create mode 100644 src/main/groovy/ru/vyarus/gradle/plugin/python/service/value/CacheValueSource.groovy create mode 100644 src/main/groovy/ru/vyarus/gradle/plugin/python/service/value/StatsValueSource.groovy create mode 100644 src/test/groovy/ru/vyarus/gradle/plugin/python/StatsKitTest.groovy create mode 100644 src/test/groovy/ru/vyarus/gradle/plugin/python/UseCustomPythonForTaskKitTest.groovy diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d97def..b296a28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ * (BREAKING) Drop gradle 5 and 6 support +* (BREAKING) All plugin tasks become abstract, so any custom task types should be abstract too * Fix `alwaysInstallModules = true` not triggers pipInstall for non-strict requirements file (#94) (required for case when requirements file links other files, which changes are not tracked) * Add requirements file references (-r file) support under strict mode (#94) @@ -6,11 +7,15 @@ * Changed virtualenv version installed by default (python.virtualenvVersion) from 20.4.2 to 20.25.1 (supports python 3.7 - 3.12) * Changed default docker image (python.docker.image) from python:3.10.8-alpine3.15 to python:3.11.8-alpine3.19 -* Add python.breakSystemPackages option: adds --break-system-packages for pip install - May be required on linux to install virtualenv on apt-managed python (e.g. python3.12) -* Add host network option for docker: python.docker.useHostNetwork. - Works only on linux. When enabled, all container ports automatically exposed on host - and configured port mappings ignored +* New options (python.): + - breakSystemPackages: adds --break-system-packages for pip install + May be required on linux to install virtualenv on apt-managed python (e.g. python3.12) + - docker.useHostNetwork: use host network for docker + Works only on linux. When enabled, all container ports automatically exposed on host + and configured port mappings ignored + - printStats: show all executed python commands (including plugin internal) at the end of the build (with timings) +* To use different python for PythonTask useCustomPython = true must be declared now + (otherwise, pythonPath select by checkPython task would be used (and task's pythonPath ignored)) ### 3.0.0 (2022-10-22) * (breaking) Drop gradle 5.0-5.2 support (minimum required gradle is 5.3) diff --git a/src/main/groovy/ru/vyarus/gradle/plugin/python/PythonExtension.groovy b/src/main/groovy/ru/vyarus/gradle/plugin/python/PythonExtension.groovy index 292c1b7..aa7736a 100644 --- a/src/main/groovy/ru/vyarus/gradle/plugin/python/PythonExtension.groovy +++ b/src/main/groovy/ru/vyarus/gradle/plugin/python/PythonExtension.groovy @@ -176,6 +176,23 @@ class PythonExtension { * By default use default virtualenv behaviour: symlink environment. */ boolean envCopy + + /** + * Print all executed python commands at the end of the build (including "invisible" internal commands, used by + * plugin for configuration). + *

+ * IMPORTANT: in multi-module project only root project setting takes effect + */ + boolean printStats = false + + /** + * Enable debug messages (for cache and stats debugging). See {@link #printStats} for printing actually called + * commands (including invisible). Option for development purposes (especially with configuration cache). + *

+ * IMPORTANT: in multi-module project only root project setting takes effect + */ + boolean debug = false + /** * Requirements file support. By default, plugin will search for requirements.txt file and would read all * declarations from there and use for manual installation (together with `python.modules` property). File could diff --git a/src/main/groovy/ru/vyarus/gradle/plugin/python/PythonPlugin.groovy b/src/main/groovy/ru/vyarus/gradle/plugin/python/PythonPlugin.groovy index 76f529c..07ba0f5 100644 --- a/src/main/groovy/ru/vyarus/gradle/plugin/python/PythonPlugin.groovy +++ b/src/main/groovy/ru/vyarus/gradle/plugin/python/PythonPlugin.groovy @@ -7,6 +7,9 @@ import org.gradle.api.Project import org.gradle.api.provider.Provider import org.gradle.api.tasks.Delete import org.gradle.api.tasks.TaskProvider +import org.gradle.build.event.BuildEventsListenerRegistry +import ru.vyarus.gradle.plugin.python.cmd.env.Environment +import ru.vyarus.gradle.plugin.python.cmd.env.GradleEnvironment import ru.vyarus.gradle.plugin.python.service.EnvService import ru.vyarus.gradle.plugin.python.task.BasePythonTask import ru.vyarus.gradle.plugin.python.task.CheckPythonTask @@ -17,6 +20,8 @@ import ru.vyarus.gradle.plugin.python.task.pip.PipListTask import ru.vyarus.gradle.plugin.python.task.pip.PipUpdatesTask import ru.vyarus.gradle.plugin.python.util.RequirementsReader +import javax.inject.Inject + /** * Use-python plugin. Plugin requires python installed globally or configured path to python binary. * Alternatively, docker might be used. @@ -32,15 +37,39 @@ import ru.vyarus.gradle.plugin.python.util.RequirementsReader */ @CompileStatic @SuppressWarnings('DuplicateStringLiteral') -class PythonPlugin implements Plugin { +abstract class PythonPlugin implements Plugin { + + // need to be static and public for configuration cache support + static void configureDockerInTask(Project project, PythonExtension.Docker docker, BasePythonTask task) { + task.docker.use.convention(project.provider { docker.use }) + task.docker.image.convention(project.provider { docker.image }) + task.docker.windows.convention(project.provider { docker.windows }) + task.docker.useHostNetwork.convention(project.provider { docker.useHostNetwork }) + task.docker.ports.convention(project.provider { docker.ports }) + task.docker.exclusive.convention(false) + } + + static PythonExtension findRootExtension(Project project) { + PythonExtension rootExt = null + Project prj = project + while (prj != null) { + PythonExtension cand = prj.extensions.findByType(PythonExtension) + if (cand != null) { + rootExt = cand + } + prj = prj.parent + } + return rootExt + } - private Provider envService + @Inject + abstract BuildEventsListenerRegistry getEventsListenerRegistry() @Override void apply(Project project) { PythonExtension extension = project.extensions.create('python', PythonExtension, project) - initService(project, extension) + Provider envService = initService(project) // simplify direct tasks usage project.extensions.extraProperties.set(PipInstallTask.simpleName, PipInstallTask) @@ -48,20 +77,33 @@ class PythonPlugin implements Plugin { // configuration shortcut PythonExtension.Scope.values().each { project.extensions.extraProperties.set(it.name(), it) } - createTasks(project, extension) + createTasks(project, extension, envService) } - private void initService(Project project, PythonExtension extension) { + private Provider initService(Project project) { // service used to shutdown docker properly and hold actual python path link - envService = project.gradle.sharedServices.registerIfAbsent( - 'pythonEnvironmentService', EnvService, spec -> { }) + Provider envService = project.gradle.sharedServices.registerIfAbsent( + 'pythonEnvironmentService', EnvService, spec -> { + // root extension used + PythonExtension rootExt = findRootExtension(project) + + EnvService.Params params = spec.parameters as EnvService.Params + // only root project value counted for print stats activation + params.printStats.set(project.provider { rootExt.printStats }) + params.debug.set(project.provider { rootExt.debug }) + }) + + // it is not required, but used to prevent KILLING service too early under configuration cache + eventsListenerRegistry.onTaskCompletion(envService) + + // IMPORTANT: do not try to obtain service here (e.g. to init it) because it would cause eager parameters + // resolution!!! - // can't use service properties because each project in multi-module project must have unique path - envService.get().defaultProvider(project.path, { extension.pythonPath } as Provider) + return envService } @SuppressWarnings('BuilderMethodWithSideEffects') - private void createTasks(Project project, PythonExtension extension) { + private void createTasks(Project project, PythonExtension extension, Provider envService) { // validate installed python TaskProvider checkTask = project.tasks.register('checkPython', CheckPythonTask) { it.with { @@ -97,7 +139,7 @@ class PythonPlugin implements Plugin { } } - configureDefaults(project, extension, checkTask, installTask) + configureDefaults(project, extension, checkTask, installTask, envService) } @SuppressWarnings(['MethodSize', 'AbcMetric']) @@ -105,27 +147,36 @@ class PythonPlugin implements Plugin { private void configureDefaults(Project project, PythonExtension extension, TaskProvider checkTask, - TaskProvider installTask) { - + TaskProvider installTask, + Provider envService) { project.tasks.withType(BasePythonTask).configureEach { task -> - task.with { - // apply default path for all python tasks - task.conventionMapping.with { - // IMPORTANT: pythonPath might change after checkPythonTask (switched to environment) - pythonPath = { envService.get().getPythonPath(project.path) } - pythonBinary = { extension.pythonBinary } - validateSystemBinary = { extension.validateSystemBinary } - // important to copy map because each task must have independent instance - environment = { extension.environment ? new HashMap<>(extension.environment) : null } - } - - // can't be implemented with convention mapping, only with properties - configureDockerInTask(project, extension.docker, task) - - // all python tasks must be executed after check task to use correct environment (switch to virtualenv) - if (task.taskIdentity.type != CheckPythonTask) { - dependsOn checkTask - } + task.envService.set(envService) + task.usesService(envService) + + Environment gradleEnv = GradleEnvironment.create(project, task.name, envService, + project.provider { findRootExtension(project).debug }) + task.gradleEnv.set(gradleEnv) + doLast { + gradleEnv.printCacheState() + } + + task.conventionMapping.with { + // setting default value from extension to all tasks, but tasks would actually check + // service for actual pythonPath before python instance creation + // Only checkPython task will always use this default to initialize service value + pythonPath = { extension.pythonPath } + pythonBinary = { extension.pythonBinary } + validateSystemBinary = { extension.validateSystemBinary } + // important to copy map because each task must have independent instance + environment = { extension.environment ? new HashMap<>(extension.environment) : null } + } + + // can't be implemented with convention mapping, only with properties + configureDockerInTask(project, extension.docker, task) + + // all python tasks must be executed after check task to use correct environment (switch to virtualenv) + if (task.taskIdentity.type != CheckPythonTask) { + task.dependsOn checkTask } } @@ -144,7 +195,7 @@ class PythonPlugin implements Plugin { breakSystemPackages = { extension.breakSystemPackages } trustedHosts = { extension.trustedHosts } extraIndexUrls = { extension.extraIndexUrls } - requirements = { RequirementsReader.find(task.gradleEnv, extension.requirements) } + requirements = { RequirementsReader.find(task.gradleEnv.get(), extension.requirements) } strictRequirements = { extension.requirements.strict } } } @@ -159,7 +210,6 @@ class PythonPlugin implements Plugin { } project.tasks.withType(CheckPythonTask).configureEach { task -> - task.envService = this.envService task.conventionMapping.with { scope = { extension.scope } envPath = { extension.envPath } @@ -172,13 +222,4 @@ class PythonPlugin implements Plugin { } } } - - private void configureDockerInTask(Project project, PythonExtension.Docker docker, BasePythonTask task) { - task.docker.use.convention(project.provider { docker.use }) - task.docker.image.convention(project.provider { docker.image }) - task.docker.windows.convention(project.provider { docker.windows }) - task.docker.useHostNetwork.convention(project.provider { docker.useHostNetwork }) - task.docker.ports.convention(project.provider { docker.ports }) - task.docker.exclusive.convention(false) - } } diff --git a/src/main/groovy/ru/vyarus/gradle/plugin/python/cmd/docker/ContainerManager.groovy b/src/main/groovy/ru/vyarus/gradle/plugin/python/cmd/docker/ContainerManager.groovy index b017e02..34d1d4d 100644 --- a/src/main/groovy/ru/vyarus/gradle/plugin/python/cmd/docker/ContainerManager.groovy +++ b/src/main/groovy/ru/vyarus/gradle/plugin/python/cmd/docker/ContainerManager.groovy @@ -175,6 +175,13 @@ class ContainerManager { return container != null } + /** + * @return container name or null if container not started yet + */ + String getContainerName() { + return container?.running ? container.containerName : null + } + /** * Stops started container or do nothing if not started. */ diff --git a/src/main/groovy/ru/vyarus/gradle/plugin/python/cmd/docker/DockerFactory.groovy b/src/main/groovy/ru/vyarus/gradle/plugin/python/cmd/docker/DockerFactory.groovy index 4d4f9ce..662afca 100644 --- a/src/main/groovy/ru/vyarus/gradle/plugin/python/cmd/docker/DockerFactory.groovy +++ b/src/main/groovy/ru/vyarus/gradle/plugin/python/cmd/docker/DockerFactory.groovy @@ -51,9 +51,12 @@ class DockerFactory { /** * Shuts down started containers. Called at the end of the build. */ + @SuppressWarnings('UnnecessaryGetter') static synchronized void shutdownAll() { - CONTAINERS.values().each { it.stop() } - CONTAINERS.clear() + if (!CONTAINERS.isEmpty()) { + CONTAINERS.values().each { it.stop() } + CONTAINERS.clear() + } } /** diff --git a/src/main/groovy/ru/vyarus/gradle/plugin/python/cmd/env/Environment.groovy b/src/main/groovy/ru/vyarus/gradle/plugin/python/cmd/env/Environment.groovy index 32016bb..0b52a63 100644 --- a/src/main/groovy/ru/vyarus/gradle/plugin/python/cmd/env/Environment.groovy +++ b/src/main/groovy/ru/vyarus/gradle/plugin/python/cmd/env/Environment.groovy @@ -7,6 +7,8 @@ import java.util.function.Supplier /** * Environment-specific apis provider. Object used as lightweight alternative to gradle {@link org.gradle.api.Project} * (which was used before), because project is not compatible with configuration cache. + *

+ * NOTE: configuration cache stores entire objects (so they are created only when configuration cache is not enabled). * * @author Vyacheslav Rusakov * @since 15.03.2024 @@ -102,4 +104,26 @@ interface Environment { * @param value value */ void updateGlobalCache(String key, Object value) + + /** + * Print debug message if debug enabled. + * + * @param msg message + */ + void debug(String msg) + + /** + * Save command execution stat. Counts only python execution (possible direct docker commands ignored). + * + * @param containerName docker container name + * @param cmd executed command (cleared!) + * @param start start time + * @param success execution success + */ + void stat(String containerName, String cmd, long start, boolean success) + + /** + * Prints cache state (for debug), but only if debug enabled in the root project. + */ + void printCacheState() } diff --git a/src/main/groovy/ru/vyarus/gradle/plugin/python/cmd/env/GradleEnvironment.groovy b/src/main/groovy/ru/vyarus/gradle/plugin/python/cmd/env/GradleEnvironment.groovy index 7c236ca..d24e19b 100644 --- a/src/main/groovy/ru/vyarus/gradle/plugin/python/cmd/env/GradleEnvironment.groovy +++ b/src/main/groovy/ru/vyarus/gradle/plugin/python/cmd/env/GradleEnvironment.groovy @@ -5,61 +5,93 @@ import org.gradle.api.Action import org.gradle.api.Project import org.gradle.api.internal.file.FileOperations import org.gradle.api.logging.Logger +import org.gradle.api.provider.Provider import org.gradle.process.ExecOperations import org.gradle.process.ExecResult import org.gradle.process.ExecSpec +import ru.vyarus.gradle.plugin.python.service.EnvService +import ru.vyarus.gradle.plugin.python.service.stat.PythonStat +import ru.vyarus.gradle.plugin.python.service.value.CacheValueSource +import ru.vyarus.gradle.plugin.python.service.value.StatsValueSource import javax.inject.Inject -import java.util.concurrent.ConcurrentHashMap import java.util.function.Supplier /** * Configuration cache compatible implementation (substitutes gradle {@link org.gradle.api.Project} object usage). *

- * Caches stored in gradle extended properties so all environment entities could share the same cache maps. - * For root project global and project cache is the same map. + * Environment instance is created per python task, but all instances share same caches and stats (even under + * configuration cache). + *

+ * Configuration cache caches entire object (so it is not created when configuration cache is enabled), but + * caches and stats are always passed resolves from (singleton) service. * * @author Vyacheslav Rusakov * @since 15.03.2024 */ @CompileStatic +@SuppressWarnings(['Println', 'DuplicateStringLiteral']) abstract class GradleEnvironment implements Environment { - private static final String CACHE_KEY = 'plugin.python.project.cache' - private final Logger logger private final File projectDir private final File rootDir private final String rootName private final String projectPath - private final Map cacheGlobal - private final Map cacheProject + private final String taskName + + // same cache for all projects + private final Provider> cacheGlobal + // same cache for all instances within one project + private final Provider> cacheProject + // same list for all projects + private final Provider> stats + private final Provider debug @SuppressWarnings('SynchronizedMethod') - static synchronized Environment create(Project project) { - // NOTE: different instance created for each task, but cache instance would be the same! + static synchronized Environment create(Project project, String taskName, + Provider service, Provider debug) { + // NOTE: different instance created for each task, but cache and stat instances would be the same! project.objects.newInstance(GradleEnvironment, - project.logger, project.projectDir, project.rootDir, project.rootProject.name, project.path, - lookupCache(project.rootProject), lookupCache(project)) + project.logger, project.projectDir, project.rootDir, project.rootProject.name, project.path, taskName, + // gradle can't cache it - always the same instance! + project.providers.of(CacheValueSource) { + it.parameters.service.set(service) + it.parameters.project.set(project.rootProject.path) + }, + project.providers.of(CacheValueSource) { + it.parameters.service.set(service) + it.parameters.project.set(project.path) + }, + project.providers.of(StatsValueSource) { + it.parameters.service.set(service) + }, + debug) } @Inject @SuppressWarnings(['ParameterCount', 'AbstractClassWithPublicConstructor']) GradleEnvironment(Logger logger, - File projectDir, - File rootDir, - String rootName, - String projectPath, - Map globalCache, - Map projectCache) { + File projectDir, + File rootDir, + String rootName, + String projectPath, + String taskName, + Provider> globalCache, + Provider> projectCache, + Provider> stats, + Provider debug) { this.logger = logger this.projectDir = projectDir this.rootDir = rootDir this.rootName = rootName this.projectPath = projectPath + this.taskName = taskName // note for root project it would be the same maps this.cacheGlobal = globalCache this.cacheProject = projectCache + this.stats = stats + this.debug = debug } Logger getLogger() { @@ -123,24 +155,84 @@ abstract class GradleEnvironment implements Environment { @Override public T projectCache(String key, Supplier value) { - return getOrCompute(false, cacheProject, key, value) + return getOrCompute(false, cacheProject.get(), key, value) } @Override public T globalCache(String key, Supplier value) { - return getOrCompute(true, cacheGlobal, key, value) + return getOrCompute(true, cacheGlobal.get(), key, value) } @Override void updateProjectCache(String key, Object value) { - logger.info("[$projectPath] updated cache value: $key = $value") - cacheProject.put(key, value) + Map cache = cacheProject.get() + if (debug.get()) { + // instance important for configuration cache mode where different objects could appear (but shouldn't + // because ValueSource objects used) + println "[CACHE$projectPath$taskName] Project cache update $key=$value (instance: " + + "${System.identityHashCode(cache)})" + } + cache.put(key, value) } @Override void updateGlobalCache(String key, Object value) { - logger.info("[$projectPath] updated global cache value: $key = $value") - cacheGlobal.put(key, value) + Map cache = cacheGlobal.get() + if (debug.get()) { + println "[CACHE$projectPath$taskName] Global cache update $key=$value (instance: " + + "${System.identityHashCode(cache)})" + } + cache.put(key, value) + } + + @Override + @SuppressWarnings('ConfusingMethodName') + void debug(String msg) { + if (debug.get()) { + println "[DEBUG$projectPath:$taskName] ".replaceAll('::', ':') + msg + } + } + + @Override + void stat(String containerName, String cmd, long start, boolean success) { + List statList = stats.get() + synchronized (statList) { + statList.add(new PythonStat( + containerName: containerName, + projectPath: projectPath, + taskName: taskName, + cmd: cmd, + start: start, + success: success, + duration: System.currentTimeMillis() - start + )) + if (debug.get()) { + println "[STATS$projectPath$taskName] Stat registered: stats instance " + + "${System.identityHashCode(statList)}, count ${statList.size()}\n\tfor: $cmd" + } + } + } + + @Override + void printCacheState() { + if (debug.get()) { + StringBuilder res = new StringBuilder( + "\n--------------------------------------------------- state after $projectPath$taskName \n") + if (projectPath == ':') { + res.append('\n\tGLOBAL CACHE is the same as project cache for root project\n') + } else { + Map cache = cacheGlobal.get() + res.append("\tGLOBAL CACHE (instance ${System.identityHashCode(cache)}) [${cache.size()}]\n") + cache.each { res.append("\t\t$it.key = $it.value\n") } + } + + Map cache = cacheProject.get() + res.append("\n\tPROJECT CACHE (instance ${System.identityHashCode(cache)}) [${cache.size()}]\n") + cache.each { res.append("\t\t$it.key = $it.value\n") } + + res.append('-------------------------------------------------------------------------\n') + println res.toString() + } } @Inject @@ -149,15 +241,6 @@ abstract class GradleEnvironment implements Environment { @Inject protected abstract FileOperations getFs() - private static Map lookupCache(Project project) { - Map cache = project.findProperty(CACHE_KEY) as Map - if (cache == null) { - cache = new ConcurrentHashMap<>() - project.extensions.extraProperties.set(CACHE_KEY, cache) - } - return cache - } - private T getOrCompute(boolean global, Map cache, String key, Supplier value) { synchronized (cache) { // computeIfAbsent not used here because actions MAY also call cacheable functions, which is not allowed diff --git a/src/main/groovy/ru/vyarus/gradle/plugin/python/cmd/exec/PythonBinary.groovy b/src/main/groovy/ru/vyarus/gradle/plugin/python/cmd/exec/PythonBinary.groovy index 0c719e7..97bd240 100644 --- a/src/main/groovy/ru/vyarus/gradle/plugin/python/cmd/exec/PythonBinary.groovy +++ b/src/main/groovy/ru/vyarus/gradle/plugin/python/cmd/exec/PythonBinary.groovy @@ -34,8 +34,6 @@ final class PythonBinary { private static final String NL = '\n' private static final Object SYNC = new Object() - // used to avoid duplicate global python3 binary detection on linux - private static volatile Boolean python3detected private final Environment environment @@ -45,7 +43,7 @@ final class PythonBinary { private final String sourcePythonBinary private boolean validateBinary = true - private ContainerManager docker + private ContainerManager dockerManager private DockerConfig dockerConfig // keeping actual work dir and environment here is important for docker to prevent redundant container @@ -83,7 +81,7 @@ final class PythonBinary { beforeInit('Docker config must be set before initialization') if (docker) { this.dockerConfig = docker - this.docker = DockerFactory.getContainer(docker, environment) + this.dockerManager = DockerFactory.getContainer(docker, environment) } } @@ -116,7 +114,7 @@ final class PythonBinary { * @return path with corrected separators or path as is if docker not used */ String targetOsCanonicalPath(String path) { - return docker ? docker.canonicalPath(path) : path + return docker ? dockerManager.canonicalPath(path) : path } /** @@ -144,17 +142,17 @@ final class PythonBinary { if (docker) { // in case of docker path might be either local or inside docker (depending on resolution method) // this way it would always be a docker path - res = docker.toDockerPath(res) + res = dockerManager.toDockerPath(res) } return res } boolean isWindows() { - return docker ? docker.windows : Os.isFamily(Os.FAMILY_WINDOWS) + return docker ? dockerManager.windows : Os.isFamily(Os.FAMILY_WINDOWS) } boolean isDocker() { - return docker != null + return dockerManager != null } /** @@ -174,7 +172,7 @@ final class PythonBinary { init() if (docker) { // if path leads inside project then it might be checked locally - String localPath = docker.toLocalPath(path) + String localPath = dockerManager.toLocalPath(path) if (localPath != null) { return new File(localPath).exists() } @@ -207,13 +205,12 @@ final class PythonBinary { if (docker) { // change paths (according to docker mapping) // performed here in order to see actual command in logs and error message, plus, cleanups applied here - docker.convertCommand(cmd) + dockerManager.convertCommand(cmd) // called here to show container start/stop logs before executed command prepareEnvironment() } - String commandLine = cmd.join(SPACE) - String formattedCommand = cleanLoggedCommand(commandLine.replace('\r', '').replace(NL, SPACE)) + String formattedCommand = cleanLoggedCommand(cmd) environment.logger.log(logLevel, "[python] $formattedCommand") long start = System.currentTimeMillis() @@ -321,28 +318,23 @@ final class PythonBinary { return executable } - // note: @Memoized not used because it would store link to Project object which could lead to significant - // memory leak. And that's why this check is performed outside of getPythonBinary method @CompileStatic(TypeCheckingMode.SKIP) @SuppressWarnings('AssignmentToStaticFieldFromInstanceMethod') private boolean detectPython3Binary() { - synchronized (SYNC) { - // root project property used for cache execution result in multi-module project - if (python3detected == null) { - // on windows python binary could not be named python3 - if (windows) { - python3detected = false - } else { - python3detected = rawExec([PYTHON3, '--version'] as String[]) != null - } + return environment.globalCache("python3.binary:${docker ? dockerManager.containerName : ''}", { + // on windows python binary could not be named python3 + if (windows) { + return false + } else { + return rawExec([PYTHON3, '--version'] as String[]) != null } - return python3detected - } + }) } - private String cleanLoggedCommand(String cmd) { - String res = cmd - logCleaners.each { res = it.clear(cmd) } + private String cleanLoggedCommand(String[] cmd) { + String commandLine = cmd.join(SPACE) + String res = commandLine.replace('\r', '').replace(NL, SPACE) + logCleaners.each { res = it.clear(res) } return res } @@ -355,11 +347,11 @@ final class PythonBinary { private void prepareEnvironment() { if (dockerConfig.exclusive) { environment.logger.lifecycle("[docker] executing command in exclusive container '{}' \n{}", - dockerConfig.image, docker.formatContainerInfo(dockerConfig, workDir, envVars)) + dockerConfig.image, dockerManager.formatContainerInfo(dockerConfig, workDir, envVars)) } else { // first executed command will start container and all subsequent calls would use already // started container - docker.restartIfRequired(dockerConfig, workDir, envVars) + dockerManager.restartIfRequired(dockerConfig, workDir, envVars) } } @@ -373,16 +365,27 @@ final class PythonBinary { // IMPORTANT: single point of execution for all python commands @SuppressWarnings('UnnecessarySetter') private int rawExec(String[] command, OutputStream out) { - if (docker) { - // required here for direct raw execution case (normally, start logs must appear before executed command) - if (!docker.started) { - docker.restartIfRequired(dockerConfig, workDir, envVars) + long start = System.currentTimeMillis() + int res = -1 + String cmdForLog = cleanLoggedCommand(command) + try { + environment.debug(cmdForLog) + if (docker) { + // required here for direct raw execution case (normally, start logs must appear before + // executed command) + if (!dockerManager.started) { + dockerManager.restartIfRequired(dockerConfig, workDir, envVars) + } + res = dockerConfig.exclusive + ? dockerManager.execExclusive(command, out, dockerConfig, workDir, envVars) + : dockerManager.exec(command, out) + } else { + res = environment.exec(command, out, out, workDir, envVars) } - return dockerConfig.exclusive - ? docker.execExclusive(command, out, dockerConfig, workDir, envVars) - : docker.exec(command, out) + } finally { + environment.stat(docker ? dockerManager.containerName : null, cmdForLog, start, res == 0) } - return environment.exec(command, out, out, workDir, envVars) + return res } /** diff --git a/src/main/groovy/ru/vyarus/gradle/plugin/python/service/EnvService.groovy b/src/main/groovy/ru/vyarus/gradle/plugin/python/service/EnvService.groovy index 7cbffcd..a5355e3 100644 --- a/src/main/groovy/ru/vyarus/gradle/plugin/python/service/EnvService.groovy +++ b/src/main/groovy/ru/vyarus/gradle/plugin/python/service/EnvService.groovy @@ -3,67 +3,141 @@ package ru.vyarus.gradle.plugin.python.service import groovy.transform.CompileStatic import org.gradle.api.logging.Logger import org.gradle.api.logging.Logging -import org.gradle.api.provider.Provider +import org.gradle.api.provider.Property import org.gradle.api.services.BuildService import org.gradle.api.services.BuildServiceParameters +import org.gradle.tooling.events.FinishEvent +import org.gradle.tooling.events.OperationCompletionListener import ru.vyarus.gradle.plugin.python.cmd.docker.DockerFactory +import ru.vyarus.gradle.plugin.python.service.stat.PythonStat +import ru.vyarus.gradle.plugin.python.service.stat.StatsPrinter import java.util.concurrent.ConcurrentHashMap /** - * Service holds actual link to used python path. Actual path is resolved after virtual environment creation or - * detection. + * Service manage all required state for plugin: resolved python paths, cache and stats. One service instance used + * for all projects. This service is essential for configuration cache compatibility. *

- * Also, configuration cache compatible way to listen for build finish (instead of project.gradle.buildFinished). + * All python tasks have pythonPath property, initialized with extension default value. But checkPython task + * could change this default (due to switching to virtualenv) and so this service holds all actual paths + * (for all projects). Python tasks read path value directly from service instead of task property in order to use + * correct value (could be overridden in exact task). + *

+ * Also, service is the only way to collect python execution stats in one place and manage cache instances + * (gradle properties are not available under configuration cache). Note that gradle could not "cache" python + * executions cache state in any case because configuration cache records state before actual execution (where + * all python executions occur). * * @author Vyacheslav Rusakov * @since 14.03.2024 */ -@SuppressWarnings('AbstractClassWithoutAbstractMethod') +@SuppressWarnings(['AbstractClassWithoutAbstractMethod', 'Println']) @CompileStatic -abstract class EnvService implements BuildService, AutoCloseable { +abstract class EnvService implements BuildService, OperationCompletionListener, AutoCloseable { private final Logger logger = Logging.getLogger(EnvService) private final Object sync = new Object() + // project path -- python path private final Map pythonPaths = new ConcurrentHashMap<>() - // project path -- default python path provider - private final Map> defaultProviders = new ConcurrentHashMap<>() - void defaultProvider(String projectPath, Provider provider) { - defaultProviders.put(projectPath, provider) + // caches managed in service to proper support for configuration cache (be able to use same cache instances) + private final Map> caches = new ConcurrentHashMap<>() + // python execution statistics + private final List stats = [] + + /** + * Called by {@link ru.vyarus.gradle.plugin.python.task.CheckPythonTask} to set actual path (counting + * virtual environment). Would be called one or two times for each project: first default value and second when + * switching to environment + * + * @param projectPath project path + * @param pythonPath python path to use + */ + void setPythonPath(String projectPath, String pythonPath) { + synchronized (sync) { + if (pythonPath != null) { + this.pythonPaths.put(projectPath, pythonPath) + } else { + this.pythonPaths.remove(projectPath) + } + } + if (parameters.debug.get()) { + println("[DEBUG] set python path for '$projectPath' to $pythonPath") + } + logger.info('Python path for {} changed to {}', projectPath, pythonPath) } + /** + * {@link ru.vyarus.gradle.plugin.python.task.CheckPythonTask} always runs before other python tasks in order to + * select correct python environment (create virtual environment if required). All other tasks have to use + * service value directly because only it would contain actual path (tasks pythonPath properties would only + * contain extension default). + * + * @param projectPath project path + * @return actual python path to use in all tasks (which honor virtual environments) + */ String getPythonPath(String projectPath) { synchronized (sync) { - String path = pythonPaths.get(projectPath) - // lazy init - if (path == null) { - Provider init = defaultProviders.remove(projectPath) - if (init != null) { - path = init.get() - if (path != null) { - pythonPaths.put(projectPath, path) - } - } - } - return path + return pythonPaths.get(projectPath) } } - void setPythonPath(String projectPath, String pythonPath) { + /** + * NOTE: Don't use directly for configuration-cacheable objects (otherwise there would be different maps in all + * places when gradle is working from configuration cache). + * Use {@link ru.vyarus.gradle.plugin.python.service.value.CacheValueSource}. + * + * @param project project path + * @return project-wide cache instance + */ + Map getCache(String project) { synchronized (sync) { - logger.info('Python path for {} changed to {}', projectPath, pythonPath) - this.pythonPaths.put(projectPath, pythonPath) + Map res = caches.get(project) + if (res == null) { + res = new ConcurrentHashMap<>() + caches.put(project, res) + } + return res } } + /** + * NOTE: don't use directly! Instead use {@link ru.vyarus.gradle.plugin.python.service.value.StatsValueSource}. + * + * @return stats container which must be used by all python instance + */ + List getStats() { + return stats + } + + @Override + @SuppressWarnings('EmptyMethodInAbstractClass') + void onFinish(FinishEvent finishEvent) { + // not used - jus a way to prevent killing service too early + } + @Override void close() throws Exception { - logger.info('Shutdown docker containers ({} active)', DockerFactory.activeContainersCount) + if (parameters.debug.get()) { + println "[DEBUG] Shutdown python docker containers ${DockerFactory.activeContainersCount}" + } // close started docker containers at the end (mainly for tests, because docker instances are // project-specific and there would be problem in gradle tests always started in new dir) DockerFactory.shutdownAll() + if (parameters.printStats.get()) { + String stats = StatsPrinter.print(stats) + if (stats != null && !stats.empty) { + println stats + } + } } + + interface Params extends BuildServiceParameters { + Property getPrintStats() + + Property getDebug() + } + } diff --git a/src/main/groovy/ru/vyarus/gradle/plugin/python/service/stat/PythonStat.groovy b/src/main/groovy/ru/vyarus/gradle/plugin/python/service/stat/PythonStat.groovy new file mode 100644 index 0000000..3bdc5e2 --- /dev/null +++ b/src/main/groovy/ru/vyarus/gradle/plugin/python/service/stat/PythonStat.groovy @@ -0,0 +1,46 @@ +package ru.vyarus.gradle.plugin.python.service.stat + +import groovy.transform.CompileStatic +import org.jetbrains.annotations.NotNull + +/** + * Python command execution statistic. Also tracks internal direct commands execution (just simpler to count all), + * + * @author Vyacheslav Rusakov + * @since 22.03.2024 + */ +@CompileStatic +class PythonStat implements Comparable { + // docker + String containerName + String projectPath + String taskName + String cmd + long start + long duration + boolean success + + boolean parallel + + @Override + int compareTo(@NotNull PythonStat pythonStat) { + return start <=> pythonStat.start + } + + boolean inParallel(PythonStat stat) { + return startIn(this, stat) || startIn(stat, this) + } + + String getFullTaskName() { + return "$projectPath:$taskName".replaceAll('::', ':') + } + + @Override + String toString() { + return "$fullTaskName:${System.identityHashCode(this)}" + } + + private static boolean startIn(PythonStat stat, PythonStat stat2) { + return stat.start <= stat2.start && stat.start + stat.duration > stat2.start + } +} diff --git a/src/main/groovy/ru/vyarus/gradle/plugin/python/service/stat/StatsPrinter.groovy b/src/main/groovy/ru/vyarus/gradle/plugin/python/service/stat/StatsPrinter.groovy new file mode 100644 index 0000000..92ea924 --- /dev/null +++ b/src/main/groovy/ru/vyarus/gradle/plugin/python/service/stat/StatsPrinter.groovy @@ -0,0 +1,83 @@ +package ru.vyarus.gradle.plugin.python.service.stat + +import groovy.transform.CompileStatic +import ru.vyarus.gradle.plugin.python.util.DurationFormatter + +import java.text.SimpleDateFormat + +/** + * Python execution statistics print utility. + * + * @author Vyacheslav Rusakov + * @since 28.03.2024 + */ +@CompileStatic +class StatsPrinter { + + @SuppressWarnings(['SimpleDateFormatMissingLocale', 'Println']) + static String print(List stats) { + if (stats.empty) { + return '' + } + Set sorted = new TreeSet(stats) + StringBuilder res = new StringBuilder('\nPython execution stats:\n\n') + boolean dockerUsed = stats.stream().anyMatch { it.containerName != null } + SimpleDateFormat timeFormat = new SimpleDateFormat('HH:mm:ss:SSS') + StatCollector collector = new StatCollector(sorted) + String format = dockerUsed ? '%-37s %-3s%-12s %-20s %-10s %-8s %s%n' + : '%-37s %-3s%-12s %s %-10s %-8s %s%n' + res.append(String.format( + format, 'task', '', 'started', dockerUsed ? 'docker container' : '', 'duration', '', '')) + + for (PythonStat stat : (sorted)) { + collector.collect() + res.append(String.format(format, stat.fullTaskName, stat.parallel ? '||' : '', + timeFormat.format(stat.start), + stat.containerName ?: '', DurationFormatter.format(stat.duration), + stat.success ? '' : 'FAILED', stat.cmd)) + } + res.append('\n Executed ').append(stats.size()).append(' commands in ') + .append(DurationFormatter.format(collector.overall)).append(' (overall)\n') + + if (!collector.duplicates.isEmpty()) { + res.append('\n Duplicate executions:\n') + collector.duplicates.each { + res.append("\n\t\t$it.key (${it.value.size()})\n") + it.value.each { + res.append("\t\t\t$it.fullTaskName\n") + } + } + } + return res.toString() + } + + @SuppressWarnings('NestedForLoop') + static class StatCollector { + long overall = 0 + + Map> duplicates = [:] + + StatCollector(Set stats) { + for (PythonStat stat : stats) { + for (PythonStat stat2 : stats) { + if (stat != stat2 && stat.inParallel(stat2)) { + stat.parallel = true + stat2.parallel = true + } + } + + List dups = duplicates.get(stat.cmd) + if (dups == null) { + dups = [] + duplicates.put(stat.cmd, dups) + } + dups.add(stat) + + overall += stat.duration + } + duplicates.removeAll { + it.value.size() == 1 + } + } + } +} diff --git a/src/main/groovy/ru/vyarus/gradle/plugin/python/service/value/CacheValueSource.groovy b/src/main/groovy/ru/vyarus/gradle/plugin/python/service/value/CacheValueSource.groovy new file mode 100644 index 0000000..8867455 --- /dev/null +++ b/src/main/groovy/ru/vyarus/gradle/plugin/python/service/value/CacheValueSource.groovy @@ -0,0 +1,29 @@ +package ru.vyarus.gradle.plugin.python.service.value + +import groovy.transform.CompileStatic +import org.gradle.api.provider.Property +import org.gradle.api.provider.ValueSource +import org.gradle.api.provider.ValueSourceParameters +import ru.vyarus.gradle.plugin.python.service.EnvService + +/** + * Required to prevent configuration cache from storing inner cache maps (which would make all python instances + * depend on its own cache map, making cache useless). + * + * @author Vyacheslav Rusakov + * @since 26.03.2024 + */ +@CompileStatic +@SuppressWarnings('AbstractClassWithoutAbstractMethod') +abstract class CacheValueSource implements ValueSource, CacheParams> { + + Map obtain() { + return parameters.service.get() + .getCache(parameters.project.get()) + } + + interface CacheParams extends ValueSourceParameters { + Property getService() + Property getProject() + } +} diff --git a/src/main/groovy/ru/vyarus/gradle/plugin/python/service/value/StatsValueSource.groovy b/src/main/groovy/ru/vyarus/gradle/plugin/python/service/value/StatsValueSource.groovy new file mode 100644 index 0000000..4afe676 --- /dev/null +++ b/src/main/groovy/ru/vyarus/gradle/plugin/python/service/value/StatsValueSource.groovy @@ -0,0 +1,28 @@ +package ru.vyarus.gradle.plugin.python.service.value + +import groovy.transform.CompileStatic +import org.gradle.api.provider.Property +import org.gradle.api.provider.ValueSource +import org.gradle.api.provider.ValueSourceParameters +import ru.vyarus.gradle.plugin.python.service.EnvService +import ru.vyarus.gradle.plugin.python.service.stat.PythonStat + +/** + * Required to prevent configuration cache from storing inner stats list (which would make all python instances + * depend on its own stats list, hiding stats). + * + * @author Vyacheslav Rusakov + * @since 26.03.2024 + */ +@CompileStatic +@SuppressWarnings('AbstractClassWithoutAbstractMethod') +abstract class StatsValueSource implements ValueSource, StatsParams> { + + List obtain() { + return parameters.service.get().stats + } + + interface StatsParams extends ValueSourceParameters { + Property getService() + } +} diff --git a/src/main/groovy/ru/vyarus/gradle/plugin/python/task/BasePythonTask.groovy b/src/main/groovy/ru/vyarus/gradle/plugin/python/task/BasePythonTask.groovy index 60b3459..a4e903a 100644 --- a/src/main/groovy/ru/vyarus/gradle/plugin/python/task/BasePythonTask.groovy +++ b/src/main/groovy/ru/vyarus/gradle/plugin/python/task/BasePythonTask.groovy @@ -17,7 +17,7 @@ import ru.vyarus.gradle.plugin.python.cmd.docker.ContainerManager import ru.vyarus.gradle.plugin.python.cmd.docker.DockerConfig import ru.vyarus.gradle.plugin.python.cmd.docker.DockerFactory import ru.vyarus.gradle.plugin.python.cmd.env.Environment -import ru.vyarus.gradle.plugin.python.cmd.env.GradleEnvironment +import ru.vyarus.gradle.plugin.python.service.EnvService import ru.vyarus.gradle.plugin.python.util.CliUtils import ru.vyarus.gradle.plugin.python.util.OutputLogger @@ -32,21 +32,35 @@ import java.nio.file.Path * @since 01.12.2017 */ @CompileStatic -class BasePythonTask extends ConventionTask { +abstract class BasePythonTask extends ConventionTask { /** * Path to directory with python executable. Not required if python installed globally. * Automatically set from {@link ru.vyarus.gradle.plugin.python.PythonExtension#pythonPath}, but could * be overridden manually. + *

+ * NOTE: Normally, this property would be ignored and python resolved by checkPython task would be used. + * In order to force this setting (e.g. to use global python for exact python task) set + * {@link #useCustomPython} to true. */ @Input @Optional String pythonPath + /** + * By default, {@link #pythonPath} property is ignored and used python resolved by checkPython task. This option + * must be enabled in order to force manually configured python path usage instead of global python settings + * (in most cases, virtual environment, selected by checkPython task). + */ + @Input + Boolean useCustomPython = false + /** * Python binary name. When empty: use python3 or python for linux and python for windows. * Automatically set from {@link ru.vyarus.gradle.plugin.python.PythonExtension#pythonBinary}, but could * be overridden manually. + *

+ * NOTE: binary is ignored when pythonPath set */ @Input @Optional @@ -100,11 +114,16 @@ class BasePythonTask extends ConventionTask { @Nested DockerEnv docker = project.objects.newInstance(DockerEnv) + // service holds actual pythonPath and global cache. All tasks would use it for lazy default, but check task + // could CHANGE it (that's why it's important to call it before all other tasks) + @Internal + abstract Property getEnvService() + // Special object - lightweight project replacement to avoid calling project in actions. // Plus, this object is configuration cache friendly // Applied to all python tasks in plugin @Internal - Environment gradleEnv = GradleEnvironment.create(project) + abstract Property getGradleEnv() private Python pythonCache @@ -203,7 +222,7 @@ class BasePythonTask extends ConventionTask { * @param dir string path or File object for local directory to change file permissions on within docker */ void dockerChown(String dir) { - dockerChown(gradleEnv.file(dir).toPath()) + dockerChown(gradleEnv.get().file(dir).toPath()) } /** @@ -228,8 +247,10 @@ class BasePythonTask extends ConventionTask { @Internal protected Python getPython() { - // changes to path or binary would trigger python object re-creation - buildPython(getPythonPath(), getPythonBinary()) + // use service to resolve pythonPath instead of property (but only when custom path not forced) + String path = getUseCustomPython() ? getPythonPath() + : envService.get().getPythonPath(gradleEnv.get().projectPath) + buildPython(path, getPythonBinary()) } /** @@ -243,7 +264,7 @@ class BasePythonTask extends ConventionTask { return } - Path projectDir = gradleEnv.rootDir.toPath() + Path projectDir = gradleEnv.get().rootDir.toPath() int uid = (int) Files.getAttribute(projectDir, 'unix:uid', LinkOption.NOFOLLOW_LINKS) int gid = (int) Files.getAttribute(projectDir, 'unix:gid', LinkOption.NOFOLLOW_LINKS) dockerExec(['chown', '-Rh', "$uid:$gid", dir.toAbsolutePath()]) @@ -266,7 +287,7 @@ class BasePythonTask extends ConventionTask { } String[] args = CliUtils.parseArgs(cmd) // it would be pre-started container (used in checkPython) - ContainerManager manager = DockerFactory.getContainer(getDocker().toConfig(), gradleEnv) + ContainerManager manager = DockerFactory.getContainer(getDocker().toConfig(), gradleEnv.get()) // restart container if task parameters differ manager.restartIfRequired(getDocker().toConfig(), getWorkDir(), getEnvironment()) // rewrite paths from host to docker fs @@ -279,7 +300,7 @@ class BasePythonTask extends ConventionTask { // note: groovy memoized can't be used because of configuration cache! private Python buildPython(String pythonPath, String pythonBinary) { if (pythonCache == null) { - pythonCache = new Python(gradleEnv, pythonPath, pythonBinary) + pythonCache = new Python(gradleEnv.get(), pythonPath, pythonBinary) .logLevel(getLogLevel()) .workDir(getWorkDir()) .pythonArgs(getPythonArgs()) diff --git a/src/main/groovy/ru/vyarus/gradle/plugin/python/task/CheckPythonTask.groovy b/src/main/groovy/ru/vyarus/gradle/plugin/python/task/CheckPythonTask.groovy index ab31cdc..aa321bc 100644 --- a/src/main/groovy/ru/vyarus/gradle/plugin/python/task/CheckPythonTask.groovy +++ b/src/main/groovy/ru/vyarus/gradle/plugin/python/task/CheckPythonTask.groovy @@ -2,9 +2,7 @@ package ru.vyarus.gradle.plugin.python.task import groovy.transform.CompileStatic import org.gradle.api.GradleException -import org.gradle.api.provider.Provider import org.gradle.api.tasks.Input -import org.gradle.api.tasks.Internal import org.gradle.api.tasks.Optional import org.gradle.api.tasks.TaskAction import org.gradle.process.internal.ExecException @@ -12,7 +10,6 @@ import ru.vyarus.gradle.plugin.python.PythonExtension import ru.vyarus.gradle.plugin.python.cmd.Pip import ru.vyarus.gradle.plugin.python.cmd.Python import ru.vyarus.gradle.plugin.python.cmd.Virtualenv -import ru.vyarus.gradle.plugin.python.service.EnvService import ru.vyarus.gradle.plugin.python.task.pip.BasePipTask import ru.vyarus.gradle.plugin.python.util.CliUtils import ru.vyarus.gradle.plugin.python.util.PythonExecutionFailed @@ -36,7 +33,7 @@ import ru.vyarus.gradle.plugin.python.util.PythonExecutionFailed */ @CompileStatic @SuppressWarnings('UnnecessaryGetter') -class CheckPythonTask extends BasePipTask { +abstract class CheckPythonTask extends BasePipTask { private static final String PROP_VENV_INSTALLED = 'virtualenv.installed' private static final Object SYNC = new Object() @@ -94,18 +91,16 @@ class CheckPythonTask extends BasePipTask { @Input boolean envCopy - // service holds actual pythonPath. All tasks would use it for lazy default, but check task could CHANGE it - // (that's why it's important to call it before all other tasks) - @Internal - Provider envService - @TaskAction @SuppressWarnings('UnnecessaryGetter') void run() { + // use extension value to initialize service default (it is the only place where it could be done) + envService.get().setPythonPath(gradleEnv.get().projectPath, getPythonPath()) + boolean envRequested = getScope() >= PythonExtension.Scope.VIRTUALENV_OR_USER Virtualenv env = envRequested // synchronize work dir between python instances - ? new Virtualenv(gradleEnv, getPythonPath(), getPythonBinary(), getEnvPath()) + ? new Virtualenv(gradleEnv.get(), getPythonPath(), getPythonBinary(), getEnvPath()) .validateSystemBinary(getValidateSystemBinary()) .withDocker(getDocker().toConfig()) .workDir(getWorkDir()) @@ -146,7 +141,7 @@ class CheckPythonTask extends BasePipTask { private void checkPython() { // important because python could change on second execution - Python python = new Python(gradleEnv, pythonPath, pythonBinary) + Python python = new Python(gradleEnv.get(), pythonPath, pythonBinary) .workDir(getWorkDir()) .environment(getEnvironment()) .validateSystemBinary(isValidateSystemBinary()) @@ -177,7 +172,7 @@ class CheckPythonTask extends BasePipTask { private void checkPip() { // important because python could change on second execution - Pip pip = new Pip(gradleEnv, getPythonPath(), getPythonBinary()) + Pip pip = new Pip(gradleEnv.get(), getPythonPath(), getPythonBinary()) .userScope(false) .workDir(getWorkDir()) .environment(getEnvironment()) @@ -204,7 +199,7 @@ class CheckPythonTask extends BasePipTask { @SuppressWarnings('MethodSize') private boolean checkEnv(Virtualenv env) { - Pip pip = new Pip(gradleEnv, getPythonPath(), getPythonBinary()) + Pip pip = new Pip(gradleEnv.get(), getPythonPath(), getPythonBinary()) .userScope(true) .breakSystemPackages(isBreakSystemPackages()) .workDir(getWorkDir()) @@ -213,17 +208,17 @@ class CheckPythonTask extends BasePipTask { .withDocker(getDocker().toConfig()) .validate() // to avoid calling pip in EACH module (in multi-module project) to verify virtualenv existence - Boolean venvInstalled = gradleEnv.globalCache(PROP_VENV_INSTALLED, null) + Boolean venvInstalled = gradleEnv.get().globalCache(PROP_VENV_INSTALLED, null) if (venvInstalled == null) { venvInstalled = pip.isInstalled(env.name) - gradleEnv.updateGlobalCache(PROP_VENV_INSTALLED, venvInstalled) + gradleEnv.get().updateGlobalCache(PROP_VENV_INSTALLED, venvInstalled) } if (!venvInstalled) { if (isInstallVirtualenv()) { // automatically install virtualenv if allowed (in --user) // by default, exact (configured) version used to avoid side effects!) pip.install(env.name + (getVirtualenvVersion() ? "==${getVirtualenvVersion()}" : '')) - gradleEnv.updateGlobalCache(PROP_VENV_INSTALLED, true) + gradleEnv.get().updateGlobalCache(PROP_VENV_INSTALLED, true) } else if (getScope() == PythonExtension.Scope.VIRTUALENV) { // virtualenv strictly required - fail throw new GradleException('Virtualenv is not installed. Please install it ' + @@ -261,8 +256,9 @@ class CheckPythonTask extends BasePipTask { @SuppressWarnings('UnnecessaryGetter') private void switchEnvironment(Virtualenv env) { // switch environment and check again - envService.get().setPythonPath(gradleEnv.projectPath, env.pythonPath) + envService.get().setPythonPath(gradleEnv.get().projectPath, env.pythonPath) this.pythonPath = env.pythonPath + // note: after changing pythonPath, configured pythonBinary would be actually ignored and so no need to change checkPython() // only if pip required or requirements file present diff --git a/src/main/groovy/ru/vyarus/gradle/plugin/python/task/PythonTask.groovy b/src/main/groovy/ru/vyarus/gradle/plugin/python/task/PythonTask.groovy index e831ad7..d8a7892 100644 --- a/src/main/groovy/ru/vyarus/gradle/plugin/python/task/PythonTask.groovy +++ b/src/main/groovy/ru/vyarus/gradle/plugin/python/task/PythonTask.groovy @@ -19,7 +19,7 @@ import ru.vyarus.gradle.plugin.python.cmd.Python * @since 11.11.2017 */ @CompileStatic -class PythonTask extends BasePythonTask { +abstract class PythonTask extends BasePythonTask { /** * Create work directory if it doesn't exist. Enabled by default. @@ -108,7 +108,7 @@ class PythonTask extends BasePythonTask { private void initWorkDirIfRequired() { String dir = getWorkDir() if (dir && isCreateWorkDir()) { - File wrkd = gradleEnv.file(dir) + File wrkd = gradleEnv.get().file(dir) if (!wrkd.exists()) { wrkd.mkdirs() } diff --git a/src/main/groovy/ru/vyarus/gradle/plugin/python/task/pip/BasePipTask.groovy b/src/main/groovy/ru/vyarus/gradle/plugin/python/task/pip/BasePipTask.groovy index ebdb7dd..2e67497 100644 --- a/src/main/groovy/ru/vyarus/gradle/plugin/python/task/pip/BasePipTask.groovy +++ b/src/main/groovy/ru/vyarus/gradle/plugin/python/task/pip/BasePipTask.groovy @@ -16,7 +16,7 @@ import ru.vyarus.gradle.plugin.python.util.RequirementsReader * @since 01.12.2017 */ @CompileStatic -class BasePipTask extends BasePythonTask { +abstract class BasePipTask extends BasePythonTask { /** * List of modules to install. Module declaration format: 'name:version'. @@ -188,7 +188,7 @@ class BasePipTask extends BasePythonTask { List res = RequirementsReader.read(file) if (!res.isEmpty()) { logger.warn('{} modules to install read from requirements file: {} (strict mode)', - res.size(), RequirementsReader.relativePath(gradleEnv, file)) + res.size(), RequirementsReader.relativePath(gradleEnv.get(), file)) } requirementModulesCache = res } else { diff --git a/src/main/groovy/ru/vyarus/gradle/plugin/python/task/pip/PipInstallTask.groovy b/src/main/groovy/ru/vyarus/gradle/plugin/python/task/pip/PipInstallTask.groovy index 33fb934..d382870 100644 --- a/src/main/groovy/ru/vyarus/gradle/plugin/python/task/pip/PipInstallTask.groovy +++ b/src/main/groovy/ru/vyarus/gradle/plugin/python/task/pip/PipInstallTask.groovy @@ -20,7 +20,7 @@ import java.util.concurrent.ConcurrentHashMap * @since 11.11.2017 */ @CompileStatic -class PipInstallTask extends BasePipTask { +abstract class PipInstallTask extends BasePipTask { // sync to avoid parallel installation into THE SAME environment private static final Map SYNC = new ConcurrentHashMap<>() @@ -57,6 +57,7 @@ class PipInstallTask extends BasePipTask { private List modulesToInstallCache + @SuppressWarnings('AbstractClassWithPublicConstructor') PipInstallTask() { // providers required to workaround configuration cache Provider onlyIfProvider = project.provider { modulesInstallationRequired } @@ -82,7 +83,7 @@ class PipInstallTask extends BasePipTask { synchronized (getSync(pip.python.binaryDir)) { if (directReqsInstallRequired) { // process requirements with pip - pip.exec("install -r ${RequirementsReader.relativePath(gradleEnv, file)}") + pip.exec("install -r ${RequirementsReader.relativePath(gradleEnv.get(), file)}") } // in non strict mode requirements would be parsed manually and installed as separate modules // see BasePipTask.getAllModules() diff --git a/src/main/groovy/ru/vyarus/gradle/plugin/python/task/pip/PipListTask.groovy b/src/main/groovy/ru/vyarus/gradle/plugin/python/task/pip/PipListTask.groovy index 0c44b17..bfa98a7 100644 --- a/src/main/groovy/ru/vyarus/gradle/plugin/python/task/pip/PipListTask.groovy +++ b/src/main/groovy/ru/vyarus/gradle/plugin/python/task/pip/PipListTask.groovy @@ -14,7 +14,7 @@ import org.gradle.api.tasks.TaskAction * @since 15.12.2017 */ @CompileStatic -class PipListTask extends BasePipTask { +abstract class PipListTask extends BasePipTask { /** * To see all modules from global scope, when user scope used. diff --git a/src/main/groovy/ru/vyarus/gradle/plugin/python/task/pip/PipUpdatesTask.groovy b/src/main/groovy/ru/vyarus/gradle/plugin/python/task/pip/PipUpdatesTask.groovy index ecad345..7e58956 100644 --- a/src/main/groovy/ru/vyarus/gradle/plugin/python/task/pip/PipUpdatesTask.groovy +++ b/src/main/groovy/ru/vyarus/gradle/plugin/python/task/pip/PipUpdatesTask.groovy @@ -11,7 +11,7 @@ import org.gradle.api.tasks.TaskAction * @since 01.12.2017 */ @CompileStatic -class PipUpdatesTask extends BasePipTask { +abstract class PipUpdatesTask extends BasePipTask { /** * True to show all available updates. By default (false): show only updates for configured modules. diff --git a/src/test/groovy/ru/vyarus/gradle/plugin/python/AbstractKitTest.groovy b/src/test/groovy/ru/vyarus/gradle/plugin/python/AbstractKitTest.groovy index 0beab3a..8ea477b 100644 --- a/src/test/groovy/ru/vyarus/gradle/plugin/python/AbstractKitTest.groovy +++ b/src/test/groovy/ru/vyarus/gradle/plugin/python/AbstractKitTest.groovy @@ -8,6 +8,7 @@ import org.gradle.testkit.runner.GradleRunner import ru.vyarus.gradle.plugin.python.cmd.Virtualenv import ru.vyarus.gradle.plugin.python.cmd.env.Environment import ru.vyarus.gradle.plugin.python.cmd.env.GradleEnvironment +import ru.vyarus.gradle.plugin.python.service.EnvService import spock.lang.Specification import spock.lang.TempDir @@ -98,6 +99,13 @@ abstract class AbstractKitTest extends Specification { .replace("\r", '') } + String unifyStats(String text) { + return unifyString(text) + .replaceAll(/\d{2}:\d{2}:\d{2}:\d{3}/, '11:11:11:111') + .replaceAll(/(\d\.?)+(ms|s)\s+/, '11ms ') + .replaceAll(/11ms\s+\(overall\)/, '11ms (overall)') + } + // custom virtualenv to use for simulations Virtualenv env(String path = '.gradle/python', String binary = null) { new Virtualenv(gradleEnv(ProjectBuilder.builder() @@ -109,6 +117,12 @@ abstract class AbstractKitTest extends Specification { } Environment gradleEnv(Project project) { - GradleEnvironment.create(project) + GradleEnvironment.create(project, "gg", project.gradle.sharedServices.registerIfAbsent( + 'pythonEnvironmentService', EnvService, spec -> { + EnvService.Params params = spec.parameters as EnvService.Params + params.printStats.set(false) + params.debug.set(false) + } + ), project.provider { false }) } } diff --git a/src/test/groovy/ru/vyarus/gradle/plugin/python/AbstractTest.groovy b/src/test/groovy/ru/vyarus/gradle/plugin/python/AbstractTest.groovy index 5856426..0b6e87f 100644 --- a/src/test/groovy/ru/vyarus/gradle/plugin/python/AbstractTest.groovy +++ b/src/test/groovy/ru/vyarus/gradle/plugin/python/AbstractTest.groovy @@ -5,6 +5,7 @@ import org.gradle.api.Project import org.gradle.testfixtures.ProjectBuilder import ru.vyarus.gradle.plugin.python.cmd.env.Environment import ru.vyarus.gradle.plugin.python.cmd.env.GradleEnvironment +import ru.vyarus.gradle.plugin.python.service.EnvService import spock.lang.Specification import spock.lang.TempDir @@ -18,7 +19,8 @@ abstract class AbstractTest extends Specification { boolean isWin = Os.isFamily(Os.FAMILY_WINDOWS) - @TempDir File testProjectDir + @TempDir + File testProjectDir Project project(Closure config = null) { projectBuilder(config).build() @@ -43,7 +45,13 @@ abstract class AbstractTest extends Specification { } Environment gradleEnv(Project project) { - GradleEnvironment.create(project) + GradleEnvironment.create(project, "gg", project.gradle.sharedServices.registerIfAbsent( + 'pythonEnvironmentService', EnvService, spec -> { + EnvService.Params params = spec.parameters as EnvService.Params + // only root project value counted for print stats activation + params.printStats.set(false) + params.debug.set(false) + }), project.provider { false }) } static class ExtendedProjectBuilder { diff --git a/src/test/groovy/ru/vyarus/gradle/plugin/python/ConfigurationCacheSupportKitTest.groovy b/src/test/groovy/ru/vyarus/gradle/plugin/python/ConfigurationCacheSupportKitTest.groovy index 4f3fd2c..57eb094 100644 --- a/src/test/groovy/ru/vyarus/gradle/plugin/python/ConfigurationCacheSupportKitTest.groovy +++ b/src/test/groovy/ru/vyarus/gradle/plugin/python/ConfigurationCacheSupportKitTest.groovy @@ -112,7 +112,6 @@ class ConfigurationCacheSupportKitTest extends AbstractKitTest { """ when: "run task" - debug() BuildResult result = run('--configuration-cache', '--configuration-cache-problems=warn', 'checkPython') then: "no configuration cache incompatibilities" @@ -149,6 +148,7 @@ class ConfigurationCacheSupportKitTest extends AbstractKitTest { """ when: "run task" + println "\n\n----------------------------------------------------------------------------------" BuildResult result = run('--configuration-cache', '--configuration-cache-problems=warn', 'pipList') then: "no configuration cache incompatibilities" @@ -193,6 +193,7 @@ class ConfigurationCacheSupportKitTest extends AbstractKitTest { """ when: "run task" + println "\n\n----------------------------------------------------------------------------------" BuildResult result = run('--configuration-cache', '--configuration-cache-problems=warn', 'pipUpdates') then: "no configuration cache incompatibilities" @@ -239,6 +240,7 @@ class ConfigurationCacheSupportKitTest extends AbstractKitTest { when: "run task" + println "\n\n----------------------------------------------------------------------------------" result = run('--configuration-cache', '--configuration-cache-problems=warn', 'pipUpdates') then: "no configuration cache incompatibilities" diff --git a/src/test/groovy/ru/vyarus/gradle/plugin/python/StatsKitTest.groovy b/src/test/groovy/ru/vyarus/gradle/plugin/python/StatsKitTest.groovy new file mode 100644 index 0000000..25a2184 --- /dev/null +++ b/src/test/groovy/ru/vyarus/gradle/plugin/python/StatsKitTest.groovy @@ -0,0 +1,168 @@ +package ru.vyarus.gradle.plugin.python + +import org.gradle.testkit.runner.BuildResult +import org.gradle.testkit.runner.TaskOutcome +import ru.vyarus.gradle.plugin.python.cmd.Pip + +/** + * @author Vyacheslav Rusakov + * @since 28.03.2024 + */ +class StatsKitTest extends AbstractKitTest { + + def "Check env plugin execution"() { + setup: + build """ + plugins { + id 'ru.vyarus.use-python' + } + + python { + scope = VIRTUALENV + pip 'extract-msg:0.28.0' + + printStats = true + } + + tasks.register('sample', PythonTask) { + command = '-c print(\\'samplee\\')' + } + + """ + + when: "run task" + BuildResult result = run('sample') + + then: "task successful" + result.task(':sample').outcome == TaskOutcome.SUCCESS + result.output =~ /extract-msg\s+0.28.0/ + result.output.contains('samplee') + + unifyStats(result.output).contains("""task started duration +:checkPython 11:11:11:111 11ms python3 --version +:checkPython 11:11:11:111 11ms python3 -c exec("import sys;ver=sys.version_info;print(str(ver.major)+'.'+str(ver.minor)+'.'+str(ver.micro));print(sys.prefix);print(sys.executable)") +:checkPython 11:11:11:111 11ms python3 -m pip --version +:checkPython 11:11:11:111 11ms python3 -m pip show virtualenv +:checkPython 11:11:11:111 11ms python3 -m virtualenv --version +:checkPython 11:11:11:111 11ms python3 -m virtualenv .gradle/python +:checkPython 11:11:11:111 11ms .gradle/python/bin/python -c exec("import sys;ver=sys.version_info;print(str(ver.major)+'.'+str(ver.minor)+'.'+str(ver.micro));print(sys.prefix);print(sys.executable)") +:checkPython 11:11:11:111 11ms .gradle/python/bin/python -m pip --version +:pipInstall 11:11:11:111 11ms .gradle/python/bin/python -m pip freeze +:pipInstall 11:11:11:111 11ms .gradle/python/bin/python -m pip install extract-msg==0.28.0 +:pipInstall 11:11:11:111 11ms .gradle/python/bin/python -m pip list --format=columns +:sample 11:11:11:111 11ms .gradle/python/bin/python -c exec("print('samplee')") + + Executed 12 commands in 11ms (overall) +""") + } + + + def "Check failed task"() { + setup: + build """ + plugins { + id 'ru.vyarus.use-python' + } + + python { + printStats = true + } + + tasks.register('sample', PythonTask) { + command = '-c printt(\\'samplee\\')' + } + + """ + + when: "run task" + BuildResult result = runFailed('sample') + + then: "task failed" + unifyStats(result.output).contains("""Python execution stats: + +task started duration +:checkPython 11:11:11:111 11ms python3 --version +:checkPython 11:11:11:111 11ms python3 -c exec("import sys;ver=sys.version_info;print(str(ver.major)+'.'+str(ver.minor)+'.'+str(ver.micro));print(sys.prefix);print(sys.executable)") +:sample 11:11:11:111 11ms FAILED python3 -c exec("printt('samplee')") + + Executed 3 commands in 11ms (overall) +""") + } + + def "Check list task"() { + + setup: + // to show at least something + new Pip(gradleEnv()).install('extract-msg==0.28.0') + + build """ + plugins { + id 'ru.vyarus.use-python' + } + + python.scope = USER + python.printStats = true + """ + + when: "run task" + BuildResult result = run('pipList') + + then: "extract-msg update detected" + result.task(':pipList').outcome == TaskOutcome.SUCCESS + result.output.contains('pip list --format=columns --user') + result.output =~ /extract-msg\s+0.28.0/ + + unifyStats(result.output).contains("""Python execution stats: + +task started duration +:checkPython 11:11:11:111 11ms python3 --version +:checkPython 11:11:11:111 11ms python3 -c exec("import sys;ver=sys.version_info;print(str(ver.major)+'.'+str(ver.minor)+'.'+str(ver.micro));print(sys.prefix);print(sys.executable)") +:pipList 11:11:11:111 11ms python3 -m pip list --format=columns --user + + Executed 3 commands in 11ms (overall)""") + } + + + def "Check updates detected in environment"() { + + setup: + build """ + plugins { + id 'ru.vyarus.use-python' + } + + python { + scope = VIRTUALENV + pip 'extract-msg:0.28.0' + + printStats = true + } + + """ + + when: "install old version" + BuildResult result = run('pipInstall') + then: "installed" + result.task(':pipInstall').outcome == TaskOutcome.SUCCESS + result.output.contains('pip install extract-msg') + + + when: "run task" + result = run('pipUpdates') + + then: "extract-msg update detected" + result.task(':pipUpdates').outcome == TaskOutcome.SUCCESS + result.output.contains('The following modules could be updated:') + result.output =~ /extract-msg\s+0.28.0/ + + unifyStats(result.output).contains("""Python execution stats: + +task started duration +:checkPython 11:11:11:111 11ms python3 --version +:checkPython 11:11:11:111 11ms .gradle/python/bin/python -c exec("import sys;ver=sys.version_info;print(str(ver.major)+'.'+str(ver.minor)+'.'+str(ver.micro));print(sys.prefix);print(sys.executable)") +:checkPython 11:11:11:111 11ms .gradle/python/bin/python -m pip --version +:pipUpdates 11:11:11:111 11ms .gradle/python/bin/python -m pip list -o -l --format=columns + + Executed 4 commands in 11ms (overall)""") + } +} diff --git a/src/test/groovy/ru/vyarus/gradle/plugin/python/UseCustomPythonForTaskKitTest.groovy b/src/test/groovy/ru/vyarus/gradle/plugin/python/UseCustomPythonForTaskKitTest.groovy new file mode 100644 index 0000000..792615f --- /dev/null +++ b/src/test/groovy/ru/vyarus/gradle/plugin/python/UseCustomPythonForTaskKitTest.groovy @@ -0,0 +1,42 @@ +package ru.vyarus.gradle.plugin.python + +import org.gradle.testkit.runner.BuildResult +import org.gradle.testkit.runner.TaskOutcome + +/** + * @author Vyacheslav Rusakov + * @since 28.03.2024 + */ +class UseCustomPythonForTaskKitTest extends AbstractKitTest { + + def "Check env plugin execution"() { + setup: + build """ + plugins { + id 'ru.vyarus.use-python' + } + + python { + scope = VIRTUALENV + pip 'extract-msg:0.28.0' + } + + tasks.register('sample', PythonTask) { + // force global python usage instead of virtualenv + pythonPath = null + useCustomPython = true + command = '-c print(\\'samplee\\')' + } + + """ + + when: "run task" + BuildResult result = run('sample') + + then: "task successful" + result.task(':sample').outcome == TaskOutcome.SUCCESS + result.output =~ /extract-msg\s+0.28.0/ + result.output.contains('samplee') + result.output =~ /(?m)\[python] python(3)? -c ${isWin ? 'print' : 'exec\\(\"print'}/ + } +} diff --git a/src/test/groovy/ru/vyarus/gradle/plugin/python/cmd/AbstractCliMockSupport.groovy b/src/test/groovy/ru/vyarus/gradle/plugin/python/cmd/AbstractCliMockSupport.groovy index 370366f..07a2f82 100644 --- a/src/test/groovy/ru/vyarus/gradle/plugin/python/cmd/AbstractCliMockSupport.groovy +++ b/src/test/groovy/ru/vyarus/gradle/plugin/python/cmd/AbstractCliMockSupport.groovy @@ -8,12 +8,16 @@ import org.gradle.api.logging.Logger import org.gradle.api.model.ObjectFactory import org.gradle.api.plugins.ExtensionContainer import org.gradle.api.plugins.ExtraPropertiesExtension +import org.gradle.api.provider.Provider +import org.gradle.api.provider.ProviderFactory import org.gradle.internal.file.PathToFileResolver import org.gradle.process.ExecOperations import org.gradle.process.ExecSpec import org.gradle.process.internal.DefaultExecSpec import ru.vyarus.gradle.plugin.python.cmd.env.Environment import ru.vyarus.gradle.plugin.python.cmd.env.GradleEnvironment +import ru.vyarus.gradle.plugin.python.service.stat.PythonStat +import ru.vyarus.gradle.plugin.python.service.value.CacheValueSource import ru.vyarus.gradle.plugin.python.util.ExecRes import ru.vyarus.gradle.plugin.python.util.TestLogger import spock.lang.Specification @@ -26,7 +30,8 @@ import spock.lang.TempDir abstract class AbstractCliMockSupport extends Specification { // used to overcome manual file existence check on win - @TempDir File dir + @TempDir + File dir Project project TestLogger logger @@ -47,7 +52,7 @@ abstract class AbstractCliMockSupport extends Specification { project.getProjectDir() >> { dir } project.file(_) >> { new File(dir, it[0]) } project.getRootProject() >> { project } - project.findProperty(_ as String) >> { args -> extraProps.get(args[0])} + project.findProperty(_ as String) >> { args -> extraProps.get(args[0]) } // required for GradleEnvironment ObjectFactory objects = Stub(ObjectFactory) ExecOperations exec = Stub(ExecOperations) @@ -58,14 +63,26 @@ abstract class AbstractCliMockSupport extends Specification { List params = [exec, fs] params.addAll(args[1] as Object[]) // have to use special class because GradleEnvironment is abstract (assume gradle injection) - GradleEnv.newInstance(params as Object[]) } + GradleEnv.newInstance(params as Object[]) + } project.getObjects() >> { objects } + ProviderFactory providers = Stub(ProviderFactory) + providers.of(_, _) >> { args -> + Class cls = args[0] as Class + if (CacheValueSource.isAssignableFrom(cls)) { + return { [:] } as Provider + } else { + return { [] } as Provider + } + } + project.getProviders() >> { providers } + def ext = Stub(ExtensionContainer) project.getExtensions() >> { ext } def props = Stub(ExtraPropertiesExtension) ext.getExtraProperties() >> { props } - props.set(_ as String, _) >> { args-> extraProps.put(args[0], args[1]) } + props.set(_ as String, _) >> { args -> extraProps.put(args[0], args[1]) } props.get(_ as String) >> { args -> extraProps.get(args[0]) } props.has(_ as String) >> { args -> extraProps.containsKey(args[0]) } } @@ -75,7 +92,7 @@ abstract class AbstractCliMockSupport extends Specification { } Environment gradleEnv(Project project) { - GradleEnvironment.create(project) + GradleEnvironment.create(project, "gg", {} as Provider, { false } as Provider) } // use to provide specialized output for executed commands @@ -100,7 +117,7 @@ abstract class AbstractCliMockSupport extends Specification { out = v } } - if (out==output) { + if (out == output) { println ">> Default execution, output: $out" } @@ -116,9 +133,13 @@ abstract class AbstractCliMockSupport extends Specification { ExecOperations exec FileOperations fs - GradleEnv(ExecOperations exec, FileOperations fs, Logger logger, File projectDir, File rootDir, String rootName, - String projectPath, Map globalCache, Map projectCache) { - super(logger, projectDir, rootDir, rootName, projectPath, globalCache, projectCache) + GradleEnv(ExecOperations exec, FileOperations fs, Logger logger, File projectDir, File rootDir, String rootName, + String projectPath, String taskName, + Provider> globalCache, + Provider> projectCache, + Provider> stats, + Provider debug) { + super(logger, projectDir, rootDir, rootName, projectPath, taskName, globalCache, projectCache, stats, debug) this.exec = exec this.fs = fs }