add printStats option to print all executed python commands;

add debug option (for development);
add BasePythonTask.useCustomPython task option to allow custom pythonPath for commands (otherwise, pythonPath selected by checkPython task used);
fix caches and stats under configuration cache;
all tasks become abstract (would affect all custom tasks)

CHANGELOG.md

@@ -1,16 +1,21 @@
 * (BREAKING) Drop gradle 5 and 6 support
+* (BREAKING) All plugin tasks become abstract, so any custom task types should be abstract too
 * Fix `alwaysInstallModules = true` not triggers pipInstall for non-strict requirements file (#94)
   (required for case when requirements file links other files, which changes are not tracked) 
 * Add requirements file references (-r file) support under strict mode (#94) 
 * Configuration cache compatibility (#89)
 * Changed virtualenv version installed by default (python.virtualenvVersion) from 20.4.2 to 20.25.1
   (supports python 3.7 - 3.12)
 * Changed default docker image (python.docker.image) from python:3.10.8-alpine3.15 to python:3.11.8-alpine3.19
-* Add python.breakSystemPackages option: adds --break-system-packages for pip install
-  May be required on linux to install virtualenv on apt-managed python (e.g. python3.12)
-* Add host network option for docker: python.docker.useHostNetwork. 
-   Works only on linux. When enabled, all container ports automatically exposed on host 
-   and configured port mappings ignored 
+* New options (python.):
+  - breakSystemPackages: adds --break-system-packages for pip install
+    May be required on linux to install virtualenv on apt-managed python (e.g. python3.12)
+  - docker.useHostNetwork: use host network for docker 
+     Works only on linux. When enabled, all container ports automatically exposed on host 
+     and configured port mappings ignored 
+  - printStats: show all executed python commands (including plugin internal) at the end of the build (with timings)
+* To use different python for PythonTask useCustomPython = true must be declared now
+  (otherwise, pythonPath select by checkPython task would be used (and task's pythonPath ignored))
 
 ### 3.0.0 (2022-10-22)
 * (breaking) Drop gradle 5.0-5.2 support (minimum required gradle is 5.3)

src/main/groovy/ru/vyarus/gradle/plugin/python/PythonExtension.groovy

@@ -176,6 +176,23 @@ class PythonExtension {
      * By default use default virtualenv behaviour: symlink environment.
      */
     boolean envCopy
+
+    /**
+     * Print all executed python commands at the end of the build (including "invisible" internal commands, used by
+     * plugin for configuration).
+     * <p>
+     * IMPORTANT: in multi-module project only root project setting takes effect
+     */
+    boolean printStats = false
+
+    /**
+     * Enable debug messages (for cache and stats debugging). See {@link #printStats} for printing actually called
+     * commands (including invisible). Option for development purposes (especially with configuration cache).
+     * <p>
+     * IMPORTANT: in multi-module project only root project setting takes effect
+     */
+    boolean debug = false
+
     /**
      * Requirements file support. By default, plugin will search for requirements.txt file and would read all
      * declarations from there and use for manual installation (together with `python.modules` property). File could

src/main/groovy/ru/vyarus/gradle/plugin/python/PythonPlugin.groovy

@@ -7,6 +7,9 @@ import org.gradle.api.Project
 import org.gradle.api.provider.Provider
 import org.gradle.api.tasks.Delete
 import org.gradle.api.tasks.TaskProvider
+import org.gradle.build.event.BuildEventsListenerRegistry
+import ru.vyarus.gradle.plugin.python.cmd.env.Environment
+import ru.vyarus.gradle.plugin.python.cmd.env.GradleEnvironment
 import ru.vyarus.gradle.plugin.python.service.EnvService
 import ru.vyarus.gradle.plugin.python.task.BasePythonTask
 import ru.vyarus.gradle.plugin.python.task.CheckPythonTask
@@ -17,6 +20,8 @@ import ru.vyarus.gradle.plugin.python.task.pip.PipListTask
 import ru.vyarus.gradle.plugin.python.task.pip.PipUpdatesTask
 import ru.vyarus.gradle.plugin.python.util.RequirementsReader
 
+import javax.inject.Inject
+
 /**
  * Use-python plugin. Plugin requires python installed globally or configured path to python binary.
  * Alternatively, docker might be used.
@@ -32,36 +37,73 @@ import ru.vyarus.gradle.plugin.python.util.RequirementsReader
  */
 @CompileStatic
 @SuppressWarnings('DuplicateStringLiteral')
-class PythonPlugin implements Plugin<Project> {
+abstract class PythonPlugin implements Plugin<Project> {
+
+    // need to be static and public for configuration cache support
+    static void configureDockerInTask(Project project, PythonExtension.Docker docker, BasePythonTask task) {
+        task.docker.use.convention(project.provider { docker.use })
+        task.docker.image.convention(project.provider { docker.image })
+        task.docker.windows.convention(project.provider { docker.windows })
+        task.docker.useHostNetwork.convention(project.provider { docker.useHostNetwork })
+        task.docker.ports.convention(project.provider { docker.ports })
+        task.docker.exclusive.convention(false)
+    }
+
+    static PythonExtension findRootExtension(Project project) {
+        PythonExtension rootExt = null
+        Project prj = project
+        while (prj != null) {
+            PythonExtension cand = prj.extensions.findByType(PythonExtension)
+            if (cand != null) {
+                rootExt = cand
+            }
+            prj = prj.parent
+        }
+        return rootExt
+    }
 
-    private Provider<EnvService> envService
+    @Inject
+    abstract BuildEventsListenerRegistry getEventsListenerRegistry()
 
     @Override
     void apply(Project project) {
         PythonExtension extension = project.extensions.create('python', PythonExtension, project)
 
-        initService(project, extension)
+        Provider<EnvService> envService = initService(project)
 
         // simplify direct tasks usage
         project.extensions.extraProperties.set(PipInstallTask.simpleName, PipInstallTask)
         project.extensions.extraProperties.set(PythonTask.simpleName, PythonTask)
         // configuration shortcut
         PythonExtension.Scope.values().each { project.extensions.extraProperties.set(it.name(), it) }
 
-        createTasks(project, extension)
+        createTasks(project, extension, envService)
     }
 
-    private void initService(Project project, PythonExtension extension) {
+    private Provider<EnvService> initService(Project project) {
         // service used to shutdown docker properly and hold actual python path link
-        envService = project.gradle.sharedServices.registerIfAbsent(
-                'pythonEnvironmentService', EnvService, spec -> { })
+        Provider<EnvService> envService = project.gradle.sharedServices.registerIfAbsent(
+                'pythonEnvironmentService', EnvService, spec -> {
+            // root extension used
+            PythonExtension rootExt = findRootExtension(project)
+
+            EnvService.Params params = spec.parameters as EnvService.Params
+            // only root project value counted for print stats activation
+            params.printStats.set(project.provider { rootExt.printStats })
+            params.debug.set(project.provider { rootExt.debug })
+        })
+
+        // it is not required, but used to prevent KILLING service too early under configuration cache
+        eventsListenerRegistry.onTaskCompletion(envService)
+
+        // IMPORTANT: do not try to obtain service here (e.g. to init it) because it would cause eager parameters
+        // resolution!!!
 
-        // can't use service properties because each project in multi-module project must have unique path
-        envService.get().defaultProvider(project.path, { extension.pythonPath } as Provider<String>)
+        return envService
     }
 
     @SuppressWarnings('BuilderMethodWithSideEffects')
-    private void createTasks(Project project, PythonExtension extension) {
+    private void createTasks(Project project, PythonExtension extension, Provider<EnvService> envService) {
         // validate installed python
         TaskProvider<CheckPythonTask> checkTask = project.tasks.register('checkPython', CheckPythonTask) {
             it.with {
@@ -97,35 +139,44 @@ class PythonPlugin implements Plugin<Project> {
             }
         }
 
-        configureDefaults(project, extension, checkTask, installTask)
+        configureDefaults(project, extension, checkTask, installTask, envService)
     }
 
     @SuppressWarnings(['MethodSize', 'AbcMetric'])
     @CompileStatic(TypeCheckingMode.SKIP)
     private void configureDefaults(Project project,
                                    PythonExtension extension,
                                    TaskProvider<CheckPythonTask> checkTask,
-                                   TaskProvider<PipInstallTask> installTask) {
-
+                                   TaskProvider<PipInstallTask> installTask,
+                                   Provider<EnvService> envService) {
         project.tasks.withType(BasePythonTask).configureEach { task ->
-            task.with {
-                // apply default path for all python tasks
-                task.conventionMapping.with {
-                    // IMPORTANT: pythonPath might change after checkPythonTask (switched to environment)
-                    pythonPath = { envService.get().getPythonPath(project.path) }
-                    pythonBinary = { extension.pythonBinary }
-                    validateSystemBinary = { extension.validateSystemBinary }
-                    // important to copy map because each task must have independent instance
-                    environment = { extension.environment ? new HashMap<>(extension.environment) : null }
-                }
-
-                // can't be implemented with convention mapping, only with properties
-                configureDockerInTask(project, extension.docker, task)
-
-                // all python tasks must be executed after check task to use correct environment (switch to virtualenv)
-                if (task.taskIdentity.type != CheckPythonTask) {
-                    dependsOn checkTask
-                }
+            task.envService.set(envService)
+            task.usesService(envService)
+
+            Environment gradleEnv = GradleEnvironment.create(project, task.name, envService,
+                    project.provider { findRootExtension(project).debug })
+            task.gradleEnv.set(gradleEnv)
+            doLast {
+                gradleEnv.printCacheState()
+            }
+
+            task.conventionMapping.with {
+                // setting default value from extension to all tasks, but tasks would actually check
+                // service for actual pythonPath before python instance creation
+                // Only checkPython task will always use this default to initialize service value
+                pythonPath = { extension.pythonPath }
+                pythonBinary = { extension.pythonBinary }
+                validateSystemBinary = { extension.validateSystemBinary }
+                // important to copy map because each task must have independent instance
+                environment = { extension.environment ? new HashMap<>(extension.environment) : null }
+            }
+
+            // can't be implemented with convention mapping, only with properties
+            configureDockerInTask(project, extension.docker, task)
+
+            // all python tasks must be executed after check task to use correct environment (switch to virtualenv)
+            if (task.taskIdentity.type != CheckPythonTask) {
+                task.dependsOn checkTask
             }
         }
 
@@ -144,7 +195,7 @@ class PythonPlugin implements Plugin<Project> {
                 breakSystemPackages = { extension.breakSystemPackages }
                 trustedHosts = { extension.trustedHosts }
                 extraIndexUrls = { extension.extraIndexUrls }
-                requirements = { RequirementsReader.find(task.gradleEnv, extension.requirements) }
+                requirements = { RequirementsReader.find(task.gradleEnv.get(), extension.requirements) }
                 strictRequirements = { extension.requirements.strict }
             }
         }
@@ -159,7 +210,6 @@ class PythonPlugin implements Plugin<Project> {
         }
 
         project.tasks.withType(CheckPythonTask).configureEach { task ->
-            task.envService = this.envService
             task.conventionMapping.with {
                 scope = { extension.scope }
                 envPath = { extension.envPath }
@@ -172,13 +222,4 @@ class PythonPlugin implements Plugin<Project> {
             }
         }
     }
-
-    private void configureDockerInTask(Project project, PythonExtension.Docker docker, BasePythonTask task) {
-        task.docker.use.convention(project.provider { docker.use })
-        task.docker.image.convention(project.provider { docker.image })
-        task.docker.windows.convention(project.provider { docker.windows })
-        task.docker.useHostNetwork.convention(project.provider { docker.useHostNetwork })
-        task.docker.ports.convention(project.provider { docker.ports })
-        task.docker.exclusive.convention(false)
-    }
 }

src/main/groovy/ru/vyarus/gradle/plugin/python/cmd/docker/ContainerManager.groovy

@@ -175,6 +175,13 @@ class ContainerManager {
         return container != null
     }
 
+    /**
+     * @return container name or null if container not started yet
+     */
+    String getContainerName() {
+        return container?.running ? container.containerName : null
+    }
+
     /**
      * Stops started container or do nothing if not started.
      */

src/main/groovy/ru/vyarus/gradle/plugin/python/cmd/docker/DockerFactory.groovy

@@ -51,9 +51,12 @@ class DockerFactory {
     /**
      * Shuts down started containers. Called at the end of the build.
      */
+    @SuppressWarnings('UnnecessaryGetter')
     static synchronized void shutdownAll() {
-        CONTAINERS.values().each { it.stop() }
-        CONTAINERS.clear()
+        if (!CONTAINERS.isEmpty()) {
+            CONTAINERS.values().each { it.stop() }
+            CONTAINERS.clear()
+        }
     }
 
     /**

src/main/groovy/ru/vyarus/gradle/plugin/python/cmd/env/Environment.groovy

@@ -7,6 +7,8 @@ import java.util.function.Supplier
 /**
  * Environment-specific apis provider. Object used as lightweight alternative to gradle {@link org.gradle.api.Project}
  * (which was used before), because project is not compatible with configuration cache.
+ * <p>
+ * NOTE: configuration cache stores entire objects (so they are created only when configuration cache is not enabled).
  *
  * @author Vyacheslav Rusakov
  * @since 15.03.2024
@@ -102,4 +104,26 @@ interface Environment {
      * @param value value
      */
     void updateGlobalCache(String key, Object value)
+
+    /**
+     * Print debug message if debug enabled.
+     *
+     * @param msg message
+     */
+    void debug(String msg)
+
+    /**
+     * Save command execution stat. Counts only python execution (possible direct docker commands ignored).
+     *
+     * @param containerName docker container name
+     * @param cmd executed command (cleared!)
+     * @param start start time
+     * @param success execution success
+     */
+    void stat(String containerName, String cmd, long start, boolean success)
+
+    /**
+     * Prints cache state (for debug), but only if debug enabled in the root project.
+     */
+    void printCacheState()
 }