diff --git a/.circleci/config.yml b/.circleci/config.yml
index 7fae2f8a8..9fa114d35 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -25,10 +25,12 @@ jobs:
             mkdir -p $HOME/.gradle
             echo "gradle.publish.key=$PUBLISH_KEY" > $HOME/.gradle/gradle.properties
             echo "gradle.publish.secret=$PUBLISH_SECRET" >> $HOME/.gradle/gradle.properties
-            ./gradlew -Dbootstrap publishToMavenLocal
+            ./gradlew publishToMavenLocal
       - run:
           name: Build
-          command: ./gradlew clean ciBuild --stacktrace
+          command: |
+            ./gradlew clean ciBuild --full-stacktrace
+            (cd testapp && ../gradlew help -PskipValidation)
       - store_test_results:
           path: play/plugin/build/test-results
       - save_cache:
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 698cd364f..65443b410 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -2,9 +2,11 @@
 
 ### Building the plugin
 
-1. Run `./gradlew publishToMavenLocal -Dbootstrap`
+1. Run `./gradlew publishToMavenLocal`
 1. Make your changes
-1. Test them live by running `./gradlew t[StandardTaskName]`, e.g. `./gradlew tPublishBundle`
+1. Run the tests with `./gradlew test`
+1. Ensure your changes work in a live environment: `(cd testapp && ../gradlew taskName)`, e.g.
+   `(cd testapp && ../gradlew publishBundle)`
 
 ### Adding new features
 
diff --git a/build.gradle.kts b/build.gradle.kts
index 1393f52d3..dbf671400 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -53,5 +53,3 @@ allprojects {
         }
     }
 }
-
-apply(from = "testapp-setup.gradle.kts")
diff --git a/settings.gradle.kts b/settings.gradle.kts
index ac8a3dd67..3c33f02a3 100644
--- a/settings.gradle.kts
+++ b/settings.gradle.kts
@@ -7,8 +7,3 @@ include(
 
         ":play:plugin", ":play:android-publisher"
 )
-
-// Since the test app needs ./gradlew publishToMavenLocal to work, this enables bootstrapping it.
-if (System.getProperty("bootstrap") == null) {
-    includeBuild("testapp")
-}
diff --git a/testapp-setup.gradle.kts b/testapp-setup.gradle.kts
deleted file mode 100644
index 3ce30ffa6..000000000
--- a/testapp-setup.gradle.kts
+++ /dev/null
@@ -1,25 +0,0 @@
-private val publicGppTasks = listOf(
-        "bootstrap",
-        "bootstrapRelease",
-        "promoteArtifact",
-        "promoteReleaseArtifact",
-        "publish",
-        "publishApk",
-        "publishBundle",
-        "publishListing",
-        "publishProducts",
-        "publishRelease",
-        "publishReleaseApk",
-        "publishReleaseBundle",
-        "publishReleaseListing",
-        "publishReleaseProducts",
-        "uploadReleasePrivateApk",
-        "uploadReleasePrivateBundle",
-        "installReleasePrivateArtifact"
-)
-
-for (name in publicGppTasks) {
-    tasks.register("testapp${name.capitalize()}") {
-        dependsOn(gradle.includedBuild("testapp").task(":$name"))
-    }
-}
diff --git a/testapp/build.gradle.kts b/testapp/build.gradle.kts
index 9fe4de54c..0283bd331 100644
--- a/testapp/build.gradle.kts
+++ b/testapp/build.gradle.kts
@@ -1,5 +1,9 @@
 import com.android.build.gradle.BaseExtension
+import com.github.triplet.gradle.common.utils.orNull
+import com.github.triplet.gradle.common.utils.safeCreateNewFile
 import com.github.triplet.gradle.play.PlayPublisherExtension
+import com.google.common.hash.Hashing
+import com.google.common.io.Files
 import java.io.FileInputStream
 import java.util.Properties
 
@@ -18,6 +22,11 @@ buildscript {
     }
 }
 
+buildScan {
+    termsOfServiceUrl = "https://gradle.com/terms-of-service"
+    termsOfServiceAgree = "yes"
+}
+
 allprojects {
     repositories {
         google()
@@ -85,3 +94,31 @@ dependencies {
     "implementation"("androidx.multidex:multidex:2.0.1")
     "implementation"("androidx.constraintlayout:constraintlayout:1.1.3")
 }
+
+abstract class BuildReadinessValidator : DefaultTask() {
+    @TaskAction
+    fun validate() {
+        if (project.hasProperty("skipValidation")) return
+
+        val playChecksumFile = project.layout.buildDirectory
+                .file("build-validator/play").get().asFile
+        val playPlugin = File(project.rootDir.parentFile, "play/plugin/build/libs")
+
+        val oldHashes = playChecksumFile.orNull()?.readLines().orEmpty().toSet()
+        val newHashes = playPlugin.listFiles().orEmpty().map {
+            Files.asByteSource(it).hash(Hashing.sha256()).toString()
+        }.toSet()
+
+        check(oldHashes == newHashes) {
+            playChecksumFile.safeCreateNewFile().writeText(newHashes.joinToString("\n"))
+
+            "Plugin updated. Rerun command to finish build."
+        }
+    }
+}
+
+val ready = tasks.register<BuildReadinessValidator>("validateBuildReadiness") {
+    dependsOn(gradle.includedBuild("gradle-play-publisher")
+                      .task(":play:plugin:publishToMavenLocal"))
+}
+tasks.matching { it.name != "validateBuildReadiness" }.configureEach { dependsOn(ready) }
diff --git a/testapp/gradle/wrapper/gradle-wrapper.jar b/testapp/gradle/wrapper/gradle-wrapper.jar
new file mode 100644
index 000000000..cc4fdc293
Binary files /dev/null and b/testapp/gradle/wrapper/gradle-wrapper.jar differ
diff --git a/testapp/gradle/wrapper/gradle-wrapper.properties b/testapp/gradle/wrapper/gradle-wrapper.properties
new file mode 100644
index 000000000..562e2c884
--- /dev/null
+++ b/testapp/gradle/wrapper/gradle-wrapper.properties
@@ -0,0 +1,5 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.0-all.zip
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
diff --git a/testapp/settings.gradle.kts b/testapp/settings.gradle.kts
new file mode 100644
index 000000000..ec3652b5c
--- /dev/null
+++ b/testapp/settings.gradle.kts
@@ -0,0 +1,5 @@
+plugins {
+    `gradle-enterprise`
+}
+
+includeBuild("..")