Moved analytics from tools/idea

Now we use extension points, and custom hooks to modify the platform's
behaviour.

Bug: 156497102
Test: existing
Change-Id: If7a8e341fd4ec73fba59acee72f52a1c59040e3d

analytics/BUILD

@@ -0,0 +1,20 @@
+load("//tools/base/bazel:bazel.bzl", "iml_module")
+
+# managed by go/iml_to_build
+iml_module(
+    name = "analytics",
+    srcs = ["src"],
+    iml_files = ["analytics.iml"],
+    test_srcs = ["testSrc"],
+    visibility = ["//visibility:public"],
+    # do not sort: must match IML order
+    deps = [
+        "//tools/idea/.idea/libraries:kotlin-stdlib-jdk8",
+        "//tools/idea/.idea/libraries:studio-analytics-proto",
+        "//tools/idea/.idea/libraries:HdrHistogram",
+        "//tools/analytics-library/tracker:analytics-tracker[module]",
+        "//tools/idea/.idea/libraries:JUnit4[test]",
+        "//tools/idea/.idea/libraries:protobuf",
+        "//tools/idea/platform/core-api:intellij.platform.core[module]",
+    ],
+)

analytics/analytics.iml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/testSrc" isTestSource="true" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="library" name="kotlin-stdlib-jdk8" level="project" />
+    <orderEntry type="library" name="studio-analytics-proto" level="project" />
+    <orderEntry type="library" name="HdrHistogram" level="project" />
+    <orderEntry type="module" module-name="analytics-tracker" />
+    <orderEntry type="library" scope="TEST" name="JUnit4" level="project" />
+    <orderEntry type="library" name="protobuf" level="project" />
+    <orderEntry type="module" module-name="intellij.platform.core" />
+  </component>
+</module>

analytics/src/com/android/tools/analytics/HighlightingStats.kt

@@ -0,0 +1,112 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tools.analytics
+
+import com.google.wireless.android.sdk.stats.AndroidStudioEvent
+import com.google.wireless.android.sdk.stats.EditorFileType
+import com.google.wireless.android.sdk.stats.EditorFileType.GROOVY
+import com.google.wireless.android.sdk.stats.EditorFileType.JAVA
+import com.google.wireless.android.sdk.stats.EditorFileType.JSON
+import com.google.wireless.android.sdk.stats.EditorFileType.KOTLIN
+import com.google.wireless.android.sdk.stats.EditorFileType.KOTLIN_SCRIPT
+import com.google.wireless.android.sdk.stats.EditorFileType.NATIVE
+import com.google.wireless.android.sdk.stats.EditorFileType.PROPERTIES
+import com.google.wireless.android.sdk.stats.EditorFileType.UNKNOWN
+import com.google.wireless.android.sdk.stats.EditorFileType.XML
+import com.google.wireless.android.sdk.stats.EditorHighlightingStats
+import com.intellij.concurrency.JobScheduler
+import com.intellij.openapi.Disposable
+import com.intellij.openapi.application.ApplicationManager
+import com.intellij.openapi.components.BaseComponent
+import com.intellij.openapi.editor.Document
+import com.intellij.openapi.fileEditor.FileDocumentManager
+import com.intellij.openapi.util.Disposer
+import com.intellij.openapi.vfs.VirtualFile
+import org.HdrHistogram.Recorder
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.TimeUnit
+
+/**
+ * Tracks highlighting latency across file types.
+ * To log an [AndroidStudioEvent] with the collected data, call [reportHighlightingStats].
+ */
+object HighlightingStats : BaseComponent {
+  private const val MAX_LATENCY_MS = 10 * 60 * 1000 // Limit latencies to 10 minutes to ensure reasonable histogram size.
+
+  override fun initComponent() {
+    // Send reports hourly and on application close.
+    JobScheduler.getScheduler().scheduleWithFixedDelay(this::reportHighlightingStats, 1, 1, TimeUnit.HOURS)
+    Disposer.register(ApplicationManager.getApplication(), Disposable(this::reportHighlightingStats))
+  }
+
+  /**
+   * Maps file types to latency recorders.
+   * We use [Recorder] to allow thread-safe read access from background threads.
+   */
+  private val latencyRecorders = ConcurrentHashMap<EditorFileType, Recorder>()
+
+  fun recordHighlightingLatency(document: Document, latencyMs: Long) {
+    if (latencyMs < 0 || latencyMs > MAX_LATENCY_MS) return
+    val file = FileDocumentManager.getInstance().getFile(document) ?: return
+    val fileType = convertFileType(file)
+    val recorder = latencyRecorders.computeIfAbsent(fileType) { Recorder(1) }
+    recorder.recordValue(latencyMs)
+  }
+
+  /**
+   * Logs an [AndroidStudioEvent] with editor highlighting stats.
+   * Resets statistics so that counts are not double-counted in the next report.
+   */
+  fun reportHighlightingStats() {
+    val allStats = EditorHighlightingStats.newBuilder()
+    for ((fileType, recorder) in latencyRecorders) {
+      val histogram = recorder.intervalHistogram // Automatically resets statistics for this recorder.
+      if (histogram.totalCount == 0L) {
+        continue
+      }
+      val record = EditorHighlightingStats.Stats.newBuilder().also {
+        it.fileType = fileType
+        it.histogram = histogram.toProto()
+      }
+      allStats.addByFileType(record.build())
+    }
+
+    if (allStats.byFileTypeCount == 0) {
+      return
+    }
+
+    UsageTracker.log(
+      AndroidStudioEvent.newBuilder().apply {
+        kind = AndroidStudioEvent.EventKind.EDITOR_HIGHLIGHTING_STATS
+        editorHighlightingStats = allStats.build()
+      }
+    )
+  }
+
+  /** Converts from file type name to proto enum value. */
+  private fun convertFileType(file: VirtualFile): EditorFileType = when (file.fileType.name) {
+    // We use string literals here (rather than, e.g., JsonFileType.INSTANCE.name) to avoid unnecessary
+    // dependencies on other plugins. Fortunately, these values are extremely unlikely to change.
+    "JAVA" -> JAVA
+    "Kotlin" -> if (file.extension == "kts") KOTLIN_SCRIPT else KOTLIN
+    "XML" -> XML
+    "Groovy" -> GROOVY
+    "Properties" -> PROPERTIES
+    "JSON" -> JSON
+    "ObjectiveC" -> NATIVE
+    else -> UNKNOWN
+  }
+}

analytics/src/com/android/tools/analytics/HistogramUtil.kt

@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@file:JvmName("HistogramUtil")
+
+package com.android.tools.analytics
+
+import com.google.wireless.android.sdk.stats.HistogramBin
+import org.HdrHistogram.Histogram
+import org.HdrHistogram.HistogramIterationValue
+
+typealias HistogramProto = com.google.wireless.android.sdk.stats.Histogram
+
+/**
+ * Returns the inclusive start value for the given bin.
+ */
+val HistogramIterationValue.start: Long get() {
+  // Special case: HdrHistogram encodes the first bin as 0,0 even though it's supposed to be 0,1
+  if (valueIteratedFrom == 0L && valueIteratedTo == 0L) {
+    return 0L
+  }
+  return valueIteratedFrom + 1
+}
+
+/**
+ * Returns the exclusive end value for the given bin.
+ */
+val HistogramIterationValue.end: Long get() {
+  return valueIteratedTo + 1
+}
+
+/**
+ * Converts a [Histogram] to a proto.
+ */
+fun Histogram.toProto(): HistogramProto {
+  val builder = HistogramProto.newBuilder()
+
+  var total = totalCount
+  builder.totalCount = total
+  for (value in allValues()) {
+    if (value.countAddedInThisIterationStep > 0L) {
+      // HdrHistogram has a special case for it's first bin, which uses the range 0 to 0. Subsequent bins have an inclusive
+      // upper bound and exclusive lower bound. We use inclusive lower bounds and exclusive upper bounds in the proto, so
+      // need to shuffle around some indices here.
+      builder.addBin(HistogramBin.newBuilder()
+                       .setStart(value.start)
+                       .setEnd(value.end)
+                       .setTotalSamples(total)
+                       .setSamples(value.countAddedInThisIterationStep))
+    }
+    total -= value.countAddedInThisIterationStep
+  }
+
+  return builder.build()
+}

analytics/src/com/android/tools/analytics/StudioUpdateAnalyticsUtil.kt

@@ -0,0 +1,77 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@file:JvmName("StudioUpdateAnalyticsUtil")
+
+package com.android.tools.analytics
+
+import com.android.tools.analytics.UsageTracker
+import com.google.wireless.android.sdk.stats.AndroidStudioEvent
+import com.google.wireless.android.sdk.stats.StudioUpdateFlowEvent
+
+fun logClickUpdate(newBuild: String) {
+  log(StudioUpdateFlowEvent.newBuilder().setEventKind(StudioUpdateFlowEvent.Kind.DIALOG_CLICK_UPDATE), newBuild)
+}
+
+fun logClickIgnore(newBuild: String) {
+  log(StudioUpdateFlowEvent.newBuilder().setEventKind(StudioUpdateFlowEvent.Kind.DIALOG_CLICK_IGNORE), newBuild)
+}
+
+fun logClickLater(newBuild: String) {
+  log(StudioUpdateFlowEvent.newBuilder().setEventKind(StudioUpdateFlowEvent.Kind.DIALOG_CLICK_LATER), newBuild)
+}
+
+fun logClickAction(actionName: String, newBuild: String) {
+  log(StudioUpdateFlowEvent.newBuilder()
+        .setEventKind(StudioUpdateFlowEvent.Kind.DIALOG_CLICK_ACTION)
+        .setActionName(actionName), newBuild)
+}
+
+fun logDownloadSuccess(newBuild: String) {
+  log(StudioUpdateFlowEvent.newBuilder().setEventKind(StudioUpdateFlowEvent.Kind.PATCH_DOWNLOAD_SUCCESS), newBuild)
+}
+
+fun logDownloadFailure(newBuild: String) {
+  log(StudioUpdateFlowEvent.newBuilder().setEventKind(StudioUpdateFlowEvent.Kind.PATCH_DOWNLOAD_FAILURE), newBuild)
+}
+
+fun logUpdateDialogOpenManually(newBuild: String) {
+  log(StudioUpdateFlowEvent.newBuilder()
+        .setEventKind(StudioUpdateFlowEvent.Kind.DIALOG_OPEN)
+        .setDialogTrigger(StudioUpdateFlowEvent.DialogTrigger.MANUAL), newBuild)
+}
+
+fun logUpdateDialogOpenFromNotification(newBuild: String) {
+  log(StudioUpdateFlowEvent.newBuilder()
+        .setEventKind(StudioUpdateFlowEvent.Kind.DIALOG_OPEN)
+        .setDialogTrigger(StudioUpdateFlowEvent.DialogTrigger.NOTIFICATION), newBuild)
+}
+
+fun logClickNotification(newBuild: String) {
+  log(StudioUpdateFlowEvent.newBuilder().setEventKind(StudioUpdateFlowEvent.Kind.NOTIFICATION_UPDATE_LINK_CLICKED), newBuild)
+
+}
+
+fun logNotificationShown(newBuild: String) {
+  log(StudioUpdateFlowEvent.newBuilder().setEventKind(StudioUpdateFlowEvent.Kind.NOTIFICATION_SHOWN), newBuild)
+}
+
+fun log(event: StudioUpdateFlowEvent.Builder, newBuild: String) {
+  event.setStudioNewVersion(newBuild)
+  UsageTracker.log(AndroidStudioEvent.newBuilder()
+                     .setKind(AndroidStudioEvent.EventKind.STUDIO_UPDATE_FLOW)
+                     .setStudioUpdateFlowEvent(event.build())
+  )
+}

analytics/testSrc/com/android/analytics/HistogramUtilTest.kt

@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.analytics;
+
+import com.android.tools.analytics.toProto
+import org.HdrHistogram.Histogram
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertTrue
+import org.junit.Test
+
+class HistogramUtilTest {
+  @Test
+  fun testBinBoundaries() {
+    // We repeat the test 100 times to verify bins with different numbers of digits
+    for (i in 0..100) {
+      val hist = Histogram(1)
+      hist.recordValue(i.toLong())
+      val proto = hist.toProto()
+      assertEquals("There should be one sample in the histogram", 1, proto.totalCount)
+      val binList = proto.binList
+      assertEquals("Empty bins should not be converted to protos", 1, binList.size)
+      val bin = binList[0]
+      assertTrue("The bin start value should be inclusive", bin.start <= i)
+      assertTrue("The bin end value should be exclusive", i < bin.end)
+    }
+  }
+
+  @Test
+  fun testEmptyHistogram() {
+    val emptyProto = Histogram(1).toProto()
+    assertEquals("There should be no samples in an empty histogram", 0, emptyProto.totalCount)
+  }
+}

android-test-framework/testSrc/org/jetbrains/android/AndroidTestBase.java

@@ -101,6 +101,7 @@ private static void checkUndisposedAndroidRelatedObjects() {
     DisposerExplorer.visitTree(disposable -> {
       if (disposable.getClass().getName().equals("com.android.tools.idea.adb.AdbService") ||
           disposable.getClass().getName().equals("com.android.tools.idea.adb.AdbOptionsService") ||
+          disposable.getClass().getName().startsWith("com.android.tools.analytics.HighlightingStats") ||
           (disposable instanceof ProjectImpl && (((ProjectImpl)disposable).isDefault() || ((ProjectImpl)disposable).isLight())) ||
           disposable.toString().startsWith("services of " + ProjectImpl.class.getName()) ||
           (disposable instanceof Module && ((Module)disposable).getName().equals(LightProjectDescriptor.TEST_MODULE_NAME)) ||

android/intellij.android.core.iml

@@ -19,6 +19,9 @@
     <orderEntry type="module" module-name="intellij.platform.debugger.impl" />
     <orderEntry type="module" module-name="intellij.java.execution" />
     <orderEntry type="module" module-name="intellij.platform.smRunner" />
+    <orderEntry type="module" module-name="intellij.platform.core" />
+    <orderEntry type="module" module-name="analytics" />
+    <orderEntry type="module" module-name="analytics-publisher" />
     <orderEntry type="module" module-name="intellij.junit" />
     <orderEntry type="module" module-name="intellij.java.ui" />
     <orderEntry type="module" module-name="intellij.json" />