start collecting compiler metrics optionally

Author: syphar

.sqlx/query-43d0bb3b88356af3abdae506b7699ec762a6f1debbbda49a3479fddaa8917e17.json renamed to .sqlx/query-4242ea977833de52a1b11e1bf418750e53a36222d36ab26140a7692e200fd35b.json

@@ -19,9 +19,11 @@ services:
             - "/var/run/docker.sock:/var/run/docker.sock"
             - ".rustwide-docker:/opt/docsrs/rustwide"
             - "cratesio-index:/opt/docsrs/prefix/crates.io-index"
+            - "./ignored/cratesfyi-prefix/metrics:/opt/docsrs/prefix/metrics"
             - "./static:/opt/docsrs/static:ro"
         environment:
             DOCSRS_RUSTWIDE_WORKSPACE: /opt/docsrs/rustwide
+            DOCSRS_COMPILER_METRICS_PATH: /opt/docsrs/prefix/metrics
             DOCSRS_DATABASE_URL: postgresql://cratesfyi:password@db
             DOCSRS_STORAGE_BACKEND: s3
             S3_ENDPOINT: http://s3:9000

.sqlx/query-ebb47544b1090567139f3bdf2c22993c3a3aaef41c6520095a2c3bfaf6035da6.json renamed to .sqlx/query-9633480047fdf3519d16bc5b4035b2e2d0e4403aa54c5cefbd64188ba3b41132.json

@@ -469,6 +469,7 @@ impl BuildSubcommand {
                                 .as_ref()
                                 .map(|s| PackageKind::Registry(s.as_str()))
                                 .unwrap_or(PackageKind::CratesIo),
+                            true,
                         )
                         .context("Building documentation failed")?;
                 }

docker-compose.yml

@@ -29,6 +29,7 @@ pub(crate) struct QueuedCrate {
     pub(crate) version: String,
     pub(crate) priority: i32,
     pub(crate) registry: Option<String>,
+    pub(crate) attempt: i32,
 }
 
 #[derive(Debug)]
@@ -165,7 +166,7 @@ impl AsyncBuildQueue {
 
         Ok(sqlx::query_as!(
             QueuedCrate,
-            "SELECT id, name, version, priority, registry
+            "SELECT id, name, version, priority, registry, attempt
                  FROM queue
                  WHERE attempt < $1
                  ORDER BY priority ASC, attempt ASC, id ASC",
@@ -495,7 +496,7 @@ impl BuildQueue {
         let to_process = match self.runtime.block_on(
             sqlx::query_as!(
                 QueuedCrate,
-                "SELECT id, name, version, priority, registry
+                "SELECT id, name, version, priority, registry, attempt
                  FROM queue
                  WHERE
                     attempt < $1 AND
@@ -646,7 +647,7 @@ impl BuildQueue {
                 return Err(err);
             }
 
-            builder.build_package(&krate.name, &krate.version, kind)
+            builder.build_package(&krate.name, &krate.version, kind, krate.attempt == 0)
         })?;
 
         Ok(processed)

src/bin/cratesfyi.rs

@@ -78,6 +78,10 @@ pub struct Config {
     // for the remote archives?
     pub(crate) local_archive_cache_path: PathBuf,
 
+    // Where to collect metrics for the metrics initiative.
+    // When empty, we won't collect metrics.
+    pub(crate) compiler_metrics_collection_path: Option<PathBuf>,
+
     // Content Security Policy
     pub(crate) csp_report_only: bool,
 
@@ -226,6 +230,8 @@ impl Config {
                 prefix.join("archive_cache"),
             )?,
 
+            compiler_metrics_collection_path: maybe_env("DOCSRS_COMPILER_METRICS_PATH")?,
+
             temp_dir,
 
             rustwide_workspace: env("DOCSRS_RUSTWIDE_WORKSPACE", PathBuf::from(".workspace"))?,