New tech report DB and APIs

definitions/output/reports/tech_report_geos.js

@@ -0,0 +1,31 @@
+const pastMonth = constants.fnPastMonth(constants.currentMonth)
+
+publish('tech_report_geos', {
+  schema: 'reports',
+  type: 'table',
+  tags: ['tech_report']
+}).query(ctx => `
+SELECT
+  geo,
+  adoption AS origins
+FROM ${ctx.ref('reports', 'tech_report_adoption')}
+WHERE
+  date = '${pastMonth}'
+  AND rank = 'ALL'
+  AND technology = 'ALL'
+  AND version = 'ALL'
+  ${constants.devRankFilter}
+`).postOps(ctx => `
+  SELECT
+    reports.run_export_job(
+      JSON '''{
+        "destination": "firestore",
+        "config": {
+          "database": "tech-report-api-${constants.environment}",
+          "collection": "geos",
+          "type": "dict"
+        },
+        "query": "SELECT * FROM ${ctx.self()}"
+      }'''
+    );
+  `)

definitions/output/reports/tech_report_ranks.js

@@ -0,0 +1,31 @@
+const pastMonth = constants.fnPastMonth(constants.currentMonth)
+
+publish('tech_report_ranks', {
+  schema: 'reports',
+  type: 'table',
+  tags: ['tech_report']
+}).query(ctx => `
+SELECT
+  rank,
+  adoption AS origins
+FROM ${ctx.ref('reports', 'tech_report_adoption')}
+WHERE
+  date = '${pastMonth}'
+  AND geo = 'ALL'
+  AND technology = 'ALL'
+  AND version = 'ALL'
+  ${constants.devRankFilter}
+`).postOps(ctx => `
+  SELECT
+    reports.run_export_job(
+      JSON '''{
+        "destination": "firestore",
+        "config": {
+          "database": "tech-report-api-${constants.environment}",
+          "collection": "ranks",
+          "type": "dict"
+        },
+        "query": "SELECT * FROM ${ctx.self()}"
+      }'''
+    );
+  `)

infra/tech-report-api/src/controllers/pageWeightController.js

@@ -0,0 +1,82 @@
+const firestore = require('../utils/db');
+const { convertToArray, createSuccessResponse, createErrorResponse } = require('../utils/helpers');
+
+const TABLE = 'page_weight';
+
+/**
+ * Get the latest date in the collection
+ */
+const getLatestDate = async () => {
+  const query = firestore.collection(TABLE).orderBy('date', 'desc').limit(1);
+  const snapshot = await query.get();
+  if (!snapshot.empty) {
+    return snapshot.docs[0].data().date;
+  }
+  return null;
+};
+
+/**
+ * List Page Weight data with filtering
+ */
+const listPageWeightData = async (req, res) => {
+  try {
+    const params = req.query;
+    const data = [];
+
+    // Required parameters check
+    if (!params.technology) {
+      return res.status(400).send(createErrorResponse([
+        ['technology', 'missing technology parameter']
+      ]));
+    }
+
+    // Convert technology parameter to array
+    const techArray = convertToArray(params.technology);
+
+    // Handle 'latest' special value for start parameter
+    if (params.start && params.start === 'latest') {
+      params.start = await getLatestDate();
+    }
+
+    // Query for each technology
+    for (const technology of techArray) {
+      let query = firestore.collection(TABLE);
+
+      // Apply filters
+      if (params.start) {
+        query = query.where('date', '>=', params.start);
+      }
+
+      if (params.end) {
+        query = query.where('date', '<=', params.end);
+      }
+
+      if (params.geo) {
+        query = query.where('geo', '==', params.geo);
+      }
+
+      if (params.rank) {
+        query = query.where('rank', '==', params.rank);
+      }
+
+      // Always filter by technology
+      query = query.where('technology', '==', technology);
+
+      // Execute query
+      const snapshot = await query.get();
+      snapshot.forEach(doc => {
+        data.push(doc.data());
+      });
+    }
+
+    // Send response
+    res.status(200).send(createSuccessResponse(data));
+  } catch (error) {
+    console.error('Error fetching Page Weight data:', error);
+    res.status(400).send(createErrorResponse([['query', error.message]]));
+  }
+};
+
+module.exports = {
+  listPageWeightData
+};

infra/tf/api-gateway/main.tf

@@ -0,0 +1,50 @@
+######################################
+# API Gateway
+######################################
+# Used to expose Internal resources to external sources, such as web applications
+# See https://cloud.google.com/api-gateway/docs for more information
+# The API used by the Gateway
+resource "google_api_gateway_api" "api" {
+  provider     = google-beta # API Gateway is still in beta
+  api_id       = "api-gw-${var.environment}"
+  display_name = "The ${var.environment} API Gateway"
+  project      = var.project
+}
+# A Configuration, consisting of an OpenAPI specification
+resource "google_api_gateway_api_config" "api_config" {
+  provider             = google-beta # API Gateway is still in beta
+  api                  = google_api_gateway_api.api.api_id
+  api_config_id_prefix = "api"
+  project              = var.project
+  display_name         = "The ${var.environment} Config"
+  openapi_documents {
+    document {
+      path     = "spec.yaml"             # File name is simply sugar to show on GCP
+      contents = filebase64("spec.yaml") # This is based on *who* is call the module! 
+    }
+  }
+  gateway_config {
+    backend_config {
+      google_service_account = var.service_account_email
+    }
+  }
+}
+# The actual API Gateway
+resource "google_api_gateway_gateway" "gateway" {
+  provider     = google-beta
+  project      = var.project
+  region       = var.region
+  api_config   = google_api_gateway_api_config.api_config.id
+  gateway_id   = "${var.environment}-gw"
+  display_name = "${var.environment} Api Gateway"
+  labels = {
+    owner       = "tech_report_api"
+    environment = var.environment
+  }
+  depends_on = [google_api_gateway_api_config.api_config]
+  lifecycle {
+    replace_triggered_by = [
+      google_api_gateway_api_config.api_config
+    ]
+  }
+}

infra/tf/api-gateway/networking.tf

@@ -0,0 +1,86 @@
+resource "google_compute_global_address" "default" {
+  #count = var.environment == "prod" ? 1 : 0
+  count = 0
+
+  project      = var.project
+  name         = "httparchive-api-gateway-address"
+  address_type = "EXTERNAL"
+  ip_version   = "IPV4"
+}
+
+resource "google_compute_global_forwarding_rule" "https" {
+  #count = var.environment == "prod" ? 1 : 0
+  count = 0
+
+  provider              = google-beta
+  project               = var.project
+  name                  = "httparchive-api-gateway-https"
+  target                = google_compute_target_https_proxy.default[count.index].self_link
+  ip_address            = google_compute_global_address.default[count.index].id
+  port_range            = "443"
+  load_balancing_scheme = "EXTERNAL_MANAGED"
+}
+
+resource "google_compute_managed_ssl_certificate" "default" {
+  #count = var.environment == "prod" ? 1 : 0
+  count = 0
+
+  name = "httparchive-api-gateway-ssl"
+  managed {
+    domains = ["api.httparchive.org"]
+  }
+
+}
+
+resource "google_compute_target_https_proxy" "default" {
+  #count = var.environment == "prod" ? 1 : 0
+  count = 0
+
+  provider         = google-beta
+  project          = var.project
+  name             = "httparchive-api-gateway-https-proxy"
+  url_map          = google_compute_url_map.default[count.index].id
+  ssl_certificates = [google_compute_managed_ssl_certificate.default[count.index].id]  
+}
+
+resource "google_compute_region_network_endpoint_group" "function_neg" {
+  #count = var.environment == "prod" ? 1 : 0
+  count = 0
+
+  provider              = google-beta
+  name                  = "httparchive-api-gateway-function-neg"
+  network_endpoint_type = "SERVERLESS"
+  project               = var.project
+  region                = var.region
+
+  serverless_deployment {
+    platform = "apigateway.googleapis.com"
+    resource = google_api_gateway_gateway.gateway.gateway_id
+  }
+
+}
+
+resource "google_compute_backend_service" "backend_neg" {
+  #count = var.environment == "prod" ? 1 : 0
+  count = 0
+
+  provider              = google-beta
+  name                  = "httparchive-api-gateway-backend-neg"
+  project               = var.project
+  load_balancing_scheme = "EXTERNAL_MANAGED"
+  protocol              = "HTTP"
+  backend {
+      group = google_compute_region_network_endpoint_group.function_neg[count.index].self_link
+    }
+
+}
+
+resource "google_compute_url_map" "default" {
+  #count = var.environment == "prod" ? 1 : 0
+  count = 0
+
+  provider        = google-beta
+  project         = var.project
+  name            = "httparchive-api-gateway-url-map"
+  default_service = google_compute_backend_service.backend_neg[count.index].self_link
+}

infra/tf/api-gateway/variables.tf

@@ -0,0 +1,34 @@
+variable "environment" {
+  description = "The 'Environment' that is being created/deployed. Applied as a suffix to many resources."
+  type        = string
+}
+variable "project" {
+  description = "The ID of the project in which the resource belongs. If it is not provided, the provider project is used."
+  type        = string
+}
+variable "region" {
+  description = "The Region of this resource"
+  type        = string
+}
+variable "service_account_email" {
+  description = "Email of the service account associated with and to run the API Gateway"
+  type        = string
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+