diff --git a/common/src/address.rs b/common/src/address.rs index 9a9623fe27d..6ad3edd3b37 100644 --- a/common/src/address.rs +++ b/common/src/address.rs @@ -35,6 +35,7 @@ pub const REPO_DEPOT_PORT: u16 = 12348; pub const COCKROACH_PORT: u16 = 32221; pub const COCKROACH_ADMIN_PORT: u16 = 32222; +pub const COCKROACH_HTTP_PORT: u16 = 32223; pub const CRUCIBLE_PORT: u16 = 32345; pub const CLICKHOUSE_HTTP_PORT: u16 = 8123; pub const CLICKHOUSE_INTERSERVER_PORT: u16 = 9009; diff --git a/internal-dns/types/src/config.rs b/internal-dns/types/src/config.rs index 4552a2931ae..fbe1584aa00 100644 --- a/internal-dns/types/src/config.rs +++ b/internal-dns/types/src/config.rs @@ -544,6 +544,38 @@ impl DnsConfigBuilder { ) } + /// Higher-level shorthand for adding a Cockroach zone with several + /// services. + /// + /// # Errors + /// + /// This fails if the provided `addresses` aren't using the same IP address. + /// It also fails if the services have already been added to the + /// configuration. + pub fn host_zone_cockroach( + &mut self, + zone_id: OmicronZoneUuid, + listen_address: SocketAddrV6, + http_address: SocketAddrV6, + ) -> anyhow::Result<()> { + anyhow::ensure!( + listen_address.ip() == http_address.ip(), + "The IP address of the listen and HTTP address must match", + ); + let zone = self.host_zone(zone_id, *listen_address.ip())?; + self.service_backend_zone( + ServiceName::Cockroach, + &zone, + listen_address.port(), + )?; + self.service_backend_zone( + ServiceName::CockroachHttp, + &zone, + http_address.port(), + )?; + Ok(()) + } + /// Higher-level shorthand for adding an internal DNS zone, including /// records for both its HTTP and DNS interfaces. /// @@ -741,6 +773,10 @@ mod test { "_clickhouse-server._tcp", ); assert_eq!(ServiceName::Cockroach.dns_name(), "_cockroach._tcp",); + assert_eq!( + ServiceName::CockroachHttp.dns_name(), + "_cockroach-http._tcp", + ); assert_eq!(ServiceName::InternalDns.dns_name(), "_nameservice._tcp",); assert_eq!(ServiceName::Nexus.dns_name(), "_nexus._tcp",); assert_eq!(ServiceName::Oximeter.dns_name(), "_oximeter._tcp",); diff --git a/internal-dns/types/src/names.rs b/internal-dns/types/src/names.rs index ac314009d28..d37ad58fd87 100644 --- a/internal-dns/types/src/names.rs +++ b/internal-dns/types/src/names.rs @@ -54,7 +54,10 @@ pub enum ServiceName { ClickhouseKeeper, /// The HTTP interface to a replicated ClickHouse server. ClickhouseServer, + /// The listen address of CockroachDB, for accessing the postgres interface Cockroach, + /// The http address of CockroachDB, for accessing metrics + CockroachHttp, InternalDns, ExternalDns, Nexus, @@ -90,6 +93,7 @@ impl ServiceName { ServiceName::ClickhouseKeeper => "clickhouse-keeper", ServiceName::ClickhouseServer => "clickhouse-server", ServiceName::Cockroach => "cockroach", + ServiceName::CockroachHttp => "cockroach-http", ServiceName::ExternalDns => "external-dns", ServiceName::InternalDns => "nameservice", ServiceName::Nexus => "nexus", @@ -123,6 +127,7 @@ impl ServiceName { | ServiceName::ClickhouseKeeper | ServiceName::ClickhouseServer | ServiceName::Cockroach + | ServiceName::CockroachHttp | ServiceName::InternalDns | ServiceName::ExternalDns | ServiceName::Nexus diff --git a/nexus/db-model/src/deployment.rs b/nexus/db-model/src/deployment.rs index b89a2b09de6..f4bedf2db75 100644 --- a/nexus/db-model/src/deployment.rs +++ b/nexus/db-model/src/deployment.rs @@ -49,7 +49,7 @@ use omicron_uuid_kinds::{ GenericUuid, MupdateOverrideKind, OmicronZoneKind, OmicronZoneUuid, PhysicalDiskKind, SledKind, SledUuid, ZpoolKind, ZpoolUuid, }; -use std::net::{IpAddr, SocketAddrV6}; +use std::net::{IpAddr, SocketAddr, SocketAddrV6}; use uuid::Uuid; /// See [`nexus_types::deployment::Blueprint`]. @@ -578,11 +578,22 @@ impl BpOmicronZone { bp_omicron_zone.set_zpool_name(dataset); } BlueprintZoneType::CockroachDb( - blueprint_zone_type::CockroachDb { address, dataset }, + blueprint_zone_type::CockroachDb { + address, + http_address, + dataset, + }, ) => { // Set the common fields bp_omicron_zone.set_primary_service_ip_and_port(address); bp_omicron_zone.set_zpool_name(dataset); + + // Set the zone specific fields + bp_omicron_zone.second_service_ip = Some(IpNetwork::from( + std::net::IpAddr::V6(*http_address.ip()), + )); + bp_omicron_zone.second_service_port = + Some(SqlU16::from(http_address.port())); } BlueprintZoneType::Crucible(blueprint_zone_type::Crucible { address, @@ -734,8 +745,8 @@ impl BpOmicronZone { let external_ip_id = Self::external_ip_to_blueprint_zone_type(self.external_ip_id); - let dns_address = - omicron_zone_config::secondary_ip_and_port_to_dns_address( + let second_service_address = + omicron_zone_config::secondary_ip_and_port_to_address( self.second_service_ip, self.second_service_port, ); @@ -802,12 +813,23 @@ impl BpOmicronZone { }, ), - ZoneType::CockroachDb => BlueprintZoneType::CockroachDb( - blueprint_zone_type::CockroachDb { - address: primary_address, - dataset: dataset?, - }, - ), + ZoneType::CockroachDb => { + let second_service_address = + second_service_address.and_then(|addr| match addr { + SocketAddr::V6(addr) => Ok(addr), + _ => Err(anyhow!( + "Unexpected IPv4 address for CRDB HTTP service" + )), + }); + + BlueprintZoneType::CockroachDb( + blueprint_zone_type::CockroachDb { + address: primary_address, + http_address: second_service_address?, + dataset: dataset?, + }, + ) + } ZoneType::Crucible => { BlueprintZoneType::Crucible(blueprint_zone_type::Crucible { address: primary_address, @@ -825,7 +847,7 @@ impl BpOmicronZone { http_address: primary_address, dns_address: OmicronZoneExternalFloatingAddr { id: external_ip_id?, - addr: dns_address?, + addr: second_service_address?, }, nic: nic?, }, @@ -835,7 +857,7 @@ impl BpOmicronZone { dataset: dataset?, http_address: primary_address, dns_address: omicron_zone_config::to_internal_dns_address( - dns_address?, + second_service_address?, )?, gz_address: self .dns_gz_address diff --git a/nexus/db-model/src/inventory.rs b/nexus/db-model/src/inventory.rs index 6461af1e2cf..320cc429a80 100644 --- a/nexus/db-model/src/inventory.rs +++ b/nexus/db-model/src/inventory.rs @@ -1955,11 +1955,10 @@ impl InvOmicronSledConfigZone { nic_row.map(Into::into), )?; - let dns_address = - omicron_zone_config::secondary_ip_and_port_to_dns_address( - self.second_service_ip, - self.second_service_port, - ); + let dns_address = omicron_zone_config::secondary_ip_and_port_to_address( + self.second_service_ip, + self.second_service_port, + ); let ntp_dns_servers = omicron_zone_config::ntp_dns_servers_to_omicron_internal( diff --git a/nexus/db-model/src/omicron_zone_config.rs b/nexus/db-model/src/omicron_zone_config.rs index 74e2b9637b9..fa599fa540f 100644 --- a/nexus/db-model/src/omicron_zone_config.rs +++ b/nexus/db-model/src/omicron_zone_config.rs @@ -90,8 +90,8 @@ pub fn dataset_zpool_name_to_omicron_zone_dataset( .ok_or_else(|| anyhow!("expected dataset zpool name, found none")) } -/// Convert the secondary ip and port to a dns address -pub fn secondary_ip_and_port_to_dns_address( +/// Convert the secondary ip and port to an address +pub fn secondary_ip_and_port_to_address( second_service_ip: Option, second_service_port: Option, ) -> anyhow::Result { diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index bd270fb4c78..6f9ac3853d9 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -16,7 +16,7 @@ use std::{collections::BTreeMap, sync::LazyLock}; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: Version = Version::new(150, 0, 0); +pub const SCHEMA_VERSION: Version = Version::new(151, 0, 0); /// List of all past database schema versions, in *reverse* order /// @@ -28,6 +28,7 @@ static KNOWN_VERSIONS: LazyLock> = LazyLock::new(|| { // | leaving the first copy as an example for the next person. // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), + KnownVersion::new(151, "crdb-add-http-service"), KnownVersion::new(150, "add-last-reconciliation-orphaned-datasets"), KnownVersion::new(149, "bp-add-target-release-min-gen"), KnownVersion::new(148, "clean-misplaced-m2s"), diff --git a/nexus/db-queries/src/db/datastore/deployment.rs b/nexus/db-queries/src/db/datastore/deployment.rs index 93a71bec015..ef63bbf4e10 100644 --- a/nexus/db-queries/src/db/datastore/deployment.rs +++ b/nexus/db-queries/src/db/datastore/deployment.rs @@ -649,10 +649,7 @@ impl DataStore { format!("zone {zone_id}: parse from database") }) .map_err(|e| { - Error::internal_error(&format!( - "{:#}", - e.to_string() - )) + Error::internal_error(&format!("{:?}", e)) })?; sled_config.zones.insert(zone); } diff --git a/nexus/db-queries/src/db/datastore/inventory.rs b/nexus/db-queries/src/db/datastore/inventory.rs index 605cc68b5d7..94d99d269e0 100644 --- a/nexus/db-queries/src/db/datastore/inventory.rs +++ b/nexus/db-queries/src/db/datastore/inventory.rs @@ -2569,9 +2569,7 @@ impl DataStore { .with_context(|| { format!("zone {:?}: parse from database", zone_id) }) - .map_err(|e| { - Error::internal_error(&format!("{:#}", e.to_string())) - })?; + .map_err(|e| Error::internal_error(&format!("{:?}", e)))?; config_with_id.config.zones.insert(zone); } diff --git a/nexus/reconfigurator/execution/src/dns.rs b/nexus/reconfigurator/execution/src/dns.rs index 5a52212db08..809ea1ca6d9 100644 --- a/nexus/reconfigurator/execution/src/dns.rs +++ b/nexus/reconfigurator/execution/src/dns.rs @@ -362,6 +362,7 @@ mod test { use nexus_types::internal_api::params::DnsRecord; use nexus_types::internal_api::params::Srv; use nexus_types::silo::silo_dns_name; + use omicron_common::address::COCKROACH_HTTP_PORT; use omicron_common::address::IpRange; use omicron_common::address::Ipv6Subnet; use omicron_common::address::RACK_PREFIX; @@ -518,8 +519,14 @@ mod test { ) } OmicronZoneType::CockroachDb { address, dataset } => { + let http_address = + SocketAddrV6::new(*address.ip(), COCKROACH_HTTP_PORT, 0, 0); BlueprintZoneType::CockroachDb( - blueprint_zone_type::CockroachDb { address, dataset }, + blueprint_zone_type::CockroachDb { + address, + http_address, + dataset, + }, ) } OmicronZoneType::Crucible { address, dataset } => { @@ -977,6 +984,7 @@ mod test { ServiceName::Clickhouse, ServiceName::ClickhouseNative, ServiceName::Cockroach, + ServiceName::CockroachHttp, ServiceName::InternalDns, ServiceName::ExternalDns, ServiceName::Nexus, diff --git a/nexus/reconfigurator/execution/src/omicron_zones.rs b/nexus/reconfigurator/execution/src/omicron_zones.rs index a659b7a2226..b3849811e55 100644 --- a/nexus/reconfigurator/execution/src/omicron_zones.rs +++ b/nexus/reconfigurator/execution/src/omicron_zones.rs @@ -323,6 +323,7 @@ mod test { zone_type: BlueprintZoneType::CockroachDb( blueprint_zone_type::CockroachDb { address: "[::1]:0".parse().unwrap(), + http_address: "[::1]:0".parse().unwrap(), dataset: OmicronZoneDataset { pool_name: format!("oxp_{}", Uuid::new_v4()) .parse() diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs index 3a4bfde7f7c..083212872cb 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs @@ -1505,9 +1505,12 @@ impl<'a> BlueprintBuilder<'a> { self.sled_select_zpool(sled_id, ZoneKind::CockroachDb)?; let port = omicron_common::address::COCKROACH_PORT; let address = SocketAddrV6::new(underlay_ip, port, 0, 0); + let port = omicron_common::address::COCKROACH_HTTP_PORT; + let http_address = SocketAddrV6::new(underlay_ip, port, 0, 0); let zone_type = BlueprintZoneType::CockroachDb(blueprint_zone_type::CockroachDb { address, + http_address, dataset: OmicronZoneDataset { pool_name }, }); let filesystem_pool = pool_name; diff --git a/nexus/src/app/background/tasks/crdb_node_id_collector.rs b/nexus/src/app/background/tasks/crdb_node_id_collector.rs index 8a3f4ab6358..6440654d038 100644 --- a/nexus/src/app/background/tasks/crdb_node_id_collector.rs +++ b/nexus/src/app/background/tasks/crdb_node_id_collector.rs @@ -128,6 +128,9 @@ impl CockroachNodeIdCollector { // This trait exists so we can inject addresses in our unit tests below that // aren't required to have admin servers listening on the fixed // `COCKROACH_ADMIN_PORT`. +// +// TODO: Is this as necessary, now that the blueprint can identify information +// about the CockroachDB HTTP address? Seems like it might be removable now. trait CockroachAdminFromBlueprint { fn cockroach_admin_addrs<'a>( &'a self, @@ -277,6 +280,12 @@ mod tests { zone_type: BlueprintZoneType::CockroachDb( blueprint_zone_type::CockroachDb { address: addr, + http_address: SocketAddrV6::new( + *addr.ip(), + COCKROACH_ADMIN_PORT, + 0, + 0, + ), dataset: OmicronZoneDataset { pool_name: format!("oxp_{}", zpool_id) .parse() diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs index 39c83e25116..0cc63d4339b 100644 --- a/nexus/test-utils/src/lib.rs +++ b/nexus/test-utils/src/lib.rs @@ -515,11 +515,13 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { let zone_id = OmicronZoneUuid::new_v4(); let zpool_id = ZpoolUuid::new_v4(); eprintln!("DB address: {}", address); - self.rack_init_builder.add_service_to_dns( - zone_id, - address, - ServiceName::Cockroach, - ); + + let http_address = database.http_addr(); + self.rack_init_builder + .internal_dns_config + .host_zone_cockroach(zone_id, address, http_address) + .expect("Failed to set up CockroachDB DNS"); + let pool_name = illumos_utils::zpool::ZpoolName::new_external(zpool_id) .to_string() .parse() @@ -531,6 +533,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { zone_type: BlueprintZoneType::CockroachDb( blueprint_zone_type::CockroachDb { address, + http_address, dataset: OmicronZoneDataset { pool_name }, }, ), diff --git a/nexus/tests/integration_tests/cockroach.rs b/nexus/tests/integration_tests/cockroach.rs new file mode 100644 index 00000000000..3a09515e510 --- /dev/null +++ b/nexus/tests/integration_tests/cockroach.rs @@ -0,0 +1,63 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Integration tests for accessing CockroachDB + +use internal_dns_resolver::Resolver; +use internal_dns_types::names::ServiceName; +use nexus_test_utils_macros::nexus_test; + +// Creates an internal DNS resolver pointing to the given DNS server address +fn create_test_resolver(cptestctx: &ControlPlaneTestContext) -> Resolver { + let dns_addr = cptestctx.internal_dns.dns_server.local_address(); + Resolver::new_from_addrs(cptestctx.logctx.log.clone(), &[dns_addr]) + .expect("Should be able to create resolver") +} + +type ControlPlaneTestContext = + nexus_test_utils::ControlPlaneTestContext; + +// Validates that ControlPlaneTestContext setup creates both the listening and HTTP addresses of +// CockroachDB, and stores them in DNS. +#[nexus_test] +async fn test_cockroach_dns(cptestctx: &ControlPlaneTestContext) { + // Create DNS resolver pointing to our test DNS server + let resolver = create_test_resolver(cptestctx); + + // Test that both services are discoverable via DNS using strongly-typed service names + let cockroach_targets = resolver + .lookup_srv(ServiceName::Cockroach) + .await + .expect("Should find CockroachDB PostgreSQL service"); + assert_eq!( + cockroach_targets.len(), + 1, + "Should have exactly one Cockroach PostgreSQL record" + ); + + let cockroach_http_targets = resolver + .lookup_srv(ServiceName::CockroachHttp) + .await + .expect("Should find CockroachDB HTTP service"); + assert_eq!( + cockroach_http_targets.len(), + 1, + "Should have exactly one Cockroach HTTP record" + ); + + // Verify they point to the same target but different ports + let (pg_target, pg_port) = &cockroach_targets[0]; + let (http_target, http_port) = &cockroach_http_targets[0]; + + // Should be same target hostname, different ports + assert_eq!( + pg_target, http_target, + "Both services should point to same target host" + ); + assert_ne!(pg_port, http_port, "Services should use different ports"); + + // Verify both ports are non-zero (actual allocated ports) + assert_ne!(*pg_port, 0, "PostgreSQL service should have allocated port"); + assert_ne!(*http_port, 0, "HTTP service should have allocated port"); +} diff --git a/nexus/tests/integration_tests/mod.rs b/nexus/tests/integration_tests/mod.rs index 8aabc169f5c..5ca180ece0f 100644 --- a/nexus/tests/integration_tests/mod.rs +++ b/nexus/tests/integration_tests/mod.rs @@ -14,6 +14,7 @@ mod authn_http; mod authz; mod basic; mod certificates; +mod cockroach; mod commands; mod console_api; mod crucible_replacements; diff --git a/nexus/types/src/deployment/execution/dns.rs b/nexus/types/src/deployment/execution/dns.rs index a8537afb606..e2aa7b67e33 100644 --- a/nexus/types/src/deployment/execution/dns.rs +++ b/nexus/types/src/deployment/execution/dns.rs @@ -96,8 +96,19 @@ pub fn blueprint_internal_dns_config( continue 'all_zones; } BlueprintZoneType::CockroachDb( - blueprint_zone_type::CockroachDb { address, .. }, - ) => (ServiceName::Cockroach, address), + blueprint_zone_type::CockroachDb { + address, http_address, .. + }, + ) => { + let listen_addr = *address; + let http_addr = *http_address; + dns_builder.host_zone_cockroach( + zone.id, + listen_addr, + http_addr, + )?; + continue 'all_zones; + } BlueprintZoneType::Nexus(blueprint_zone_type::Nexus { internal_address, .. diff --git a/nexus/types/src/deployment/zone_type.rs b/nexus/types/src/deployment/zone_type.rs index f922136ff8d..039ae393863 100644 --- a/nexus/types/src/deployment/zone_type.rs +++ b/nexus/types/src/deployment/zone_type.rs @@ -435,6 +435,7 @@ pub mod blueprint_zone_type { )] pub struct CockroachDb { pub address: SocketAddrV6, + pub http_address: SocketAddrV6, pub dataset: OmicronZoneDataset, } diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index 83fc60421a5..961661b70e0 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -3087,6 +3087,9 @@ "dataset": { "$ref": "#/components/schemas/OmicronZoneDataset" }, + "http_address": { + "type": "string" + }, "type": { "type": "string", "enum": [ @@ -3097,6 +3100,7 @@ "required": [ "address", "dataset", + "http_address", "type" ] }, diff --git a/schema/crdb/crdb-add-http-service/up01.sql b/schema/crdb/crdb-add-http-service/up01.sql new file mode 100644 index 00000000000..5c8f43ef4a9 --- /dev/null +++ b/schema/crdb/crdb-add-http-service/up01.sql @@ -0,0 +1,9 @@ +set local disallow_full_table_scans = off; + +UPDATE bp_omicron_zone +SET second_service_ip = primary_service_ip, + second_service_port = 32223 +WHERE zone_type = 'cockroach_db'; + +set local disallow_full_table_scans = on; + diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 0195bfd06cc..36766f51067 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -5760,7 +5760,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - (TRUE, NOW(), NOW(), '150.0.0', NULL) + (TRUE, NOW(), NOW(), '151.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/sled-agent/src/rack_setup/plan/service.rs b/sled-agent/src/rack_setup/plan/service.rs index 7aec5989b19..9121dccc801 100644 --- a/sled-agent/src/rack_setup/plan/service.rs +++ b/sled-agent/src/rack_setup/plan/service.rs @@ -21,10 +21,10 @@ use nexus_types::deployment::{ blueprint_zone_type, }; use omicron_common::address::{ - DENDRITE_PORT, DNS_HTTP_PORT, DNS_PORT, Ipv6Subnet, MGD_PORT, MGS_PORT, - NEXUS_INTERNAL_PORT, NTP_PORT, NUM_SOURCE_NAT_PORTS, REPO_DEPOT_PORT, - RSS_RESERVED_ADDRESSES, ReservedRackSubnet, SLED_PREFIX, get_sled_address, - get_switch_zone_address, + COCKROACH_HTTP_PORT, COCKROACH_PORT, DENDRITE_PORT, DNS_HTTP_PORT, + DNS_PORT, Ipv6Subnet, MGD_PORT, MGS_PORT, NEXUS_INTERNAL_PORT, NTP_PORT, + NUM_SOURCE_NAT_PORTS, REPO_DEPOT_PORT, RSS_RESERVED_ADDRESSES, + ReservedRackSubnet, SLED_PREFIX, get_sled_address, get_switch_zone_address, }; use omicron_common::api::external::{MacAddr, Vni}; use omicron_common::api::internal::shared::{ @@ -454,10 +454,10 @@ impl Plan { }; let id = OmicronZoneUuid::new_v4(); let ip = sled.addr_alloc.next().expect("Not enough addrs"); - let port = omicron_common::address::COCKROACH_PORT; - let address = SocketAddrV6::new(ip, port, 0, 0); + let listen_address = SocketAddrV6::new(ip, COCKROACH_PORT, 0, 0); + let http_address = SocketAddrV6::new(ip, COCKROACH_HTTP_PORT, 0, 0); dns_builder - .host_zone_with_one_backend(id, ServiceName::Cockroach, address) + .host_zone_cockroach(id, listen_address, http_address) .unwrap(); let dataset_name = sled.alloc_dataset_from_u2s(DatasetKind::Cockroach)?; @@ -467,7 +467,8 @@ impl Plan { id, zone_type: BlueprintZoneType::CockroachDb( blueprint_zone_type::CockroachDb { - address, + address: listen_address, + http_address, dataset: OmicronZoneDataset { pool_name: *dataset_name.pool(), }, diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index a60da9210db..99aed1750a0 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -75,7 +75,9 @@ use omicron_common::address::SLED_PREFIX; use omicron_common::address::TFPORTD_PORT; use omicron_common::address::WICKETD_NEXUS_PROXY_PORT; use omicron_common::address::WICKETD_PORT; -use omicron_common::address::{BOOTSTRAP_ARTIFACT_PORT, COCKROACH_ADMIN_PORT}; +use omicron_common::address::{ + BOOTSTRAP_ARTIFACT_PORT, COCKROACH_ADMIN_PORT, COCKROACH_HTTP_PORT, +}; use omicron_common::address::{ CLICKHOUSE_ADMIN_PORT, CLICKHOUSE_TCP_PORT, get_internal_dns_server_addresses, @@ -1687,6 +1689,11 @@ impl ServiceManager { addr.set_port(COCKROACH_ADMIN_PORT); addr.to_string() }; + let http_address = { + let mut addr = *address; + addr.set_port(COCKROACH_HTTP_PORT); + addr.to_string() + }; let nw_setup_service = Self::zone_network_setup_install( Some(&info.underlay_address), @@ -1699,6 +1706,11 @@ impl ServiceManager { // Configure the CockroachDB service. let cockroachdb_config = PropertyGroupBuilder::new("config") .add_property("listen_addr", "astring", address.to_string()) + .add_property( + "http_addr", + "astring", + http_address.to_string(), + ) .add_property("store", "astring", "/data"); let cockroachdb_service = ServiceBuilder::new("oxide/cockroachdb").add_instance( diff --git a/smf/cockroachdb/manifest.xml b/smf/cockroachdb/manifest.xml index 6ed18b1bca6..5acd78b8b90 100644 --- a/smf/cockroachdb/manifest.xml +++ b/smf/cockroachdb/manifest.xml @@ -29,6 +29,7 @@ + diff --git a/smf/cockroachdb/method_script.sh b/smf/cockroachdb/method_script.sh index 1d33ef94a6b..50ecc6ec200 100755 --- a/smf/cockroachdb/method_script.sh +++ b/smf/cockroachdb/method_script.sh @@ -7,6 +7,7 @@ set -o pipefail . /lib/svc/share/smf_include.sh LISTEN_ADDR="$(svcprop -c -p config/listen_addr "${SMF_FMRI}")" +HTTP_ADDR="$(svcprop -c -p config/http_addr "${SMF_FMRI}")" DATASTORE="$(svcprop -c -p config/store "${SMF_FMRI}")" # We need to tell CockroachDB the DNS names or IP addresses of the other nodes @@ -25,7 +26,7 @@ fi args=( '--insecure' '--listen-addr' "$LISTEN_ADDR" - '--http-addr' '127.0.0.1:8080' + '--http-addr' "$HTTP_ADDR" '--store' "$DATASTORE" '--join' "$JOIN_ADDRS" ) diff --git a/test-utils/src/dev/db.rs b/test-utils/src/dev/db.rs index b9e8fd0e9ea..4017094b5d3 100644 --- a/test-utils/src/dev/db.rs +++ b/test-utils/src/dev/db.rs @@ -12,7 +12,7 @@ use nexus_config::PostgresConfigWithUrl; use std::collections::BTreeMap; use std::ffi::{OsStr, OsString}; use std::fmt; -use std::net::SocketAddr; +use std::net::SocketAddrV6; use std::ops::Deref; use std::os::unix::process::ExitStatusExt; use std::path::Path; @@ -119,7 +119,7 @@ impl CockroachStarterBuilder { builder .arg("start-single-node") .arg("--insecure") - .arg("--http-addr=:0"); + .arg("--http-addr=[::1]:0"); builder } @@ -563,7 +563,7 @@ pub struct CockroachInstance { /// child process id pid: u32, /// address of the HTTP API - http_addr: SocketAddr, + http_addr: SocketAddrV6, /// PostgreSQL config to use to connect to CockroachDB as a SQL client pg_config: PostgresConfigWithUrl, /// handle to child process, if it hasn't been cleaned up already @@ -581,7 +581,7 @@ impl CockroachInstance { } /// Returns the HTTP address where CockroachDB's web UI is accessible - pub fn http_addr(&self) -> SocketAddr { + pub fn http_addr(&self) -> SocketAddrV6 { self.http_addr } @@ -904,11 +904,11 @@ pub async fn wipe( /// Parses the HTTP address from CockroachDB's stdout file. /// -/// Looks for a line like "webui: http://127.0.0.1:39953" +/// Looks for a line like `webui: http://[::1]:39953` /// and extracts the socket address. async fn parse_http_addr_from_stdout( stdout_path: &std::path::Path, -) -> Result, anyhow::Error> { +) -> Result, anyhow::Error> { use std::str::FromStr; // Try to read the stdout file @@ -935,7 +935,7 @@ async fn parse_http_addr_from_stdout( let host_port = http_part.split('/').next().unwrap_or(http_part); // Parse as a socket address - match SocketAddr::from_str(host_port) { + match SocketAddrV6::from_str(host_port) { Ok(addr) => return Ok(Some(addr)), Err(parse_err) => { return Err(anyhow!( @@ -1611,20 +1611,10 @@ mod test { ); // The HTTP address should be on localhost - match http_addr.ip() { - std::net::IpAddr::V4(ipv4) => { - assert!( - ipv4.is_loopback(), - "HTTP address should be on loopback interface" - ); - } - std::net::IpAddr::V6(ipv6) => { - assert!( - ipv6.is_loopback(), - "HTTP address should be on loopback interface" - ); - } - } + assert!( + http_addr.ip().is_loopback(), + "HTTP address should be on loopback interface" + ); eprintln!("CockroachDB HTTP address: {}", http_addr); @@ -1848,16 +1838,9 @@ mod test { ) .await .expect("failed to write test stdout"); - let result = super::parse_http_addr_from_stdout(&stdout_path) + super::parse_http_addr_from_stdout(&stdout_path) .await - .expect("parsing should succeed"); - assert!(result.is_some(), "should return Some when webui line found"); - let addr = result.unwrap(); - assert_eq!( - addr.ip(), - std::net::IpAddr::V4(std::net::Ipv4Addr::new(127, 0, 0, 1)) - ); - assert_eq!(addr.port(), 39953); + .expect_err("parsing should not succeed for IPv4 address"); // Test case: IPv6 address fs::write(&stdout_path, "webui: http://[::1]:12345\n") @@ -1868,12 +1851,7 @@ mod test { .expect("parsing should succeed"); assert!(result.is_some(), "should return Some for IPv6 address"); let addr = result.unwrap(); - assert_eq!( - addr.ip(), - std::net::IpAddr::V6(std::net::Ipv6Addr::new( - 0, 0, 0, 0, 0, 0, 0, 1 - )) - ); + assert_eq!(addr.ip(), &std::net::Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1)); assert_eq!(addr.port(), 12345); }