diff --git a/config/manifests/bases/mongodb-kubernetes.clusterserviceversion.yaml b/config/manifests/bases/mongodb-kubernetes.clusterserviceversion.yaml index fdde19397..810b0a97d 100644 --- a/config/manifests/bases/mongodb-kubernetes.clusterserviceversion.yaml +++ b/config/manifests/bases/mongodb-kubernetes.clusterserviceversion.yaml @@ -8,9 +8,9 @@ metadata: certified: "true" containerImage: quay.io/mongodb/mongodb-kubernetes:1.1.0 createdAt: "" - description: The MongoDB Controllers for Kubernetes enable easy deploys of - MongoDB into Kubernetes clusters, using our management, monitoring and - backup platforms, Ops Manager and Cloud Manager. + description: The MongoDB Controllers for Kubernetes enable easy deploys of MongoDB + into Kubernetes clusters, using our management, monitoring and backup platforms, + Ops Manager and Cloud Manager. features.operators.openshift.io/disconnected: "true" features.operators.openshift.io/fips-compliant: "false" features.operators.openshift.io/proxy-aware: "false" @@ -51,8 +51,7 @@ spec: x-descriptors: - urn:alm:descriptor:com.tectonic.ui:text - urn:alm:descriptor:com.tectonic.ui:fieldGroup:ClusterConfiguration - - description: In a Replica Set deployment type, specifies the amount of - members. + - description: In a Replica Set deployment type, specifies the amount of members. displayName: Members of a Replica Set path: members x-descriptors: @@ -66,8 +65,7 @@ spec: - description: Project configuration for this deployment displayName: Ops Manager project configuration path: opsManager - - description: Name of the ConfigMap with the configuration for this - project + - description: Name of the ConfigMap with the configuration for this project displayName: Ops Manager Project Configuration path: opsManager.configMapRef.name x-descriptors: @@ -166,8 +164,7 @@ spec: x-descriptors: - urn:alm:descriptor:com.tectonic.ui:text - urn:alm:descriptor:com.tectonic.ui:fieldGroup:ClusterConfiguration - - description: In a Replica Set deployment type, specifies the amount of - members. + - description: In a Replica Set deployment type, specifies the amount of members. displayName: Members of a Replica Set path: members x-descriptors: @@ -181,8 +178,7 @@ spec: - description: Project configuration for this deployment displayName: Ops Manager project configuration path: opsManager - - description: Name of the ConfigMap with the configuration for this - project + - description: Name of the ConfigMap with the configuration for this project displayName: Ops Manager Project Configuration path: opsManager.configMapRef.name x-descriptors: @@ -194,8 +190,8 @@ spec: x-descriptors: - urn:alm:descriptor:com.tectonic.ui:booleanSwitch - urn:alm:descriptor:com.tectonic.ui:fieldGroup:ClusterConfiguration - - description: Optional. Specify whether to duplicate service objects - among different Kubernetes clusters. + - description: Optional. Specify whether to duplicate service objects among + different Kubernetes clusters. displayName: Duplicate Service Objects path: duplicateServiceObjects x-descriptors: @@ -256,8 +252,7 @@ spec: path: passwordSecretKeyRef.name x-descriptors: - urn:alm:descriptor:io.kubernetes:Secret - - displayName: Name of the MongoDB resource to which this user is - associated. + - displayName: Name of the MongoDB resource to which this user is associated. path: mongodbResourceRef.name x-descriptors: - urn:alm:descriptor:io.kubernetes:mongodb @@ -313,8 +308,8 @@ spec: x-descriptors: - urn:alm:descriptor:io.kubernetes:Secret - urn:alm:descriptor:com.tectonic.ui:fieldGroup:OpsManagerConfiguration - - displayName: Secret to enable TLS for Ops Manager allowing it to serve - traffic over HTTPS. + - displayName: Secret to enable TLS for Ops Manager allowing it to serve traffic + over HTTPS. path: security.tls.secretRef.name x-descriptors: - urn:alm:descriptor:io.kubernetes:Secret @@ -324,8 +319,8 @@ spec: x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number - urn:alm:descriptor:com.tectonic.ui:fieldGroup:ApplicationDatabase - - displayName: Secret containing the TLS certificate signed by known or - custom CA. + - displayName: Secret containing the TLS certificate signed by known or custom + CA. path: applicationDatabase.security.tls.secretRef.name x-descriptors: - urn:alm:descriptor:io.kubernetes:Secret diff --git a/docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_disaster_recovery.py b/docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_disaster_recovery.py deleted file mode 100644 index ece7ea32c..000000000 --- a/docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_disaster_recovery.py +++ /dev/null @@ -1,276 +0,0 @@ -from typing import Optional - -import kubernetes -import kubernetes.client -from kubetester import ( - delete_statefulset, - get_statefulset, - read_configmap, - try_load, - update_configmap, -) -from kubetester.kubetester import fixture as yaml_fixture -from kubetester.kubetester import run_periodically -from kubetester.opsmanager import MongoDBOpsManager -from kubetester.phase import Phase -from pytest import fixture, mark -from tests.common.cert.cert_issuer import create_appdb_certs -from tests.conftest import ( - MULTI_CLUSTER_MEMBER_LIST_CONFIGMAP, - get_member_cluster_api_client, -) -from tests.multicluster.conftest import cluster_spec_list - -FAILED_MEMBER_CLUSTER_NAME = "kind-e2e-cluster-3" -OM_MEMBER_CLUSTER_NAME = "kind-e2e-cluster-1" - - -@fixture(scope="module") -def ops_manager( - namespace: str, - custom_version: str, - custom_appdb_version: str, - multi_cluster_issuer_ca_configmap: str, - central_cluster_client: kubernetes.client.ApiClient, -) -> MongoDBOpsManager: - resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( - yaml_fixture("multicluster_appdb_om.yaml"), namespace=namespace - ) - - if try_load(resource): - return resource - - resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) - resource["spec"]["version"] = custom_version - - resource.allow_mdb_rc_versions() - resource.create_admin_secret(api_client=central_cluster_client) - - resource["spec"]["backup"] = {"enabled": False} - resource["spec"]["applicationDatabase"] = { - "topology": "MultiCluster", - "clusterSpecList": cluster_spec_list(["kind-e2e-cluster-2", FAILED_MEMBER_CLUSTER_NAME], [3, 2]), - "version": custom_appdb_version, - "agent": {"logLevel": "DEBUG"}, - "security": { - "certsSecretPrefix": "prefix", - "tls": {"ca": multi_cluster_issuer_ca_configmap}, - }, - } - - return resource - - -@fixture(scope="module") -def appdb_certs_secret( - namespace: str, - multi_cluster_issuer: str, - ops_manager: MongoDBOpsManager, -): - return create_appdb_certs( - namespace, - multi_cluster_issuer, - ops_manager.name + "-db", - cluster_index_with_members=[(0, 5), (1, 5), (2, 5)], - cert_prefix="prefix", - ) - - -@mark.e2e_multi_cluster_appdb_disaster_recovery -@mark.e2e_multi_cluster_appdb_disaster_recovery_force_reconfigure -def test_patch_central_namespace(namespace: str, central_cluster_client: kubernetes.client.ApiClient): - corev1 = kubernetes.client.CoreV1Api(api_client=central_cluster_client) - ns = corev1.read_namespace(namespace) - ns.metadata.labels["istio-injection"] = "enabled" - corev1.patch_namespace(namespace, ns) - - -@fixture(scope="module") -def config_version(): - class ConfigVersion: - version = 0 - - return ConfigVersion() - - -@mark.usefixtures("multi_cluster_operator") -@mark.e2e_multi_cluster_appdb_disaster_recovery -def test_create_om(ops_manager: MongoDBOpsManager, appdb_certs_secret: str, config_version): - ops_manager.update() - ops_manager.appdb_status().assert_reaches_phase(Phase.Running) - ops_manager.om_status().assert_reaches_phase(Phase.Running) - - config_version.version = ops_manager.get_automation_config_tester().automation_config["version"] - - -@mark.usefixtures("multi_cluster_operator") -@mark.e2e_multi_cluster_appdb_disaster_recovery_force_reconfigure -def test_create_om_majority_down(ops_manager: MongoDBOpsManager, appdb_certs_secret: str, config_version): - # failed cluster has majority members - ops_manager["spec"]["applicationDatabase"]["clusterSpecList"] = cluster_spec_list( - ["kind-e2e-cluster-2", FAILED_MEMBER_CLUSTER_NAME], [2, 3] - ) - - ops_manager.update() - ops_manager.appdb_status().assert_reaches_phase(Phase.Running) - ops_manager.om_status().assert_reaches_phase(Phase.Running) - - config_version.version = ops_manager.get_automation_config_tester().automation_config["version"] - - -@mark.e2e_multi_cluster_appdb_disaster_recovery -@mark.e2e_multi_cluster_appdb_disaster_recovery_force_reconfigure -def test_remove_cluster_from_operator_member_list_to_simulate_it_is_unhealthy( - namespace, central_cluster_client: kubernetes.client.ApiClient -): - member_list_cm = read_configmap( - namespace, - MULTI_CLUSTER_MEMBER_LIST_CONFIGMAP, - api_client=central_cluster_client, - ) - # this if is only for allowing re-running the test locally - # without it the test function could be executed only once until the map is populated again by running prepare-local-e2e run again - if FAILED_MEMBER_CLUSTER_NAME in member_list_cm: - member_list_cm.pop(FAILED_MEMBER_CLUSTER_NAME) - - # this will trigger operators restart as it panics on changing the configmap - update_configmap( - namespace, - MULTI_CLUSTER_MEMBER_LIST_CONFIGMAP, - member_list_cm, - api_client=central_cluster_client, - ) - - -@mark.e2e_multi_cluster_appdb_disaster_recovery -@mark.e2e_multi_cluster_appdb_disaster_recovery_force_reconfigure -def test_delete_om_and_appdb_statefulset_in_failed_cluster( - ops_manager: MongoDBOpsManager, central_cluster_client: kubernetes.client.ApiClient -): - appdb_sts_name = f"{ops_manager.name}-db-1" - try: - # delete OM to simulate losing Ops Manager application - # this is only for testing unavailability of the OM application, it's not testing losing OM cluster - # we don't delete here any additional resources (secrets, configmaps) that are required for a proper OM recovery testing - delete_statefulset( - ops_manager.namespace, - ops_manager.name, - propagation_policy="Background", - api_client=central_cluster_client, - ) - except kubernetes.client.ApiException as e: - if e.status != 404: - raise e - - try: - # delete appdb statefulset in failed member cluster to simulate full cluster outage - delete_statefulset( - ops_manager.namespace, - appdb_sts_name, - propagation_policy="Background", - api_client=get_member_cluster_api_client(FAILED_MEMBER_CLUSTER_NAME), - ) - except kubernetes.client.ApiException as e: - if e.status != 404: - raise e - - def statefulset_is_deleted(namespace: str, name: str, api_client=Optional[kubernetes.client.ApiClient]): - try: - get_statefulset(namespace, name, api_client=api_client) - return False - except kubernetes.client.ApiException as e: - if e.status == 404: - return True - else: - raise e - - run_periodically( - lambda: statefulset_is_deleted( - ops_manager.namespace, - ops_manager.name, - api_client=get_member_cluster_api_client(OM_MEMBER_CLUSTER_NAME), - ), - timeout=120, - ) - run_periodically( - lambda: statefulset_is_deleted( - ops_manager.namespace, - appdb_sts_name, - api_client=get_member_cluster_api_client(FAILED_MEMBER_CLUSTER_NAME), - ), - timeout=120, - ) - - -@mark.e2e_multi_cluster_appdb_disaster_recovery -@mark.e2e_multi_cluster_appdb_disaster_recovery_force_reconfigure -def test_appdb_is_stable_and_om_is_recreated(ops_manager: MongoDBOpsManager, config_version): - ops_manager.update() - ops_manager.appdb_status().assert_reaches_phase(Phase.Running) - ops_manager.om_status().assert_reaches_phase(Phase.Running) - - # there shouldn't be any automation config version change when one of the clusters is lost and OM is recreated - current_ac_version = ops_manager.get_automation_config_tester().automation_config["version"] - assert current_ac_version == config_version.version - - -@mark.e2e_multi_cluster_appdb_disaster_recovery -def test_add_appdb_member_to_om_cluster(ops_manager: MongoDBOpsManager, config_version): - ops_manager["spec"]["applicationDatabase"]["clusterSpecList"] = cluster_spec_list( - ["kind-e2e-cluster-2", FAILED_MEMBER_CLUSTER_NAME, OM_MEMBER_CLUSTER_NAME], - [3, 2, 1], - ) - ops_manager.update() - ops_manager.appdb_status().assert_reaches_phase(Phase.Running) - ops_manager.om_status().assert_reaches_phase(Phase.Running) - - # there should be exactly one automation config version change when we add new member - current_ac_version = ops_manager.get_automation_config_tester().automation_config["version"] - assert current_ac_version == config_version.version + 1 - - replica_set_members = ops_manager.get_automation_config_tester().get_replica_set_members(f"{ops_manager.name}-db") - assert len(replica_set_members) == 3 + 2 + 1 - - config_version.version = current_ac_version - - -@mark.e2e_multi_cluster_appdb_disaster_recovery_force_reconfigure -def test_add_appdb_member_to_om_cluster_force_reconfig(ops_manager: MongoDBOpsManager, config_version): - ops_manager["spec"]["applicationDatabase"]["clusterSpecList"] = cluster_spec_list( - ["kind-e2e-cluster-2", FAILED_MEMBER_CLUSTER_NAME, OM_MEMBER_CLUSTER_NAME], - [3, 2, 1], - ) - ops_manager.update() - ops_manager.appdb_status().assert_reaches_phase(Phase.Pending) - - ops_manager.reload() - ops_manager["metadata"]["annotations"].update({"mongodb.com/v1.forceReconfigure": "true"}) - ops_manager.update() - - # This can potentially take quite a bit of time. AppDB needs to go up and sync with OM (which will be crashlooping) - ops_manager.appdb_status().assert_reaches_phase(Phase.Running) - ops_manager.om_status().assert_reaches_phase(Phase.Running) - - replica_set_members = ops_manager.get_automation_config_tester().get_replica_set_members(f"{ops_manager.name}-db") - assert len(replica_set_members) == 3 + 2 + 1 - - config_version.version = ops_manager.get_automation_config_tester().automation_config["version"] - - -@mark.e2e_multi_cluster_appdb_disaster_recovery -@mark.e2e_multi_cluster_appdb_disaster_recovery_force_reconfigure -def test_remove_failed_member_cluster_has_been_scaled_down(ops_manager: MongoDBOpsManager, config_version): - # we remove failed member cluster - # thanks to previous spec stored in the config map, the operator recognizes we need to scale its 2 processes one by one - ops_manager["spec"]["applicationDatabase"]["clusterSpecList"] = cluster_spec_list( - ["kind-e2e-cluster-2", OM_MEMBER_CLUSTER_NAME], [3, 1] - ) - ops_manager.update() - ops_manager.appdb_status().assert_reaches_phase(Phase.Running) - ops_manager.om_status().assert_reaches_phase(Phase.Running) - - current_ac_version = ops_manager.get_automation_config_tester().automation_config["version"] - assert current_ac_version == config_version.version + 2 # two scale downs - - replica_set_members = ops_manager.get_automation_config_tester().get_replica_set_members(f"{ops_manager.name}-db") - assert len(replica_set_members) == 3 + 1 diff --git a/docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_disaster_recovery_force_reconfig.py b/docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_disaster_recovery_force_reconfig.py new file mode 100644 index 000000000..f89789fc0 --- /dev/null +++ b/docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_disaster_recovery_force_reconfig.py @@ -0,0 +1,55 @@ +import kubernetes +from pytest import fixture, mark + +from .multicluster_appdb_disaster_recovery_shared import MultiClusterAppDBDisasterRecovery + + +@mark.usefixtures("multi_cluster_operator") +@mark.e2e_multi_cluster_appdb_disaster_recovery_force_reconfigure +class TestMultiClusterAppDBDisasterRecoveryForceReconfig: + + @fixture(scope="module") + def ops_manager( + self, + namespace: str, + custom_version: str, + custom_appdb_version: str, + multi_cluster_issuer_ca_configmap: str, + central_cluster_client: kubernetes.client.ApiClient, + ): + return MultiClusterAppDBDisasterRecovery.create_ops_manager( + namespace, custom_version, custom_appdb_version, multi_cluster_issuer_ca_configmap, central_cluster_client, [2, 3] + ) + + @fixture(scope="module") + def appdb_certs_secret(self, namespace: str, multi_cluster_issuer: str, ops_manager): + return MultiClusterAppDBDisasterRecovery.create_appdb_certs_secret(namespace, multi_cluster_issuer, ops_manager) + + @fixture(scope="module") + def config_version(self): + return MultiClusterAppDBDisasterRecovery.create_config_version() + + def test_patch_central_namespace(self, namespace: str, central_cluster_client: kubernetes.client.ApiClient): + MultiClusterAppDBDisasterRecovery.test_patch_central_namespace(namespace, central_cluster_client) + + def test_create_om_majority_down(self, ops_manager, appdb_certs_secret, config_version): + MultiClusterAppDBDisasterRecovery.test_create_om_majority_down(ops_manager, appdb_certs_secret, config_version) + + def test_remove_cluster_from_operator_member_list_to_simulate_it_is_unhealthy( + self, namespace, central_cluster_client: kubernetes.client.ApiClient + ): + MultiClusterAppDBDisasterRecovery.test_remove_cluster_from_operator_member_list_to_simulate_it_is_unhealthy(namespace, central_cluster_client) + + def test_delete_om_and_appdb_statefulset_in_failed_cluster_majority_down( + self, ops_manager, central_cluster_client: kubernetes.client.ApiClient + ): + MultiClusterAppDBDisasterRecovery.test_delete_om_and_appdb_statefulset_in_failed_cluster_majority_down(ops_manager, central_cluster_client) + + def test_appdb_is_stable_and_om_is_recreated_majority_down(self, ops_manager, config_version): + MultiClusterAppDBDisasterRecovery.test_appdb_is_stable_and_om_is_recreated_majority_down(ops_manager, config_version) + + def test_add_appdb_member_to_om_cluster_force_reconfig(self, ops_manager, config_version): + MultiClusterAppDBDisasterRecovery.test_add_appdb_member_to_om_cluster_force_reconfig(ops_manager, config_version) + + def test_remove_failed_member_cluster_has_been_scaled_down_majority_down(self, ops_manager, config_version): + MultiClusterAppDBDisasterRecovery.test_remove_failed_member_cluster_has_been_scaled_down_majority_down(ops_manager, config_version) diff --git a/docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_disaster_recovery_normal.py b/docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_disaster_recovery_normal.py new file mode 100644 index 000000000..b442378b5 --- /dev/null +++ b/docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_disaster_recovery_normal.py @@ -0,0 +1,55 @@ +import kubernetes +from pytest import fixture, mark + +from .multicluster_appdb_disaster_recovery_shared import MultiClusterAppDBDisasterRecovery + + +@mark.usefixtures("multi_cluster_operator") +@mark.e2e_multi_cluster_appdb_disaster_recovery +class TestMultiClusterAppDBDisasterRecoveryNormal: + + @fixture(scope="module") + def ops_manager( + self, + namespace: str, + custom_version: str, + custom_appdb_version: str, + multi_cluster_issuer_ca_configmap: str, + central_cluster_client: kubernetes.client.ApiClient, + ): + return MultiClusterAppDBDisasterRecovery.create_ops_manager( + namespace, custom_version, custom_appdb_version, multi_cluster_issuer_ca_configmap, central_cluster_client, member_counts=[3, 2] + ) + + @fixture(scope="module") + def appdb_certs_secret(self, namespace: str, multi_cluster_issuer: str, ops_manager): + return MultiClusterAppDBDisasterRecovery.create_appdb_certs_secret(namespace, multi_cluster_issuer, ops_manager) + + @fixture(scope="module") + def config_version(self): + return MultiClusterAppDBDisasterRecovery.create_config_version() + + def test_patch_central_namespace(self, namespace: str, central_cluster_client: kubernetes.client.ApiClient): + MultiClusterAppDBDisasterRecovery.test_patch_central_namespace(namespace, central_cluster_client) + + def test_create_om(self, ops_manager, appdb_certs_secret, config_version): + MultiClusterAppDBDisasterRecovery.test_create_om(ops_manager, appdb_certs_secret, config_version) + + def test_remove_cluster_from_operator_member_list_to_simulate_it_is_unhealthy( + self, namespace, central_cluster_client: kubernetes.client.ApiClient + ): + MultiClusterAppDBDisasterRecovery.test_remove_cluster_from_operator_member_list_to_simulate_it_is_unhealthy(namespace, central_cluster_client) + + def test_delete_om_and_appdb_statefulset_in_failed_cluster( + self, ops_manager, central_cluster_client: kubernetes.client.ApiClient + ): + MultiClusterAppDBDisasterRecovery.test_delete_om_and_appdb_statefulset_in_failed_cluster(ops_manager, central_cluster_client) + + def test_appdb_is_stable_and_om_is_recreated(self, ops_manager, config_version): + MultiClusterAppDBDisasterRecovery.test_appdb_is_stable_and_om_is_recreated(ops_manager, config_version) + + def test_add_appdb_member_to_om_cluster(self, ops_manager, config_version): + MultiClusterAppDBDisasterRecovery.test_add_appdb_member_to_om_cluster(ops_manager, config_version) + + def test_remove_failed_member_cluster_has_been_scaled_down(self, ops_manager, config_version): + MultiClusterAppDBDisasterRecovery.test_remove_failed_member_cluster_has_been_scaled_down(ops_manager, config_version) diff --git a/docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_disaster_recovery_shared.py b/docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_disaster_recovery_shared.py new file mode 100644 index 000000000..419bcb90f --- /dev/null +++ b/docker/mongodb-kubernetes-tests/tests/multicluster_appdb/multicluster_appdb_disaster_recovery_shared.py @@ -0,0 +1,316 @@ +from typing import Optional + +import kubernetes +import kubernetes.client +from kubetester import ( + delete_statefulset, + get_statefulset, + read_configmap, + try_load, + update_configmap, +) +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.kubetester import run_periodically +from kubetester.mongodb import Phase +from kubetester.opsmanager import MongoDBOpsManager +from tests.conftest import create_appdb_certs, get_member_cluster_api_client +from tests.multicluster.conftest import cluster_spec_list + +FAILED_MEMBER_CLUSTER_NAME = "kind-e2e-cluster-3" +OM_MEMBER_CLUSTER_NAME = "kind-e2e-cluster-1" + + +class MultiClusterAppDBDisasterRecovery: + @staticmethod + def create_ops_manager( + namespace: str, + custom_version: str, + custom_appdb_version: str, + multi_cluster_issuer_ca_configmap: str, + central_cluster_client: kubernetes.client.ApiClient, + member_counts: list, + ) -> MongoDBOpsManager: + resource: MongoDBOpsManager = MongoDBOpsManager.from_yaml( + yaml_fixture("multicluster_appdb_om.yaml"), namespace=namespace + ) + + if try_load(resource): + return resource + + resource.api = kubernetes.client.CustomObjectsApi(central_cluster_client) + resource["spec"]["version"] = custom_version + + resource.allow_mdb_rc_versions() + resource.create_admin_secret(api_client=central_cluster_client) + + resource["spec"]["backup"] = {"enabled": False} + resource["spec"]["applicationDatabase"] = { + "topology": "MultiCluster", + "clusterSpecList": cluster_spec_list(["kind-e2e-cluster-2", FAILED_MEMBER_CLUSTER_NAME], member_counts), + "version": custom_appdb_version, + "agent": {"logLevel": "DEBUG"}, + "security": { + "certsSecretPrefix": "prefix", + "tls": {"ca": multi_cluster_issuer_ca_configmap}, + }, + } + + return resource + + @staticmethod + def create_appdb_certs_secret( + namespace: str, + multi_cluster_issuer: str, + ops_manager: MongoDBOpsManager, + ): + return create_appdb_certs( + namespace, + multi_cluster_issuer, + ops_manager.name + "-db", + cluster_index_with_members=[(0, 5), (1, 5), (2, 5)], + cert_prefix="prefix", + ) + + @staticmethod + def create_config_version(): + class ConfigVersion: + version = 0 + + return ConfigVersion() + + @staticmethod + def test_patch_central_namespace(namespace: str, central_cluster_client: kubernetes.client.ApiClient): + corev1 = kubernetes.client.CoreV1Api(api_client=central_cluster_client) + ns = corev1.read_namespace(namespace) + ns.metadata.labels["istio-injection"] = "enabled" + corev1.patch_namespace(namespace, ns) + + @staticmethod + def test_create_om(ops_manager: MongoDBOpsManager, appdb_certs_secret: str, config_version): + ops_manager.update() + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.om_status().assert_reaches_phase(Phase.Running) + + config_version.version = ops_manager.get_automation_config_tester().automation_config["version"] + + @staticmethod + def test_create_om_majority_down(ops_manager: MongoDBOpsManager, appdb_certs_secret: str, config_version): + ops_manager.update() + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.om_status().assert_reaches_phase(Phase.Running) + + config_version.version = ops_manager.get_automation_config_tester().automation_config["version"] + + @staticmethod + def test_remove_cluster_from_operator_member_list_to_simulate_it_is_unhealthy( + namespace, central_cluster_client: kubernetes.client.ApiClient + ): + member_list_cm = read_configmap( + namespace, + "mongodb-enterprise-operator-member-list", + api_client=central_cluster_client, + ) + if FAILED_MEMBER_CLUSTER_NAME in member_list_cm: + member_list_cm.pop(FAILED_MEMBER_CLUSTER_NAME) + + update_configmap( + namespace, + "mongodb-enterprise-operator-member-list", + member_list_cm, + api_client=central_cluster_client, + ) + + @staticmethod + def test_delete_om_and_appdb_statefulset_in_failed_cluster( + ops_manager: MongoDBOpsManager, central_cluster_client: kubernetes.client.ApiClient + ): + appdb_sts_name = f"{ops_manager.name}-db-1" + try: + delete_statefulset( + ops_manager.namespace, + ops_manager.name, + propagation_policy="Background", + api_client=central_cluster_client, + ) + except kubernetes.client.ApiException as e: + if e.status != 404: + raise e + + try: + delete_statefulset( + ops_manager.namespace, + appdb_sts_name, + propagation_policy="Background", + api_client=get_member_cluster_api_client(FAILED_MEMBER_CLUSTER_NAME), + ) + except kubernetes.client.ApiException as e: + if e.status != 404: + raise e + + def statefulset_is_deleted(namespace: str, name: str, api_client=Optional[kubernetes.client.ApiClient]): + try: + get_statefulset(namespace, name, api_client=api_client) + return False + except kubernetes.client.ApiException as e: + if e.status == 404: + return True + else: + raise e + + run_periodically( + lambda: statefulset_is_deleted( + ops_manager.namespace, + ops_manager.name, + api_client=get_member_cluster_api_client(OM_MEMBER_CLUSTER_NAME), + ), + timeout=120, + ) + run_periodically( + lambda: statefulset_is_deleted( + ops_manager.namespace, + appdb_sts_name, + api_client=get_member_cluster_api_client(FAILED_MEMBER_CLUSTER_NAME), + ), + timeout=120, + ) + + @staticmethod + def test_delete_om_and_appdb_statefulset_in_failed_cluster_majority_down( + ops_manager: MongoDBOpsManager, central_cluster_client: kubernetes.client.ApiClient + ): + appdb_sts_name = f"{ops_manager.name}-db-1" + try: + delete_statefulset( + ops_manager.namespace, + ops_manager.name, + propagation_policy="Background", + api_client=central_cluster_client, + ) + except kubernetes.client.ApiException as e: + if e.status != 404: + raise e + + try: + delete_statefulset( + ops_manager.namespace, + appdb_sts_name, + propagation_policy="Background", + api_client=get_member_cluster_api_client(FAILED_MEMBER_CLUSTER_NAME), + ) + except kubernetes.client.ApiException as e: + if e.status != 404: + raise e + + def statefulset_is_deleted(namespace: str, name: str, api_client=Optional[kubernetes.client.ApiClient]): + try: + get_statefulset(namespace, name, api_client=api_client) + return False + except kubernetes.client.ApiException as e: + if e.status == 404: + return True + else: + raise e + + run_periodically( + lambda: statefulset_is_deleted( + ops_manager.namespace, + ops_manager.name, + api_client=get_member_cluster_api_client(OM_MEMBER_CLUSTER_NAME), + ), + timeout=120, + ) + run_periodically( + lambda: statefulset_is_deleted( + ops_manager.namespace, + appdb_sts_name, + api_client=get_member_cluster_api_client(FAILED_MEMBER_CLUSTER_NAME), + ), + timeout=120, + ) + + @staticmethod + def test_appdb_is_stable_and_om_is_recreated(ops_manager: MongoDBOpsManager, config_version): + ops_manager.update() + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.om_status().assert_reaches_phase(Phase.Running) + + current_ac_version = ops_manager.get_automation_config_tester().automation_config["version"] + assert current_ac_version == config_version.version + + @staticmethod + def test_appdb_is_stable_and_om_is_recreated_majority_down(ops_manager: MongoDBOpsManager, config_version): + ops_manager.update() + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.om_status().assert_reaches_phase(Phase.Running) + + current_ac_version = ops_manager.get_automation_config_tester().automation_config["version"] + assert current_ac_version == config_version.version + + @staticmethod + def test_add_appdb_member_to_om_cluster(ops_manager: MongoDBOpsManager, config_version): + ops_manager["spec"]["applicationDatabase"]["clusterSpecList"] = cluster_spec_list( + ["kind-e2e-cluster-2", FAILED_MEMBER_CLUSTER_NAME, OM_MEMBER_CLUSTER_NAME], + [3, 2, 1], + ) + ops_manager.update() + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.om_status().assert_reaches_phase(Phase.Running) + + current_ac_version = ops_manager.get_automation_config_tester().automation_config["version"] + assert current_ac_version == config_version.version + 1 + + replica_set_members = ops_manager.get_automation_config_tester().get_replica_set_members(f"{ops_manager.name}-db") + assert len(replica_set_members) == 3 + 2 + 1 + + config_version.version = current_ac_version + + @staticmethod + def test_add_appdb_member_to_om_cluster_force_reconfig(ops_manager: MongoDBOpsManager, config_version): + ops_manager["spec"]["applicationDatabase"]["clusterSpecList"] = cluster_spec_list( + ["kind-e2e-cluster-2", FAILED_MEMBER_CLUSTER_NAME, OM_MEMBER_CLUSTER_NAME], + [3, 2, 1], + ) + ops_manager.update() + ops_manager.appdb_status().assert_reaches_phase(Phase.Pending) + + ops_manager.reload() + ops_manager["metadata"]["annotations"].update({"mongodb.com/v1.forceReconfigure": "true"}) + ops_manager.update() + + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.om_status().assert_reaches_phase(Phase.Running) + + replica_set_members = ops_manager.get_automation_config_tester().get_replica_set_members(f"{ops_manager.name}-db") + assert len(replica_set_members) == 3 + 2 + 1 + + config_version.version = ops_manager.get_automation_config_tester().automation_config["version"] + + @staticmethod + def test_remove_failed_member_cluster_has_been_scaled_down(ops_manager: MongoDBOpsManager, config_version): + ops_manager["spec"]["applicationDatabase"]["clusterSpecList"] = cluster_spec_list( + ["kind-e2e-cluster-2", OM_MEMBER_CLUSTER_NAME], [3, 1] + ) + ops_manager.update() + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.om_status().assert_reaches_phase(Phase.Running) + + current_ac_version = ops_manager.get_automation_config_tester().automation_config["version"] + assert current_ac_version == config_version.version + 2 + + replica_set_members = ops_manager.get_automation_config_tester().get_replica_set_members(f"{ops_manager.name}-db") + assert len(replica_set_members) == 3 + 1 + + @staticmethod + def test_remove_failed_member_cluster_has_been_scaled_down_majority_down(ops_manager: MongoDBOpsManager, config_version): + ops_manager["spec"]["applicationDatabase"]["clusterSpecList"] = cluster_spec_list( + ["kind-e2e-cluster-2", OM_MEMBER_CLUSTER_NAME], [3, 1] + ) + ops_manager.update() + ops_manager.appdb_status().assert_reaches_phase(Phase.Running) + ops_manager.om_status().assert_reaches_phase(Phase.Running) + + current_ac_version = ops_manager.get_automation_config_tester().automation_config["version"] + assert current_ac_version == config_version.version + 2 + + replica_set_members = ops_manager.get_automation_config_tester().get_replica_set_members(f"{ops_manager.name}-db") + assert len(replica_set_members) == 3 + 1 \ No newline at end of file diff --git a/helm_chart/Chart.yaml b/helm_chart/Chart.yaml index aadb482da..140da18b8 100644 --- a/helm_chart/Chart.yaml +++ b/helm_chart/Chart.yaml @@ -1,8 +1,7 @@ apiVersion: v2 name: mongodb-kubernetes -description: MongoDB Controllers for Kubernetes translate the human knowledge of - creating a MongoDB instance into a scalable, repeatable, and standardized - method. +description: MongoDB Controllers for Kubernetes translate the human knowledge of creating + a MongoDB instance into a scalable, repeatable, and standardized method. version: 1.1.0 kubeVersion: '>=1.16-0' type: application diff --git a/scripts/evergreen/deployments/test-app/templates/mongodb-enterprise-tests.yaml b/scripts/evergreen/deployments/test-app/templates/mongodb-enterprise-tests.yaml index e108716b3..f996f911f 100644 --- a/scripts/evergreen/deployments/test-app/templates/mongodb-enterprise-tests.yaml +++ b/scripts/evergreen/deployments/test-app/templates/mongodb-enterprise-tests.yaml @@ -192,11 +192,19 @@ spec: image: {{ .Values.repo }}/mongodb-kubernetes-tests:{{ .Values.tag }} # Options to pytest command should go in the pytest.ini file. command: ["pytest"] - {{ if .Values.otel_endpoint }} - args: ["-vv", "-m", "{{ .Values.taskName }}", "--trace-parent", "00-{{ .Values.otel_trace_id }}-{{ .Values.otel_parent_id }}-01", "--export-traces"] - {{ else }} - args: ["-vv", "-m", "{{ .Values.taskName }}"] - {{ end }} + args: + - "-vv" + {{- if .Values.testFile }} + - "{{ .Values.testFile }}" + {{- else }} + - "-m" + - "{{ .Values.taskName }}" + {{- end }} + {{- if .Values.otel_endpoint }} + - "--trace-parent" + - "00-{{ .Values.otel_trace_id }}-{{ .Values.otel_parent_id }}-01" + - "--export-traces" + {{- end }} imagePullPolicy: Always volumeMounts: - name: results diff --git a/scripts/evergreen/deployments/test-app/values.yaml b/scripts/evergreen/deployments/test-app/values.yaml index fe171c27a..c36ebeeae 100644 --- a/scripts/evergreen/deployments/test-app/values.yaml +++ b/scripts/evergreen/deployments/test-app/values.yaml @@ -8,6 +8,10 @@ apiKey: omApiKey orgId: "" projectId: omProjectId tag: + +# if testFile is specified, then test is executed as pytest +testFile: +# if testFile is empty, executes the test by fixture mark: pytest -m taskName: ${TASK_NAME} pytest: diff --git a/scripts/evergreen/e2e/single_e2e.sh b/scripts/evergreen/e2e/single_e2e.sh index dbb1306da..938288fa1 100755 --- a/scripts/evergreen/e2e/single_e2e.sh +++ b/scripts/evergreen/e2e/single_e2e.sh @@ -2,6 +2,7 @@ set -Eeou pipefail +test "${MDB_BASH_DEBUG:-0}" -eq 1 && set -x ## ## The script deploys a single test application and waits until it finishes. ## All the Operator deployment, configuration and teardown work is done in 'e2e' script @@ -15,6 +16,23 @@ source scripts/funcs/operator_deployment check_env_var "TEST_NAME" "The 'TEST_NAME' must be specified to run the Operator single e2e test" +find_test_file_by_fixture_mark() { + fixture_mark="$1" + + cd docker/mongodb-kubernetes-tests + if ! test_files="$(grep -l -R "mark.${fixture_mark}$" --include '*.py')"; then + >&2 echo "Cannot find any test file containing a pytest fixture mark: ${fixture_mark}" + return 1 + fi + number_of_files_matched=$(wc -l <<< "${test_files}") + if [[ ${number_of_files_matched} -gt 1 ]]; then + >&2 echo "Found more than one file with the same pytest fixture mark ${fixture_mark}:" + grep --color=auto --line-number --recursive -C2 "mark.${fixture_mark}$" --include '*.py' . + return 1 + fi + + echo -n "${test_files}" +} deploy_test_app() { printenv @@ -33,12 +51,17 @@ deploy_test_app() { BUILD_ID="${BUILD_ID:-default_build_id}" BUILD_VARIANT="${BUILD_VARIANT:-default_build_variant}" + if ! test_file=$(find_test_file_by_fixture_mark "${TASK_NAME}"); then + return 1 + fi + # note, that the 4 last parameters are used only for Mongodb resource testing - not for Ops Manager helm_params=( "--set" "taskId=${task_id:-'not-specified'}" "--set" "repo=${BASE_REPO_URL:=268558157000.dkr.ecr.us-east-1.amazonaws.com/dev}" "--set" "namespace=${NAMESPACE}" "--set" "taskName=${task_name}" + "--set" "testFile=${test_file}" "--set" "tag=${tag}" "--set" "aws.accessKey=${AWS_ACCESS_KEY_ID}" "--set" "aws.secretAccessKey=${AWS_SECRET_ACCESS_KEY}" @@ -128,7 +151,9 @@ deploy_test_app() { helm_params+=("--set" "opsManagerVersion=${ops_manager_version}") - helm template "scripts/evergreen/deployments/test-app" "${helm_params[@]}" > "${helm_template_file}" || exit 1 + echo "Executing helm template:" + echo "helm template scripts/evergreen/deployments/test-app ${helm_params[*]}" + helm template "scripts/evergreen/deployments/test-app" "${helm_params[@]}" > "${helm_template_file}" || return 1 cat "${helm_template_file}" @@ -189,7 +214,9 @@ run_tests() { prepare_operator_config_map "${operator_context}" - deploy_test_app "${test_pod_context}" + if ! deploy_test_app "${test_pod_context}"; then + return 1 + fi wait_until_pod_is_running_or_failed_or_succeeded "${test_pod_context}"