Skip to content

Commit

Permalink
Make ScyllaDB and Scylla Manager flags configurable for e2e tests
Browse files Browse the repository at this point in the history
  • Loading branch information
grzywin committed Feb 6, 2025
1 parent 7d72311 commit b2d7fb1
Show file tree
Hide file tree
Showing 7 changed files with 182 additions and 42 deletions.
12 changes: 12 additions & 0 deletions hack/.ci/lib/e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,13 @@ function run-e2e {
SO_BUCKET_NAME="${SO_BUCKET_NAME:-}"
SO_E2E_PARALLELISM="${SO_E2E_PARALLELISM:-0}"

config_file="$(realpath "$(dirname "${BASH_SOURCE[0]}")/../../../assets/config/config.yaml")"
SCYLLADB_VERSION="${SCYLLADB_VERSION:-$(yq '.operator.scyllaDBVersion' "$config_file")}"
SCYLLADB_MANAGER_VERSION="${SCYLLADB_MANAGER_VERSION:-$(yq '.operator.scyllaDBManagerVersion' "$config_file")}"
SCYLLADB_MANAGER_AGENT_VERSION="${SCYLLADB_MANAGER_AGENT_VERSION:-$(yq '.operator.scyllaDBManagerAgentVersion' "$config_file")}"
SCYLLADB_UPDATE_FROM_VERSION="${SCYLLADB_UPDATE_FROM_VERSION:-$(yq '.operatorTests.scyllaDBVersions.updateFrom' "$config_file")}"
SCYLLADB_UPGRADE_FROM_VERSION="${SCYLLADB_UPGRADE_FROM_VERSION:-$(yq '.operatorTests.scyllaDBVersions.upgradeFrom' "$config_file")}"

kubectl create namespace e2e --dry-run=client -o=yaml | kubectl_create -f=-
kubectl create clusterrolebinding e2e --clusterrole=cluster-admin --serviceaccount=e2e:default --dry-run=client -o=yaml | kubectl_create -f=-
kubectl create -n=e2e pdb my-pdb --selector='app=e2e' --min-available=1 --dry-run=client -o=yaml | kubectl_create -f=-
Expand Down Expand Up @@ -274,6 +281,11 @@ spec:
- "--object-storage-bucket=${SO_BUCKET_NAME}"
- "--gcs-service-account-key-path=${gcs_sa_in_container_path}"
- "--s3-credentials-file-path=${s3_credentials_in_container_path}"
- "--scylladb-version=${SCYLLADB_VERSION}"
- "--scylladb-manager-version=${SCYLLADB_MANAGER_VERSION}"
- "--scylladb-manager-agent-version=${SCYLLADB_MANAGER_AGENT_VERSION}"
- "--scylladb-update-from-version=${SCYLLADB_UPDATE_FROM_VERSION}"
- "--scylladb-upgrade-from-version=${SCYLLADB_UPGRADE_FROM_VERSION}"
image: "${SO_IMAGE}"
imagePullPolicy: Always
volumeMounts:
Expand Down
45 changes: 45 additions & 0 deletions hack/ci-deploy-release.sh
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,51 @@ target:
EOF
fi

if [[ -n "${SCYLLADB_VERSION:-}" ]]; then
cat << EOF | \
yq eval-all --inplace 'select(fileIndex == 0) as $f | select(fileIndex == 1) as $p | with( $f.patches; . += $p | ... style="") | $f' "${ARTIFACTS_DEPLOY_DIR}/manager/kustomization.yaml" -
patch: |-
- op: replace
path: /spec/version
value: "${SCYLLADB_VERSION}"
target:
group: scylla.scylladb.com
version: v1
kind: ScyllaCluster
name: scylla-manager-cluster
EOF
fi

if [[ -n "${SCYLLA_MANAGER_VERSION:-}" ]]; then
cat << EOF | \
yq eval-all --inplace 'select(fileIndex == 0) as $f | select(fileIndex == 1) as $p | with( $f.patches; . += $p | ... style="") | $f' "${ARTIFACTS_DEPLOY_DIR}/manager/kustomization.yaml" -
patch: |-
- op: replace
path: /spec/template/spec/containers/0/image
value: "docker.io/scylladb/scylla-manager:${SCYLLA_MANAGER_VERSION}"
target:
group: apps
version: v1
kind: Deployment
name: scylla-manager
EOF
fi

if [[ -n "${SCYLLA_MANAGER_AGENT_VERSION:-}" ]]; then
cat << EOF | \
yq eval-all --inplace 'select(fileIndex == 0) as $f | select(fileIndex == 1) as $p | with( $f.patches; . += $p | ... style="") | $f' "${ARTIFACTS_DEPLOY_DIR}/manager/kustomization.yaml" -
patch: |-
- op: replace
path: /spec/agentVersion
value: "${SCYLLA_MANAGER_AGENT_VERSION}"
target:
group: scylla.scylladb.com
version: v1
kind: ScyllaCluster
name: scylla-manager-cluster
EOF
fi

kubectl kustomize "${ARTIFACTS_DEPLOY_DIR}/manager" | kubectl_create -n=scylla-manager -f=-

kubectl -n=scylla-manager wait --timeout=5m --for='condition=Progressing=False' scyllaclusters.scylla.scylladb.com/scylla-manager-cluster
Expand Down
13 changes: 13 additions & 0 deletions hack/ci-deploy.sh
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,19 @@ if [[ -n "${SO_SCYLLACLUSTER_STORAGECLASS_NAME:-}" ]]; then
elif [[ -n "${SO_SCYLLACLUSTER_STORAGECLASS_NAME+x}" ]]; then
yq e --inplace 'del(.spec.datacenter.racks[0].storage.storageClassName)' "${DEPLOY_DIR}/manager/50_scyllacluster.yaml"
fi

if [[ -n "${SCYLLADB_VERSION:-}" ]]; then
yq e --inplace '.spec.version = env(SCYLLADB_VERSION)' "${DEPLOY_DIR}/manager/50_scyllacluster.yaml"
fi

if [[ -n "${SCYLLA_MANAGER_VERSION:-}" ]]; then
yq e --inplace '.spec.template.spec.containers[0].image |= "docker.io/scylladb/scylla-manager:" + env(SCYLLA_MANAGER_VERSION)' "${DEPLOY_DIR}/manager/50_manager_deployment.yaml"
fi

if [[ -n "${SCYLLA_MANAGER_AGENT_VERSION:-}" ]]; then
yq e --inplace '.spec.agentVersion = env(SCYLLA_MANAGER_AGENT_VERSION)' "${DEPLOY_DIR}/manager/50_scyllacluster.yaml"
fi

kubectl_create -f "${DEPLOY_DIR}"/manager

kubectl -n=scylla-manager wait --timeout=10m --for='condition=Progressing=False' scyllaclusters.scylla.scylladb.com/scylla-manager-cluster
Expand Down
93 changes: 80 additions & 13 deletions pkg/cmd/tests/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@ import (
"fmt"
"os"
"path"
"regexp"
"strings"

"github.com/onsi/ginkgo/v2"
configassets "github.com/scylladb/scylla-operator/assets/config"
scyllav1 "github.com/scylladb/scylla-operator/pkg/api/scylla/v1"
"github.com/scylladb/scylla-operator/pkg/genericclioptions"
"github.com/scylladb/scylla-operator/pkg/helpers/slices"
Expand All @@ -17,6 +19,16 @@ import (
"k8s.io/client-go/rest"
)

const (
// https://github.com/distribution/reference/blob/8c942b0459dfdcc5b6685581dd0a5a470f615bff/regexp.go#L68
referenceTagRegexp = `[\w][\w.-]{0,127}`

// https://github.com/distribution/reference/blob/8c942b0459dfdcc5b6685581dd0a5a470f615bff/regexp.go#L81
digestRegexp = `[A-Za-z][A-Za-z0-9]*(?:[-_+.][A-Za-z][A-Za-z0-9]*)*[:][[:xdigit:]]{32,}`
)

var tagWithOptionalDigestRegexp = regexp.MustCompile("^" + referenceTagRegexp + "(?:@" + digestRegexp + ")?$")

type IngressControllerOptions struct {
Address string
IngressClassName string
Expand Down Expand Up @@ -55,6 +67,11 @@ type TestFrameworkOptions struct {
objectStorageType framework.ObjectStorageType
gcsServiceAccountKey []byte
s3CredentialsFile []byte
ScyllaDBVersion string
ScyllaDBManagerVersion string
ScyllaDBManagerAgentVersion string
ScyllaDBUpdateFrom string
ScyllaDBUpgradeFrom string
}

func NewTestFrameworkOptions(streams genericclioptions.IOStreams, userAgent string) *TestFrameworkOptions {
Expand All @@ -69,12 +86,17 @@ func NewTestFrameworkOptions(streams genericclioptions.IOStreams, userAgent stri
ClientsBroadcastAddressType: string(scyllav1.BroadcastAddressTypePodIP),
StorageClassName: "",
},
ObjectStorageBucket: "",
GCSServiceAccountKeyPath: "",
S3CredentialsFilePath: "",
objectStorageType: framework.ObjectStorageTypeNone,
gcsServiceAccountKey: []byte{},
s3CredentialsFile: []byte{},
ObjectStorageBucket: "",
GCSServiceAccountKeyPath: "",
S3CredentialsFilePath: "",
objectStorageType: framework.ObjectStorageTypeNone,
gcsServiceAccountKey: []byte{},
s3CredentialsFile: []byte{},
ScyllaDBVersion: configassets.Project.Operator.ScyllaDBVersion,
ScyllaDBManagerVersion: configassets.Project.Operator.ScyllaDBManagerVersion,
ScyllaDBManagerAgentVersion: configassets.Project.Operator.ScyllaDBManagerAgentVersion,
ScyllaDBUpdateFrom: configassets.Project.OperatorTests.ScyllaDBVersions.UpdateFrom,
ScyllaDBUpgradeFrom: configassets.Project.OperatorTests.ScyllaDBVersions.UpgradeFrom,
}
}

Expand Down Expand Up @@ -118,6 +140,11 @@ func (o *TestFrameworkOptions) AddFlags(cmd *cobra.Command) {
cmd.PersistentFlags().StringVarP(&o.ObjectStorageBucket, "object-storage-bucket", "", o.ObjectStorageBucket, "Name of the object storage bucket.")
cmd.PersistentFlags().StringVarP(&o.GCSServiceAccountKeyPath, "gcs-service-account-key-path", "", o.GCSServiceAccountKeyPath, "Path to a file containing a GCS service account key.")
cmd.PersistentFlags().StringVarP(&o.S3CredentialsFilePath, "s3-credentials-file-path", "", o.S3CredentialsFilePath, "Path to the AWS credentials file providing access to the S3 bucket.")
cmd.PersistentFlags().StringVarP(&o.ScyllaDBVersion, "scylladb-version", "", o.ScyllaDBVersion, "Version of ScyllaDB to use.")
cmd.PersistentFlags().StringVarP(&o.ScyllaDBManagerVersion, "scylladb-manager-version", "", o.ScyllaDBManagerVersion, "Version of Scylla Manager to use.")
cmd.PersistentFlags().StringVarP(&o.ScyllaDBManagerAgentVersion, "scylladb-manager-agent-version", "", o.ScyllaDBManagerAgentVersion, "Version of Scylla Manager Agent to use.")
cmd.PersistentFlags().StringVarP(&o.ScyllaDBUpdateFrom, "scylladb-update-from-version", "", o.ScyllaDBUpdateFrom, "Version of ScyllaDB to update from.")
cmd.PersistentFlags().StringVarP(&o.ScyllaDBUpgradeFrom, "scylladb-upgrade-from-version", "", o.ScyllaDBUpgradeFrom, "Version of ScyllaDB to upgrade from.")
}

func (o *TestFrameworkOptions) Validate(args []string) error {
Expand Down Expand Up @@ -164,6 +191,41 @@ func (o *TestFrameworkOptions) Validate(args []string) error {
errors = append(errors, fmt.Errorf("gcs-service-account-key-path and s3-credentials-file-path can't be set simultanously"))
}

if !tagWithOptionalDigestRegexp.MatchString(o.ScyllaDBVersion) {
errors = append(errors, fmt.Errorf(
"invalid scylladb-version format: %q. Expected format: <tag>[@<digest>]",
o.ScyllaDBVersion,
))
}

if !tagWithOptionalDigestRegexp.MatchString(o.ScyllaDBUpdateFrom) {
errors = append(errors, fmt.Errorf(
"invalid scylladb-update-from-version format: %q. Expected format: <tag>[@<digest>]",
o.ScyllaDBUpdateFrom,
))
}

if !tagWithOptionalDigestRegexp.MatchString(o.ScyllaDBUpgradeFrom) {
errors = append(errors, fmt.Errorf(
"invalid scylladb-upgrade-from-version format: %q. Expected format: <tag>[@<digest>]",
o.ScyllaDBUpgradeFrom,
))
}

if !tagWithOptionalDigestRegexp.MatchString(o.ScyllaDBManagerVersion) {
errors = append(errors, fmt.Errorf(
"invalid scylladb-manager-version format: %q. Expected format: <tag>[@<digest>]",
o.ScyllaDBManagerVersion,
))
}

if !tagWithOptionalDigestRegexp.MatchString(o.ScyllaDBManagerAgentVersion) {
errors = append(errors, fmt.Errorf(
"invalid scylladb-manager-agent-version format: %q. Expected format: <tag>[@<digest>]",
o.ScyllaDBManagerAgentVersion,
))
}

if len(o.ArtifactsDir) > 0 {
_, err = os.Stat(o.ArtifactsDir)
if err != nil {
Expand Down Expand Up @@ -226,13 +288,18 @@ func (o *TestFrameworkOptions) Complete(args []string) error {
RestConfigs: slices.ConvertSlice(o.ClientConfigs, func(cc genericclioptions.ClientConfig) *rest.Config {
return cc.RestConfig
}),
ArtifactsDir: o.ArtifactsDir,
CleanupPolicy: o.CleanupPolicy,
ScyllaClusterOptions: o.scyllaClusterOptions,
ObjectStorageType: o.objectStorageType,
ObjectStorageBucket: o.ObjectStorageBucket,
GCSServiceAccountKey: o.gcsServiceAccountKey,
S3CredentialsFile: o.s3CredentialsFile,
ArtifactsDir: o.ArtifactsDir,
CleanupPolicy: o.CleanupPolicy,
ScyllaClusterOptions: o.scyllaClusterOptions,
ObjectStorageType: o.objectStorageType,
ObjectStorageBucket: o.ObjectStorageBucket,
GCSServiceAccountKey: o.gcsServiceAccountKey,
S3CredentialsFile: o.s3CredentialsFile,
ScyllaDBVersion: o.ScyllaDBVersion,
ScyllaDBManagerVersion: o.ScyllaDBManagerVersion,
ScyllaDBManagerAgentVersion: o.ScyllaDBManagerAgentVersion,
ScyllaDBUpdateFrom: o.ScyllaDBUpdateFrom,
ScyllaDBUpgradeFrom: o.ScyllaDBUpgradeFrom,
}

if o.IngressController != nil {
Expand Down
17 changes: 8 additions & 9 deletions test/e2e/framework/framework.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import (

g "github.com/onsi/ginkgo/v2"
o "github.com/onsi/gomega"
configassets "github.com/scylladb/scylla-operator/assets/config"
scyllav1 "github.com/scylladb/scylla-operator/pkg/api/scylla/v1"
scyllav1alpha1 "github.com/scylladb/scylla-operator/pkg/api/scylla/v1alpha1"
"github.com/scylladb/scylla-operator/pkg/controllerhelpers"
Expand Down Expand Up @@ -135,8 +134,8 @@ func (f *Framework) CommonLabels() map[string]string {

func (f *Framework) GetDefaultScyllaCluster() *scyllav1.ScyllaCluster {
renderArgs := map[string]any{
"scyllaDBVersion": configassets.Project.Operator.ScyllaDBVersion,
"scyllaDBManagerVersion": configassets.Project.Operator.ScyllaDBManagerAgentVersion,
"scyllaDBVersion": TestContext.ScyllaDBVersion,
"scyllaDBManagerVersion": TestContext.ScyllaDBManagerAgentVersion,
"nodeServiceType": TestContext.ScyllaClusterOptions.ExposeOptions.NodeServiceType,
"nodesBroadcastAddressType": TestContext.ScyllaClusterOptions.ExposeOptions.NodesBroadcastAddressType,
"clientsBroadcastAddressType": TestContext.ScyllaClusterOptions.ExposeOptions.ClientsBroadcastAddressType,
Expand All @@ -151,8 +150,8 @@ func (f *Framework) GetDefaultScyllaCluster() *scyllav1.ScyllaCluster {

func (f *Framework) GetDefaultZonalScyllaClusterWithThreeRacks() *scyllav1.ScyllaCluster {
renderArgs := map[string]any{
"scyllaDBVersion": configassets.Project.Operator.ScyllaDBVersion,
"scyllaDBManagerVersion": configassets.Project.Operator.ScyllaDBManagerAgentVersion,
"scyllaDBVersion": TestContext.ScyllaDBVersion,
"scyllaDBManagerVersion": TestContext.ScyllaDBManagerAgentVersion,
"nodeServiceType": TestContext.ScyllaClusterOptions.ExposeOptions.NodeServiceType,
"nodesBroadcastAddressType": TestContext.ScyllaClusterOptions.ExposeOptions.NodesBroadcastAddressType,
"clientsBroadcastAddressType": TestContext.ScyllaClusterOptions.ExposeOptions.ClientsBroadcastAddressType,
Expand All @@ -168,8 +167,8 @@ func (f *Framework) GetDefaultZonalScyllaClusterWithThreeRacks() *scyllav1.Scyll

func (f *Framework) GetDefaultScyllaDBDatacenter() *scyllav1alpha1.ScyllaDBDatacenter {
renderArgs := map[string]any{
"scyllaDBVersion": configassets.Project.Operator.ScyllaDBVersion,
"scyllaDBManagerVersion": configassets.Project.Operator.ScyllaDBManagerVersion,
"scyllaDBVersion": TestContext.ScyllaDBVersion,
"scyllaDBManagerVersion": TestContext.ScyllaDBManagerAgentVersion,
"nodeServiceType": TestContext.ScyllaClusterOptions.ExposeOptions.NodeServiceType,
"nodesBroadcastAddressType": TestContext.ScyllaClusterOptions.ExposeOptions.NodesBroadcastAddressType,
"clientsBroadcastAddressType": TestContext.ScyllaClusterOptions.ExposeOptions.ClientsBroadcastAddressType,
Expand All @@ -184,8 +183,8 @@ func (f *Framework) GetDefaultScyllaDBDatacenter() *scyllav1alpha1.ScyllaDBDatac

func (f *Framework) GetDefaultScyllaDBCluster(rkcs []*scyllav1alpha1.RemoteKubernetesCluster) *scyllav1alpha1.ScyllaDBCluster {
renderArgs := map[string]any{
"scyllaDBVersion": configassets.Project.Operator.ScyllaDBVersion,
"scyllaDBManagerVersion": configassets.Project.Operator.ScyllaDBManagerAgentVersion,
"scyllaDBVersion": TestContext.ScyllaDBVersion,
"scyllaDBManagerVersion": TestContext.ScyllaDBManagerAgentVersion,
"nodeServiceType": TestContext.ScyllaClusterOptions.ExposeOptions.NodeServiceType,
"nodesBroadcastAddressType": TestContext.ScyllaClusterOptions.ExposeOptions.NodesBroadcastAddressType,
"clientsBroadcastAddressType": TestContext.ScyllaClusterOptions.ExposeOptions.ClientsBroadcastAddressType,
Expand Down
23 changes: 14 additions & 9 deletions test/e2e/framework/testcontext.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,18 @@ const (
)

type TestContextType struct {
RestConfigs []*restclient.Config
ArtifactsDir string
CleanupPolicy CleanupPolicyType
IngressController *IngressController
ScyllaClusterOptions *ScyllaClusterOptions
ObjectStorageType ObjectStorageType
ObjectStorageBucket string
GCSServiceAccountKey []byte
S3CredentialsFile []byte
RestConfigs []*restclient.Config
ArtifactsDir string
CleanupPolicy CleanupPolicyType
IngressController *IngressController
ScyllaClusterOptions *ScyllaClusterOptions
ObjectStorageType ObjectStorageType
ObjectStorageBucket string
GCSServiceAccountKey []byte
S3CredentialsFile []byte
ScyllaDBVersion string
ScyllaDBManagerVersion string
ScyllaDBManagerAgentVersion string
ScyllaDBUpdateFrom string
ScyllaDBUpgradeFrom string
}
21 changes: 10 additions & 11 deletions test/e2e/set/scyllacluster/scyllacluster_upgrades.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (

g "github.com/onsi/ginkgo/v2"
o "github.com/onsi/gomega"
configassets "github.com/scylladb/scylla-operator/assets/config"
scyllav1 "github.com/scylladb/scylla-operator/pkg/api/scylla/v1"
"github.com/scylladb/scylla-operator/pkg/controllerhelpers"
"github.com/scylladb/scylla-operator/test/e2e/framework"
Expand Down Expand Up @@ -115,32 +114,32 @@ var _ = g.Describe("ScyllaCluster upgrades", func() {
g.Entry(describeEntry, &entry{
rackCount: 1,
rackSize: 1,
initialVersion: configassets.Project.OperatorTests.ScyllaDBVersions.UpdateFrom,
updatedVersion: configassets.Project.Operator.ScyllaDBVersion,
initialVersion: framework.TestContext.ScyllaDBUpdateFrom,
updatedVersion: framework.TestContext.ScyllaDBVersion,
}),
g.Entry(describeEntry, &entry{
rackCount: 1,
rackSize: 3,
initialVersion: configassets.Project.OperatorTests.ScyllaDBVersions.UpdateFrom,
updatedVersion: configassets.Project.Operator.ScyllaDBVersion,
initialVersion: framework.TestContext.ScyllaDBUpdateFrom,
updatedVersion: framework.TestContext.ScyllaDBVersion,
}),
g.Entry(describeEntry, &entry{
rackCount: 1,
rackSize: 1,
initialVersion: configassets.Project.OperatorTests.ScyllaDBVersions.UpgradeFrom,
updatedVersion: configassets.Project.Operator.ScyllaDBVersion,
initialVersion: framework.TestContext.ScyllaDBUpgradeFrom,
updatedVersion: framework.TestContext.ScyllaDBVersion,
}),
g.Entry(describeEntry, &entry{
rackCount: 1,
rackSize: 3,
initialVersion: configassets.Project.OperatorTests.ScyllaDBVersions.UpgradeFrom,
updatedVersion: configassets.Project.Operator.ScyllaDBVersion,
initialVersion: framework.TestContext.ScyllaDBUpgradeFrom,
updatedVersion: framework.TestContext.ScyllaDBVersion,
}),
g.Entry(describeEntry, &entry{
rackCount: 2,
rackSize: 3,
initialVersion: configassets.Project.OperatorTests.ScyllaDBVersions.UpgradeFrom,
updatedVersion: configassets.Project.Operator.ScyllaDBVersion,
initialVersion: framework.TestContext.ScyllaDBUpgradeFrom,
updatedVersion: framework.TestContext.ScyllaDBVersion,
}),
)
})

0 comments on commit b2d7fb1

Please sign in to comment.