*: support juicefs #745

Author: acekingke

Dockerfile.sidecar

@@ -43,13 +43,21 @@ RUN set -ex; \
 ARG XTRABACKUP_PKG=percona-xtrabackup-24
 RUN set -ex; \
     apt-get update; \
-    apt-get install -y --no-install-recommends gnupg2 wget lsb-release curl bc; \
+    apt-get install -y --no-install-recommends gnupg2 wget lsb-release curl bc fuse jq openssh-server; \
     wget -P /tmp --no-check-certificate https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb; \
     dpkg -i /tmp/percona-release_latest.$(lsb_release -sc)_all.deb; \
     apt-get update; \
     apt-get install -y --no-install-recommends ${XTRABACKUP_PKG}; \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-
+#ADD http://mirrors.woqutech.com/download/qfusion/files/bin/juicefs-1.0.0-rc1-linux-amd64 /usr/local/bin/juicefs
+# COPY juicefs/juicefs /usr/local/bin/juicefs
+RUN wget --no-check-certificate "https://d.juicefs.com/juicefs/releases/download/v1.0.2/juicefs-1.0.2-linux-amd64.tar.gz" && tar -zxf "juicefs-1.0.2-linux-amd64.tar.gz" ;\
+    mv juicefs /usr/local/bin/juicefs; \
+    chmod +x /usr/local/bin/juicefs ; mkdir -p /run/sshd; \
+    mkdir -p /root/.ssh; \
+    chmod 700 /root/.ssh 
 WORKDIR /
 COPY --from=builder /workspace/bin/sidecar /usr/local/bin/sidecar
-ENTRYPOINT ["sidecar"]
+COPY script/sshd.sh /sshd.sh
+COPY script/backup.sh /backup.sh
+CMD [ "sidecar" ]

api/v1alpha1/backup_types.go

@@ -21,6 +21,14 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+type JuiceOpt struct {
+	// sqlite or redis
+	JuiceMeta string `json:"juiceMeta"`
+	// backupSecrete name for S3
+	BackupSecretName string `json:"backupSecretName"`
+	JuiceName        string `json:"juiceName"`
+}
+
 // This is the backup Job CRD.
 // BackupSpec defines the desired state of Backup
 type BackupSpec struct {
@@ -40,6 +48,9 @@ type BackupSpec struct {
 	// +optional
 	NFSServerAddress string `json:"nfsServerAddress,omitempty"`
 
+	// Represents the juicefs parameters which need.
+	// +optional
+	JuiceOpt *JuiceOpt `json:"juiceOpt,omitempty"`
 	// ClusterName represents the cluster name to backup
 	ClusterName string `json:"clusterName"`
 

api/v1alpha1/zz_generated.deepcopy.go

@@ -17,13 +17,16 @@ limitations under the License.
 package syncer
 
 import (
+	"context"
 	"fmt"
+	"strings"
 
 	"github.com/presslabs/controller-util/pkg/syncer"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	v1alpha1 "github.com/radondb/radondb-mysql-kubernetes/api/v1alpha1"
@@ -33,6 +36,7 @@ import (
 )
 
 type jobSyncer struct {
+	client client.Client
 	job    *batchv1.Job
 	backup *backup.Backup
 }
@@ -50,6 +54,7 @@ func NewJobSyncer(c client.Client, backup *backup.Backup) syncer.Interface {
 	}
 
 	sync := &jobSyncer{
+		client: c,
 		job:    obj,
 		backup: backup,
 	}
@@ -174,6 +179,10 @@ func (s *jobSyncer) ensurePodSpec(in corev1.PodSpec) corev1.PodSpec {
 				MountPath: utils.XtrabckupLocal,
 			},
 		}
+	} else if s.backup.Spec.JuiceOpt != nil {
+		// Deal it for juiceOpt
+		s.buildJuicefsBackPod(&in)
+
 	} else {
 		// in.Containers[0].ImagePullPolicy = s.opt.ImagePullPolicy
 		in.Containers[0].Args = []string{
@@ -238,3 +247,88 @@ func (s *jobSyncer) ensurePodSpec(in corev1.PodSpec) corev1.PodSpec {
 	}
 	return in
 }
+
+func (s *jobSyncer) buildJuicefsBackPod(in *corev1.PodSpec) error {
+	// add volumn about pvc
+	var defMode int32 = 0600
+	var err error
+	var cmdstr string
+	in.Volumes = []corev1.Volume{
+		{
+			Name: utils.SShVolumnName,
+			VolumeSource: corev1.VolumeSource{
+				Secret: &corev1.SecretVolumeSource{
+					SecretName:  fmt.Sprintf("%s-ssh-key", s.backup.Spec.ClusterName),
+					DefaultMode: &defMode,
+				},
+			},
+		},
+	}
+
+	in.Containers[0].VolumeMounts = []corev1.VolumeMount{
+		{
+			Name:      utils.SShVolumnName,
+			MountPath: utils.SshVolumnPath,
+		},
+	}
+
+	// PodName.clusterName-mysql.Namespace
+	// sample-mysql-0.sample-mysql.default
+	hostname := fmt.Sprintf("%s.%s-mysql.%s", s.backup.Spec.HostName, s.backup.Spec.ClusterName, s.backup.Namespace)
+	if cmdstr, err = s.buildJuicefsCmd(s.backup.Spec.JuiceOpt.BackupSecretName); err != nil {
+		return err
+	}
+
+	in.Containers[0].Command = []string{"bash", "-c", "--", `cp  /etc/secret-ssh/* /root/.ssh
+chmod 600 /root/.ssh/authorized_keys ;` +
+		strings.Join([]string{
+			"ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", hostname, cmdstr,
+		}, " ")}
+
+	return nil
+}
+
+func (s *jobSyncer) buildJuicefsCmd(secName string) (string, error) {
+	juiceopt := s.backup.Spec.JuiceOpt
+	secret := &corev1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "Secret",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      secName,
+			Namespace: s.backup.Namespace,
+		},
+	}
+	err := s.client.Get(context.TODO(),
+		types.NamespacedName{Namespace: s.backup.Namespace,
+			Name: secName}, secret)
+
+	if err != nil {
+		return "", err
+	}
+	url, bucket := secret.Data["s3-endpoint"], secret.Data["s3-bucket"]
+	accesskey, secretkey := secret.Data["s3-access-key"], secret.Data["s3-secret-key"]
+	juicebucket := installBucket(string(url), string(bucket))
+	cmdstr := fmt.Sprintf(`<<EOF
+	export CLUSTER_NAME=%s
+	juicefs format --storage s3 \
+    --bucket  %s \
+    --access-key %s \
+    --secret-key %s \
+    %s \
+    %s`, s.backup.Spec.ClusterName, juicebucket, accesskey, secretkey, juiceopt.JuiceMeta, juiceopt.JuiceName)
+	cmdstr += fmt.Sprintf(`
+	juicefs mount -d %s /%s/
+	`, juiceopt.JuiceMeta, juiceopt.JuiceName)
+	cmdstr += fmt.Sprintf(`
+	source /backup.sh
+    backup
+	juicefs umount /%s/
+EOF`, juiceopt.JuiceName)
+	return cmdstr, nil
+}
+
+func installBucket(url, bucket string) string {
+	return strings.Join(strings.Split(url, "//"), "//"+bucket+".")
+}

backup/syncer/job.go

@@ -70,6 +70,22 @@ spec:
                 default: radondb/mysql57-sidecar:v2.3.0
                 description: To specify the image that will be used for sidecar container.
                 type: string
+              juiceOpt:
+                description: Represents the juicefs parameters which need.
+                properties:
+                  backupSecretName:
+                    description: backupSecrete name for S3
+                    type: string
+                  juiceMeta:
+                    description: sqlite or redis
+                    type: string
+                  juiceName:
+                    type: string
+                required:
+                - backupSecretName
+                - juiceMeta
+                - juiceName
+                type: object
               nfsServerAddress:
                 description: Represents the ip address of the nfs server.
                 type: string

config/crd/bases/mysql.radondb.com_backups.yaml

@@ -24,7 +24,9 @@ import (
 	"github.com/presslabs/controller-util/pkg/syncer"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
-	policyv1beta1 "k8s.io/api/policy/v1beta1"
+
+	// policyv1beta1 "k8s.io/api/policy/v1beta1"
+	policyv1beta1 "k8s.io/api/policy/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/labels"
@@ -125,6 +127,11 @@ func (r *MysqlClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		return ctrl.Result{}, err
 	}
 
+	secretSShSyncer := clustersyncer.NewSShKeySyncer(r.Client, instance)
+	if err = syncer.Sync(ctx, secretSShSyncer, r.Recorder); err != nil {
+		return ctrl.Result{}, err
+	}
+
 	// Todo: modify mysql cm will trigger rolling update but it will not be applied.
 	cmRev := mysqlCMSyncer.Object().(*corev1.ConfigMap).ResourceVersion
 	sctRev := secretSyncer.Object().(*corev1.Secret).ResourceVersion

controllers/mysqlcluster_controller.go

@@ -38,7 +38,7 @@ func (c *backupSidecar) getImage() string {
 }
 
 func (c *backupSidecar) getCommand() []string {
-	return []string{"sidecar", "http"}
+	return []string{"sh", "-c", "/sshd.sh ; sidecar http"}
 }
 
 func (c *backupSidecar) getEnvVars() []corev1.EnvVar {
@@ -147,5 +147,13 @@ func (c *backupSidecar) getVolumeMounts() []corev1.VolumeMount {
 			Name:      utils.SysLocalTimeZone,
 			MountPath: utils.SysLocalTimeZoneMountPath,
 		},
+		{
+			Name:      utils.SysFuseVolume,
+			MountPath: utils.SysFuseVolumnMountPath,
+		},
+		{
+			Name:      utils.SShVolumnName,
+			MountPath: utils.SshVolumnPath,
+		},
 	}
 }

mysqlcluster/container/backup.go

@@ -63,6 +63,7 @@ func getStartupProbe(name string) *corev1.Probe {
 // EnsureContainer ensure a container by the giving name.
 func EnsureContainer(name string, c *mysqlcluster.MysqlCluster) corev1.Container {
 	var ctr container
+	var security *corev1.SecurityContext = nil
 	switch name {
 	case utils.ContainerInitSidecarName:
 		ctr = &initSidecar{c, name}
@@ -80,6 +81,14 @@ func EnsureContainer(name string, c *mysqlcluster.MysqlCluster) corev1.Container
 		ctr = &auditLog{c, name}
 	case utils.ContainerBackupName:
 		ctr = &backupSidecar{c, name}
+		needAdmin := true
+		security = &corev1.SecurityContext{
+			Privileged: &needAdmin,
+			Capabilities: &corev1.Capabilities{
+				Add: []corev1.Capability{"CAP_SYS_ADMIN",
+					"DAC_READ_SEARCH",
+				},
+			}}
 	}
 
 	return corev1.Container{
@@ -95,5 +104,6 @@ func EnsureContainer(name string, c *mysqlcluster.MysqlCluster) corev1.Container
 		ReadinessProbe:  ctr.getReadinessProbe(),
 		StartupProbe:    getStartupProbe(name),
 		VolumeMounts:    ctr.getVolumeMounts(),
+		SecurityContext: security,
 	}
 }

mysqlcluster/container/container.go

@@ -180,7 +180,7 @@ func (c *MysqlCluster) EnsureVolumes() []corev1.Volume {
 			},
 		)
 	}
-
+	var defMode int32 = 0600
 	volumes = append(volumes,
 		corev1.Volume{
 			Name: utils.MysqlConfVolumeName,
@@ -246,6 +246,23 @@ func (c *MysqlCluster) EnsureVolumes() []corev1.Volume {
 				},
 			},
 		},
+		corev1.Volume{
+			Name: utils.SysFuseVolume,
+			VolumeSource: corev1.VolumeSource{
+				HostPath: &corev1.HostPathVolumeSource{
+					Path: "/dev/fuse",
+				},
+			},
+		},
+		corev1.Volume{
+			Name: utils.SShVolumnName,
+			VolumeSource: corev1.VolumeSource{
+				Secret: &corev1.SecretVolumeSource{
+					SecretName:  c.GetNameForResource(utils.SShKey),
+					DefaultMode: &defMode,
+				},
+			},
+		},
 	)
 	// add the nfs volumn mount
 	if len(c.Spec.NFSServerAddress) != 0 {
@@ -328,6 +345,8 @@ func (c *MysqlCluster) GetNameForResource(name utils.ResourceName) string {
 		return fmt.Sprintf("%s-metrics", c.Name)
 	case utils.Secret:
 		return fmt.Sprintf("%s-secret", c.Name)
+	case utils.SShKey:
+		return fmt.Sprintf("%s-ssh-key", c.Name)
 	case utils.XenonMetaData:
 		return fmt.Sprintf("%s-xenon", c.Name)
 	case utils.ConfigMap:

mysqlcluster/mysqlcluster.go

@@ -57,8 +57,8 @@ func NewHeadlessSVCSyncer(cli client.Client, c *mysqlcluster.MysqlCluster) synce
 		// Use `publishNotReadyAddresses` to be able to access pods even if the pod is not ready.
 		service.Spec.PublishNotReadyAddresses = true
 
-		if len(service.Spec.Ports) != 2 {
-			service.Spec.Ports = make([]corev1.ServicePort, 2)
+		if len(service.Spec.Ports) != 3 {
+			service.Spec.Ports = make([]corev1.ServicePort, 3)
 		}
 
 		service.Spec.Ports[0].Name = utils.MysqlPortName
@@ -68,6 +68,10 @@ func NewHeadlessSVCSyncer(cli client.Client, c *mysqlcluster.MysqlCluster) synce
 		service.Spec.Ports[1].Name = utils.XBackupPortName
 		service.Spec.Ports[1].Port = utils.XBackupPort
 		service.Spec.Ports[1].TargetPort = intstr.FromInt(utils.XBackupPort)
+		// ssh port
+		service.Spec.Ports[2].Name = utils.SshPortName
+		service.Spec.Ports[2].Port = utils.SshPort
+		service.Spec.Ports[2].TargetPort = intstr.FromInt(utils.SshPort)
 		return nil
 	})
 }

mysqlcluster/syncer/headless_service.go

@@ -18,7 +18,7 @@ package syncer
 
 import (
 	"github.com/presslabs/controller-util/pkg/syncer"
-	policyv1beta1 "k8s.io/api/policy/v1beta1"
+	policyv1beta1 "k8s.io/api/policy/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"sigs.k8s.io/controller-runtime/pkg/client"