Merge pull request #66 from kumarashit/dock_multic_fix

Fix the snapshot profile for multicloud snapshot of block devices to S3

contrib/backup/driver.go

@@ -40,7 +40,7 @@ var ctorFunMap = map[string]ctorFun{}
 func NewBackup(backupDriverName string) (BackupDriver, error) {
 	fun, exist := ctorFunMap[backupDriverName]
 	if !exist {
-		return nil, fmt.Errorf("specified backup driver does not exist")
+		return nil, fmt.Errorf("specified backup driver does not exist [%s]", backupDriverName)
 	}
 
 	drv, err := fun()

contrib/backup/multicloud/client.go

@@ -36,7 +36,7 @@ const (
 	DefaultTenantId      = "adminTenantId"
 	DefaultTimeout       = 60 // in Seconds
 	DefaultUploadTimeout = 30 // in Seconds
-	ApiVersion           = "v1"
+	ApiVersion           = ""
 )
 
 type Client struct {
@@ -175,7 +175,7 @@ func (c *Client) doRequest(method, u string, in interface{}, cb ReqSettingCB) ([
 		return nil, nil, err
 	}
 
-	log.V(5).Infof("%s: %s OK\n", method, u)
+	log.Infof("%s: %s OK\n", method, u)
 	rbody, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		log.Errorf("Get byte[] from response failed, method: %s\n url: %s\n error: %v", method, u, err)
@@ -201,7 +201,7 @@ func (c *Client) request(method, p string, in, out interface{}, cb ReqSettingCB)
 	}
 
 	if out != nil {
-		log.V(5).Infof("Response:\n%s\n", string(b))
+		log.Infof("Response:\n%s\n", string(b))
 		err := xml.Unmarshal(b, out)
 		if err != nil {
 			log.Errorf("unmarshal error, reason:%v", err)
@@ -253,13 +253,13 @@ type CompleteMultipartUploadResult struct {
 }
 
 func (c *Client) UploadObject(bucketName, objectKey string, data []byte) error {
-	p := path.Join("s3", bucketName, objectKey)
+	p := path.Join(bucketName, objectKey)
 	err := c.request("PUT", p, data, nil, nil)
 	return err
 }
 
 func (c *Client) ListObject(bucketName string) (*ListObjectResponse, error) {
-	p := path.Join("s3", bucketName)
+	p := path.Join(bucketName)
 	object := &ListObjectResponse{}
 	if err := c.request("GET", p, nil, object, nil); err != nil {
 		return nil, err
@@ -268,13 +268,13 @@ func (c *Client) ListObject(bucketName string) (*ListObjectResponse, error) {
 }
 
 func (c *Client) RemoveObject(bucketName, objectKey string) error {
-	p := path.Join("s3", bucketName, objectKey)
+	p := path.Join(bucketName, objectKey)
 	err := c.request("DELETE", p, nil, nil, nil)
 	return err
 }
 
 func (c *Client) InitMultiPartUpload(bucketName, objectKey string) (*InitiateMultipartUploadResult, error) {
-	p := path.Join("s3", bucketName, objectKey)
+	p := path.Join(bucketName, objectKey)
 	p += "?uploads"
 	out := &InitiateMultipartUploadResult{}
 	if err := c.request("PUT", p, nil, out, nil); err != nil {
@@ -285,7 +285,7 @@ func (c *Client) InitMultiPartUpload(bucketName, objectKey string) (*InitiateMul
 
 func (c *Client) UploadPart(bucketName, objectKey string, partNum int64, uploadId string, data []byte, size int64) (*UploadPartResult, error) {
 	log.Infof("upload part buf size:%d", len(data))
-	p := path.Join("s3", bucketName, objectKey)
+	p := path.Join(bucketName, objectKey)
 	p += fmt.Sprintf("?partNumber=%d&uploadId=%s", partNum, uploadId)
 	out := &UploadPartResult{}
 	reqSettingCB := func(req *httplib.BeegoHTTPRequest) error {
@@ -305,7 +305,7 @@ func (c *Client) CompleteMultipartUpload(
 	uploadId string,
 	input *CompleteMultipartUpload) (*CompleteMultipartUploadResult, error) {
 
-	p := path.Join("s3", bucketName, objectKey)
+	p := path.Join(bucketName, objectKey)
 	p += fmt.Sprintf("?uploadId=%s", uploadId)
 	out := &CompleteMultipartUploadResult{}
 	if err := c.request("PUT", p, input, nil, nil); err != nil {

contrib/backup/multicloud/driver.go

@@ -63,7 +63,7 @@ type MultiCloud struct {
 
 func (m *MultiCloud) loadConf(p string) (*MultiCloudConf, error) {
 	conf := &MultiCloudConf{
-		Endpoint:      "http://127.0.0.1:8088",
+		Endpoint:      "http://127.0.0.1:8090",
 		UploadTimeout: DefaultUploadTimeout,
 	}
 	confYaml, err := ioutil.ReadFile(p)
@@ -82,6 +82,7 @@ func (m *MultiCloud) SetUp() error {
 	// Set the default value
 	var err error
 	if m.conf, err = m.loadConf(ConfFile); err != nil {
+		glog.Errorf("setting up failed for MC")
 		return err
 	}
 
@@ -108,7 +109,7 @@ func (m *MultiCloud) Backup(backup *backup.BackupSpec, volFile *os.File) error {
 	key := backup.Id
 	initResp, err := m.client.InitMultiPartUpload(bucket, key)
 	if err != nil {
-		glog.Errorf("Init part failed, err:%v", err)
+		glog.Errorf("Init part failed, err:%v key %v", err, key)
 		return err
 	}
 

contrib/drivers/lvm/lvm.go

@@ -401,7 +401,6 @@ func (d *Driver) deleteUploadedSnapshot(backupId string, bucket string) error {
 
 func (d *Driver) CreateSnapshot(opt *pb.CreateVolumeSnapshotOpts) (snap *model.VolumeSnapshotSpec, err error) {
 	var snapName = snapshotPrefix + opt.GetId()
-
 	lvPath, ok := opt.GetMetadata()[KLvPath]
 	if !ok {
 		err := errors.New("can't find 'lvPath' in snapshot metadata")
@@ -420,6 +419,7 @@ func (d *Driver) CreateSnapshot(opt *pb.CreateVolumeSnapshotOpts) (snap *model.V
 	metadata := map[string]string{KLvsPath: lvsPath}
 
 	if bucket, ok := opt.Metadata["bucket"]; ok {
+
 		//nvmet right now can not support snap volume serve as nvme target
 		if vg == opensdsnvmepool {
 			log.Infof("nvmet right now can not support snap volume serve as nvme target")
@@ -532,7 +532,6 @@ func (d *Driver) InitializeSnapshotConnection(opt *pb.CreateSnapshotAttachmentOp
 	if hostIP == "" {
 		hostIP = d.conf.TgtBindIp
 	}
-
 	lvsPath, ok := opt.GetMetadata()[KLvsPath]
 	if !ok {
 		err := errors.New("Failed to find logic volume path in volume attachment metadata!")
@@ -543,13 +542,13 @@ func (d *Driver) InitializeSnapshotConnection(opt *pb.CreateSnapshotAttachmentOp
 	if d.conf.EnableChapAuth {
 		chapAuth = []string{utils.RandSeqWithAlnum(20), utils.RandSeqWithAlnum(16)}
 	}
-
 	accPro := opt.AccessProtocol
 	if accPro == nvmeofAccess {
 		log.Infof("nvmet right now can not support snap volume serve as nvme target")
 		log.Infof("still create snapshot connection by iscsi")
 		accPro = iscsiAccess
 	}
+	accPro = iscsiAccess
 	t := targets.NewTarget(d.conf.TgtBindIp, d.conf.TgtConfDir, accPro)
 	data, err := t.CreateExport(opt.GetSnapshotId(), lvsPath, hostIP, initiator, chapAuth)
 	if err != nil {

pkg/dock/discovery/discovery.go

@@ -29,14 +29,16 @@ import (
 	log "github.com/golang/glog"
 	"github.com/sodafoundation/dock/contrib/connector"
 	"github.com/sodafoundation/dock/contrib/drivers"
+	// Initialize the multicloud driver once
+	uuid "github.com/satori/go.uuid"
+	_ "github.com/sodafoundation/dock/contrib/backup/multicloud"
 	fd "github.com/sodafoundation/dock/contrib/drivers/filesharedrivers"
 	"github.com/sodafoundation/dock/contrib/drivers/utils/config"
 	c "github.com/sodafoundation/dock/pkg/context"
 	"github.com/sodafoundation/dock/pkg/db"
 	"github.com/sodafoundation/dock/pkg/model"
 	"github.com/sodafoundation/dock/pkg/utils"
 	. "github.com/sodafoundation/dock/pkg/utils/config"
-	uuid "github.com/satori/go.uuid"
 )
 
 const (

pkg/dock/dock.go

@@ -24,6 +24,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"time"
 
 	log "github.com/golang/glog"
 	"github.com/sodafoundation/dock/contrib/connector"
@@ -35,6 +36,7 @@ import (
 	"github.com/sodafoundation/dock/pkg/model"
 	pb "github.com/sodafoundation/dock/pkg/model/proto"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/keepalive"
 
 	_ "github.com/sodafoundation/dock/contrib/connector/fc"
 	_ "github.com/sodafoundation/dock/contrib/connector/iscsi"
@@ -72,7 +74,10 @@ func NewDockServer(dockType, port string) *dockServer {
 // backends, and then start the listen mechanism of dock module.
 func (ds *dockServer) Run() error {
 	// New Grpc Server
-	s := grpc.NewServer()
+	s := grpc.NewServer(grpc.KeepaliveParams(keepalive.ServerParameters{
+		MaxConnectionIdle: 5 * time.Minute, // Keep the connection alive
+	}),
+	)
 	// Register dock service.
 	pb.RegisterProvisionDockServer(s, ds)
 	pb.RegisterAttachDockServer(s, ds)