[BACKPORT 2.20.2][PLAT-12836] Fixed resolve symlinks across filesystems

Summary:
Original commit: 8b982b5 / D32628
os.Rename (and the rename syscall) will not work across filesystems, which some customers are
setting up as part of their replicated migration. Instead, we will run an "mv" command which will work
and be idempotent (not atomic though) across filesystems.

In addition, cleaned up some other minor logging issues and we won't fail if replicated uninstall fails

Test Plan:
setup vm with 2 extra mountpoints at /opt/yugabyte and /opt/ybanywhere

1. Ran through a full migration without issues
2. Failed a migration during symlink resolution. Fixed the failure and retried. Saw success

Reviewers: muthu, sanketh

Reviewed By: muthu

Subscribers: yugaware, muthu, sanketh

Tags: #jenkins-ready

Differential Revision: https://phorge.dev.yugabyte.com/D33280

managed/yba-installer/cmd/platform.go

@@ -9,7 +9,6 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -636,10 +635,11 @@ func (plat Platform) FinishReplicatedMigrate() error {
 			log.DebugLF("skipping directory " + file.Name() + " as it is not a symlink")
 			continue
 		}
-		err = common.ResolveSymlink(filepath.Join(
-			common.GetBaseInstall(), "data/yb-platform/releases", file.Name()))
+		src := filepath.Join(common.GetReplicatedBaseDir(), "releases", file.Name())
+		target := filepath.Join(common.GetBaseInstall(), "data/yb-platform/releases", file.Name())
+		err = common.ResolveSymlink(src, target)
 		if err != nil {
-			return fmt.Errorf("Could not complete migration of platform: %w", err)
+			return fmt.Errorf("could not complete migration of platform: %w", err)
 		}
 	}
 	return nil
@@ -699,7 +699,7 @@ func createPemFormatKeyAndCert() error {
 
 func (plat Platform) symlinkReplicatedData() error {
 	// First do the previous releases.
-	releases, err := ioutil.ReadDir(filepath.Join(common.GetReplicatedBaseDir(), "releases/"))
+	releases, err := os.ReadDir(filepath.Join(common.GetReplicatedBaseDir(), "releases/"))
 	if err != nil {
 		return fmt.Errorf("could not read replicated releases dir: %w", err)
 	}

managed/yba-installer/cmd/prometheus.go

@@ -26,7 +26,7 @@ import (
 type prometheusDirectories struct {
 	SystemdFileLocation string
 	ConfFileLocation    string // This is used during config generation
-	WebConfFile					string
+	WebConfFile         string
 	templateFileName    string
 	DataDir             string
 	PromDir             string
@@ -37,7 +37,7 @@ func newPrometheusDirectories() prometheusDirectories {
 	return prometheusDirectories{
 		SystemdFileLocation: common.SystemdDir + "/prometheus.service",
 		ConfFileLocation:    common.GetSoftwareRoot() + "/prometheus/conf/prometheus.yml",
-		WebConfFile:				 common.GetSoftwareRoot() + "/prometheus/conf/web.yml",
+		WebConfFile:         common.GetSoftwareRoot() + "/prometheus/conf/web.yml",
 		templateFileName:    "yba-installer-prometheus.yml",
 		DataDir:             common.GetBaseInstall() + "/data/prometheus",
 		PromDir:             common.GetSoftwareRoot() + "/prometheus",
@@ -387,14 +387,27 @@ func (prom Prometheus) FixBasicAuth() error {
 
 // FinishReplicatedMigrate completest the replicated migration prometheus specific tasks
 func (prom Prometheus) FinishReplicatedMigrate() error {
-	links := []string{
-		filepath.Join(prom.DataDir, "storage"),
-		filepath.Join(prom.DataDir, "swamper_targets"),
-		filepath.Join(prom.DataDir, "swamper_rules"),
+	rootDir := common.GetReplicatedBaseDir()
+	linkDirs := []struct {
+		src  string
+		dest string
+	}{
+		{
+			filepath.Join(rootDir, "prometheusv2"),
+			filepath.Join(prom.DataDir, "storage"),
+		},
+		{
+			filepath.Join(rootDir, "/yugaware/swamper_targets"),
+			filepath.Join(prom.DataDir, "swamper_targets"),
+		},
+		{
+			filepath.Join(rootDir, "yugaware/swamper_rules"),
+			filepath.Join(prom.DataDir, "swamper_rules"),
+		},
 	}
 
-	for _, link := range links {
-		if err := common.ResolveSymlink(link); err != nil {
+	for _, link := range linkDirs {
+		if err := common.ResolveSymlink(link.src, link.dest); err != nil {
 			return fmt.Errorf("could not complete prometheus migration: %w", err)
 		}
 	}

managed/yba-installer/cmd/replicated_migrate.go

@@ -304,19 +304,22 @@ Are you sure you want to continue?`
 		if err := state.TransitionStatus(ybactlstate.FinishingStatus); err != nil {
 			log.Fatal("Failed to update status: " + err.Error())
 		}
+		common.SetReplicatedBaseDir(state.Replicated.StoragePath)
 
 		for _, name := range serviceOrder {
 			if err := services[name].FinishReplicatedMigrate(); err != nil {
 				log.Fatal("could not finish replicated migration for " + name + ": " + err.Error())
 			}
 		}
 		if err := replflow.Uninstall(); err != nil {
-			log.Fatal("unable to uninstall replicated: " + err.Error())
+			log.Error("unable to uninstall replicated: " + err.Error())
+			log.Info("Please manually uninstall replicated")
 		}
 		state.CurrentStatus = ybactlstate.InstalledStatus
 		if err := ybactlstate.StoreState(state); err != nil {
 			log.Fatal("Failed to save state: " + err.Error())
 		}
+		log.Info("Completed migration")
 	},
 }
 

managed/yba-installer/pkg/common/utils.go

@@ -10,7 +10,9 @@ import (
 	"database/sql"
 	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"io/fs"
 	"net"
 	"os"
 	"os/exec"
@@ -146,19 +148,93 @@ func Symlink(src string, dest string) error {
 	return out.Error
 }
 
-// ResolveSymlink will read the given symlink and move the original file to the symlinks
-// destination. This requires the given path is a symlink, and will not check if it is a real file
-// or a symlink
-func ResolveSymlink(path string) error {
-	orig, err := os.Readlink(path)
-	if err != nil {
-		return fmt.Errorf("failed to resolve symlink for %s: %w", path, err)
+func ResolveSymlink(source, target string) error {
+	_, tErr := os.Stat(target)
+	_, sErr := os.Stat(source)
+	// Handle non errNotExist errors
+	if tErr != nil && !errors.Is(tErr, fs.ErrNotExist) {
+		return fmt.Errorf("failed to stat target %s: %w", target, tErr)
+	}
+	if sErr != nil && !errors.Is(sErr, fs.ErrNotExist) {
+		return fmt.Errorf("failed to stat source %s: %w", target, sErr)
+	}
+
+	// Handle neither source nor target existing
+	if errors.Is(tErr, fs.ErrNotExist) && errors.Is(sErr, fs.ErrNotExist) {
+		msg := fmt.Sprintf("Neither source %s nor target %s exist", source, target)
+		log.Error(msg)
+		return fmt.Errorf(msg)
+		// Handle only target existing (already resolved)
+	} else if tErr == nil && errors.Is(sErr, fs.ErrNotExist) {
+		log.Debug(fmt.Sprintf("Symlink %s -> %s already resolved", source, target))
+		return nil
 	}
-	if err := os.Remove(path); err != nil {
-		return fmt.Errorf("failed to remove the symlink %s: %w", path, err)
+
+	// Remove the target if needed
+	if tErr == nil {
+		if err := os.RemoveAll(target); err != nil {
+			log.Error("failed to delete link target " + target)
+			return fmt.Errorf("failed to remove target %s: %w", target, err)
+		}
 	}
-	if err := os.Rename(orig, path); err != nil {
-		return fmt.Errorf("failed to move %s -> %s: %w", orig, path, err)
+	// Try to do an os.Rename. This will fail across filesystems
+	if err := os.Rename(source, target); err != nil {
+		if strings.Contains(err.Error(), "invalid cross-device link") {
+			log.Debug("cross-device link detected, using fallback copy implementation")
+			return resolveSymlinkFallback(source, target)
+		}
+		log.Error(fmt.Sprintf("failed to rename %s -> %s", source, target))
+		return fmt.Errorf("resolve symlink failed to rename %s->%s: %w", source, target, err)
+	}
+	return nil
+}
+
+// resolveSymlinkFallback will, given a source (file/dir) and target (symlink), it will move the source to
+// the target destination. This supports moving across filesystems/devices, and is idempotent.
+// Logic to be idempotent:
+// 1. if the source directory exists always copy it to the target
+// 1.A delete the target before copy if needed
+// 2. move the source to source-tmp
+// 3. Best effort to delete source-tmp.
+// Fail if neither source nor target exist
+func resolveSymlinkFallback(source, target string) error {
+	srcTmpName := fmt.Sprintf("%s-tmp", source)
+
+	// Source still exists, copy to target
+	if _, sErr := os.Stat(source); sErr == nil {
+		// Delete Target if it exists
+		if _, tErr := os.Stat(target); tErr == nil {
+			if err := RemoveAll(target); err != nil {
+				return fmt.Errorf("target directory %s could not be deleted: %w", target, err)
+			}
+		} else if errors.Is(tErr, fs.ErrNotExist) {
+			log.DebugLF("target directory already deleted")
+		} else {
+			return fmt.Errorf("could not determine status of target directory %s: %w", target, tErr)
+		}
+		log.Debug("Copy symlink source to target")
+		if out := shell.Run("cp", "-r", source, target); !out.SucceededOrLog() {
+			return fmt.Errorf("copying from %s -> %s failed while resolving symlink: %w",
+				source, target, out.Error)
+		}
+
+		log.DebugLF("Rename source to temp name")
+		if err := os.Rename(source, srcTmpName); err != nil {
+			return fmt.Errorf("rename to temp dir: %w", err)
+		}
+	} else if errors.Is(sErr, fs.ErrNotExist) {
+		log.DebugLF("no source directory found")
+		if _, tErr := os.Stat(target); errors.Is(tErr, fs.ErrNotExist) {
+			log.Error("no source or target found")
+			return fmt.Errorf("could not find source %s nor target %s directories", source, target)
+		}
+	} else {
+		return fmt.Errorf("could not determine status of source directory %s: %w", source, sErr)
+	}
+
+	// Remove the temp dir if it exists. This is best effort, and can be cleaned up manually later
+	if err := RemoveAll(srcTmpName); err != nil {
+		log.Warn("failed to remove backup source directory " + srcTmpName + ": " + err.Error())
 	}
 	return nil
 }

managed/yba-installer/pkg/replicated/replflow/uninstall.go

@@ -31,7 +31,7 @@ func Uninstall() error {
 	replCtl := replicatedctl.New(replicatedctl.Config{})
 	config, err := replCtl.AppConfigExport()
 	if err != nil {
-		logging.Fatal("failed to export replicated app config: " + err.Error())
+		return fmt.Errorf("failed to export replicated app config: %w", err)
 	}
 	rootDir, ok := config.ConfigEntries[replicatedctl.StoragePathKey]
 	if !ok {