Storage: Show correct disk size

doc/api-extensions.md

@@ -2538,3 +2538,8 @@ Adds a new {config:option}`device-unix-hotplug-device-conf:subsystem` configurat
 
 ## `storage_ceph_osd_pool_size`
 This introduces the configuration keys {config:option}`storage-ceph-pool-conf:ceph.osd.pool_size`, and {config:option}`storage-cephfs-pool-conf:cephfs.osd_pool_size` to be used when adding or updating a `ceph` or `cephfs` storage pool to instruct LXD to create set the replication size for the underlying OSD pools.
+
+## `storage_volumes_instance_rootfs_size`
+
+Introduces a `volatile.rootfs.size` key, previously only supported for images, for container and virtual machine volumes. This key is used to store a volume's current size at any point.
+Necessary to get accurate root disk sizes for instances.

lxd/db/storage_volumes.go

@@ -372,21 +372,33 @@ func storageVolumeSnapshotConfig(ctx context.Context, tx *ClusterTx, volumeSnaps
 	}, volumeSnapshotID)
 }
 
-// UpdateStoragePoolVolume updates the storage volume attached to a given storage pool.
-func (c *ClusterTx) UpdateStoragePoolVolume(ctx context.Context, projectName string, volumeName string, volumeType int, poolID int64, volumeDescription string, volumeConfig map[string]string) error {
+// UpdateStoragePoolVolumeConfig updates a storage volume's config on the database.
+func (c *ClusterTx) UpdateStoragePoolVolumeConfig(volumeName string, volumeID int64, volumeConfig map[string]string) error {
 	isSnapshot := strings.Contains(volumeName, shared.SnapshotDelimiter)
 
-	volume, err := c.GetStoragePoolVolume(ctx, poolID, projectName, volumeType, volumeName, true)
+	err := storageVolumeConfigClear(c.tx, volumeID, isSnapshot)
+	if err != nil {
+		return err
+	}
+
+	err = storageVolumeConfigAdd(c.tx, volumeID, volumeConfig, isSnapshot)
 	if err != nil {
 		return err
 	}
 
-	err = storageVolumeConfigClear(c.tx, volume.ID, isSnapshot)
+	return nil
+}
+
+// UpdateStoragePoolVolume updates the storage volume attached to a given storage pool.
+func (c *ClusterTx) UpdateStoragePoolVolume(ctx context.Context, projectName string, volumeName string, volumeType int, poolID int64, volumeDescription string, volumeConfig map[string]string) error {
+	isSnapshot := strings.Contains(volumeName, shared.SnapshotDelimiter)
+
+	volume, err := c.GetStoragePoolVolume(ctx, poolID, projectName, volumeType, volumeName, true)
 	if err != nil {
 		return err
 	}
 
-	err = storageVolumeConfigAdd(c.tx, volume.ID, volumeConfig, isSnapshot)
+	err = c.UpdateStoragePoolVolumeConfig(volume.Name, volume.ID, volumeConfig)
 	if err != nil {
 		return err
 	}

lxd/storage/backend_lxd.go

@@ -3271,14 +3271,27 @@ func (b *lxdBackend) GetInstanceUsage(inst instance.Instance) (*VolumeUsage, err
 	// If driver does not support getting volume usage, this value would be -1.
 	val.Used = size
 
-	// Get the total size.
-	_, rootDiskConf, err := instancetype.GetRootDiskDevice(inst.ExpandedDevices().CloneNative())
-	if err != nil {
-		return nil, err
+	// The instance volume 'volatile.rootfs.size' config key is the most accurate representation of the instance root disk size.
+	// The device's own 'volatile.rootfs.size' is not influenced by 'volume.size' on the pool level and may be inaccurate when
+	// the instance does not support online resizing.
+	sizeStr := dbVol.Config["volatile.rootfs.size"]
+
+	// If the instance isn't running, return the value its volume will assume when it starts even if the
+	// volume quota hasn't been set yet.
+	if !inst.IsRunning() {
+		// Get the total size.
+		_, rootDiskConf, err := instancetype.GetRootDiskDevice(inst.ExpandedDevices().CloneNative())
+		if err != nil {
+			return nil, err
+		}
+
+		rootDiskSize, ok := rootDiskConf["size"]
+		if ok {
+			sizeStr = rootDiskSize
+		}
 	}
 
-	sizeStr, ok := rootDiskConf["size"]
-	if ok {
+	if sizeStr != "" {
 		total, err := units.ParseByteSizeString(sizeStr)
 		if err != nil {
 			return nil, err
@@ -3302,7 +3315,7 @@ func (b *lxdBackend) GetInstanceUsage(inst instance.Instance) (*VolumeUsage, err
 
 // SetInstanceQuota sets the quota on the instance's root volume.
 // Returns ErrInUse if the instance is running and the storage driver doesn't support online resizing.
-func (b *lxdBackend) SetInstanceQuota(inst instance.Instance, size string, vmStateSize string, op *operations.Operation) error {
+func (b *lxdBackend) SetInstanceQuota(inst instance.Instance, size string, vmStateSize string, op *operations.Operation) (err error) {
 	l := b.logger.AddContext(logger.Ctx{"project": inst.Project().Name, "instance": inst.Name(), "size": size, "vm_state_size": vmStateSize})
 	l.Debug("SetInstanceQuota started")
 	defer l.Debug("SetInstanceQuota finished")
@@ -3322,6 +3335,18 @@ func (b *lxdBackend) SetInstanceQuota(inst instance.Instance, size string, vmSta
 		return err
 	}
 
+	// If the volume resize succeeds, update the instance volume's 'size' config on the database.
+	defer func() {
+		if err == nil {
+			newConfig := dbVol.Config
+			newConfig["volatile.rootfs.size"] = size
+
+			err = b.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
+				return tx.UpdateStoragePoolVolumeConfig(dbVol.Name, dbVol.ID, newConfig)
+			})
+		}
+	}()
+
 	// Apply the main volume quota.
 	vol := b.GetVolume(volType, contentVolume, volStorageName, dbVol.Config)
 	err = b.driver.SetVolumeQuota(vol, size, false, op)

lxd/storage/utils.go

@@ -724,9 +724,9 @@ func validateVolumeCommonRules(vol drivers.Volume) map[string]func(string) error
 		rules["block.filesystem"] = validate.IsAny
 	}
 
-	// volatile.rootfs.size is only used for image volumes.
-	if vol.Type() == drivers.VolumeTypeImage {
-		rules["volatile.rootfs.size"] = validate.Optional(validate.IsInt64)
+	// volatile.rootfs.size is only used for image and instance volumes.
+	if shared.ValueInSlice(vol.Type(), []drivers.VolumeType{drivers.VolumeTypeImage, drivers.VolumeTypeContainer, drivers.VolumeTypeContainer}) {
+		rules["volatile.rootfs.size"] = validate.Optional(validate.IsSize)
 	}
 
 	return rules

lxd/storage_volumes_state.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"strconv"
 
 	"github.com/gorilla/mux"
 
@@ -146,6 +147,27 @@ func storagePoolVolumeTypeStateGet(d *Daemon, r *http.Request) response.Response
 		if err != nil && err != storageDrivers.ErrNotSupported {
 			return response.SmartError(err)
 		}
+
+		// If the instance is stopped and its size was recently changed, the volume may not have been resized yet
+		// as this only happens on instance start.
+		// When this is the case, fallback to getting the total size from the volume's config.
+		if !inst.IsRunning() {
+			volType, err := storagePools.InstanceTypeToVolumeType(inst.Type())
+			if err != nil {
+				return response.SmartError(err)
+			}
+
+			// Load storage volume from database.
+			dbVol, err := storagePools.VolumeDBGet(pool, inst.Project().Name, inst.Name(), volType)
+			if err != nil {
+				return response.SmartError(err)
+			}
+
+			// If size is not set, move on with total 0.
+			volumeSize, _ := strconv.ParseInt(dbVol.Config["volatile.rootfs.size"], 10, 64)
+
+			usage.Total = volumeSize
+		}
 	}
 
 	// Prepare the state struct.

shared/version/api.go

@@ -428,6 +428,7 @@ var APIExtensions = []string{
 	"unix_device_hotplug_ownership_inherit",
 	"unix_device_hotplug_subsystem_device_option",
 	"storage_ceph_osd_pool_size",
+	"storage_volumes_instance_rootfs_size",
 }
 
 // APIExtensionsCount returns the number of available API extensions.

test/suites/storage.sh

@@ -952,6 +952,15 @@ EOF
   LXD_DIR="${LXD_DIR}"
   storage_pool="lxdtest-$(basename "${LXD_DIR}")-pool26"
   lxc storage create "$storage_pool" "$lxd_backend"
+
+  if [ "${lxd_backend}" = "zfs" ]; then
+    get_instance_size "zfs"
+    lxc storage set "$storage_pool" volume.zfs.block_mode true
+    get_instance_size "zfs-block"
+  else
+    get_instance_size "${lxd_backend}"
+  fi
+
   lxc init -s "${storage_pool}" testimage c1
   # The storage pool will not be removed since it has c1 attached to it
   ! lxc storage delete "${storage_pool}" || false
@@ -1025,3 +1034,71 @@ EOF
   LXD_DIR="${LXD_DIR}"
   kill_lxd "${LXD_STORAGE_DIR}"
 }
+
+get_instance_size() {
+  driver="{$1}"
+
+  lxc init -s "${storage_pool}" testimage c1
+  # Set 'volume.size' and create new container
+  lxc storage set "${storage_pool}" volume.size="7GiB"
+  lxc init -s "${storage_pool}" testimage c2
+
+  # Non block-based drivers.
+  if [ "${driver}" = "btrfs" ] || [ "${driver}" = "dir" ] || [ "${driver}" = "zfs" ]; then
+    # Client doesn't show totals for unbounded containers
+    [ -z "$(lxc info c1 | awk '/Disk total:/ {found=1} found && /root:/ {print $2; exit}')" ]
+    [ -z "$(lxc info c2 | awk '/Disk total:/ {found=1} found && /root:/ {print $2; exit}')" ]
+    # Check volume also has empty 'volatile.rootfs.size' and volume state also don't show total
+    [ -z "$(lxc storage volume get container/c1 volatile.rootfs.size)" ]
+    ! lxc storage volume info container/c1 | grep "Total:" || false
+    [ -z "$(lxc storage volume get container/c2 volatile.rootfs.size)" ]
+    ! lxc storage volume info container/c2 | grep "Total:" || false
+  fi
+
+  # Block-based drivers
+  if [ "${driver}" = "lvm" ] || [ "${driver}" = "ceph" ] || [ "${driver}" = "zfs-block" ]; then
+    [ "$(lxc info c1 | awk '/Disk total:/ {found=1} found && /root:/ {print $2; exit}')" = "10GiB" ]
+    [ "$(lxc info c2 | awk '/Disk total:/ {found=1} found && /root:/ {print $2; exit}')" = "7GiB" ]
+
+    # Check volume 'volatile.rootfs.size' and volume state also show correct total size
+    [ "$(lxc storage volume get container/c1 volatile.rootfs.size)" = "10GiB" ]
+    [ "$(lxc storage volume info container/c1 | grep "Total:")"  = "10GiB" ]
+    [ "$(lxc storage volume get container/c2 volatile.rootfs.size)" = "7GiB" ]
+    [ "$(lxc storage volume info container/c2 | grep "Total:")"  = "7GiB" ]
+  fi
+
+  # Shrink c1 and grow c2
+  # Device 'size' should override default size and 'volume.size'
+  lxc start c1
+  lxc start c2
+  ! lxc storage volume set container/c1 size="7GiB" || false
+  lxc config device set c1 root size="7GiB"
+  lxc config device set c2 root size="10GiB"
+
+  # Non block-based drivers.
+  if [ "${driver}" = "btrfs" ] || [ "${driver}" = "dir" ] || [ "${driver}" = "zfs" ]; then
+    # Device 'size' always applies right away
+    [ "$(lxc info c1 | awk '/Disk total:/ {found=1} found && /root:/ {print $2; exit}')" = "7GiB" ]
+    [ "$(lxc info c2 | awk '/Disk total:/ {found=1} found && /root:/ {print $2; exit}')" = "10GiB" ]
+
+    # Also visible from volume 'volatile.rootfs.size' config and volume state
+    [ "$(lxc storage volume get container/c1 volatile.rootfs.size)" = "7GiB" ]
+    [ "$(lxc storage volume info container/c1 | awk '/Disk total:/ {found=1} found && /root:/ {print $2; exit}')"  = "7GiB" ]
+  fi
+
+  # Block-based drivers
+  if [ "${driver}" = "lvm" ] || [ "${driver}" = "ceph" ] || [ "${driver}" = "zfs-block" ]; then
+    [ "$(lxc info c2 | awk '/Disk total:/ {found=1} found && /root:/ {print $2; exit}')" = "7GiB" ]
+    # Shrinking does not apply while instance is running
+    [ "$(lxc info c1 | awk '/Disk total:/ {found=1} found && /root:/ {print $2; exit}')" = "10GiB" ]
+    lxc stop -f c1
+    # While instance is stopped, show new instance root disk size even though
+    # The volume itself only gets resized on instance start
+    [ "$(lxc info c1 | awk '/Disk total:/ {found=1} found && /root:/ {print $2; exit}')" = "7GiB" ]
+    [ "$(lxc storage volume get container/c1 volatile.rootfs.size)" = "7GiB" ]
+    lxc start c1
+    [ "$(lxc info c1 | awk '/Disk total:/ {found=1} found && /root:/ {print $2; exit}')" = "7GiB" ]
+  fi
+
+  lxc delete -f c1 c2
+}