Merge pull request #2796 from spacedriveapp/eng-294-volume-awareness

[ENG-294] Volume awareness
spacedriveapp · Dec 9, 2024 · 64ff5b3 · 64ff5b3
2 parents 7a7935b + d160255
commit 64ff5b3
Show file tree

Hide file tree

Showing 26 changed files with 3,379 additions and 952 deletions.
diff --git a/.cspell/project_words.txt b/.cspell/project_words.txt
@@ -1,6 +1,7 @@
 akar
 allred
 alluxio
+APFS
 augusto
 automount
 benja
@@ -17,11 +18,13 @@ elon
 encryptor
 Exif
 Flac
+fsevent
 graps
 haden
 haoyuan
 haris
 Iconoir
+inotify
 josephjacks
 justinhoffman
 Keychain
@@ -32,15 +35,18 @@ lesterlee
 Loas
 lütke
 marietti
+mbps
 mehrzad
 Mjpeg
 Mmap
+mpscrr
 mytton
 narkhede
 naveen
 neha
 noco
 Normalised
+NTFS
 OSSC
 poonen
 rauch
@@ -52,6 +58,7 @@ rspc
 rspcws
 rywalker
 sanjay
+sdvol
 sharma
 skippable
 spacedrive
@@ -64,8 +71,10 @@ thumbstrips
 tobiaslutke
 tokio
 typecheck
+unwatch
 uuid
 vdfs
 vijay
+winapi
 zacharysmith
 zxcvbn
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/core/Cargo.toml b/core/Cargo.toml
@@ -87,6 +87,8 @@ base91           = "0.1.0"
 ctor             = "0.2.8"
 directories      = "5.0"
 flate2           = "1.0"
+fsevent          = "2.1.2"
+hex              = "0.4.3"
 hostname         = "0.4.0"
 http-body        = "1.0"
 http-range       = "0.1.5"

diff --git a/core/crates/sync/src/backfill.rs b/core/crates/sync/src/backfill.rs
@@ -1,7 +1,7 @@
 use sd_prisma::{
 	prisma::{
 		crdt_operation, device, exif_data, file_path, label, label_on_object, location, object,
-		storage_statistics, tag, tag_on_object, PrismaClient, SortOrder,
+		tag, tag_on_object, volume, PrismaClient, SortOrder,
 	},
 	prisma_sync,
 };
@@ -47,7 +47,7 @@ pub async fn backfill_operations(sync: &SyncManager) -> Result<(), Error> {
 			backfill_device(&db, sync, local_device).await?;
 
 			(
-				backfill_storage_statistics(&db, sync, local_device_id),
+				backfill_volumes(&db, sync, local_device_id),
 				paginate_tags(&db, sync),
 				paginate_locations(&db, sync, local_device_id),
 				paginate_objects(&db, sync, local_device_id),
@@ -102,15 +102,15 @@ async fn backfill_device(
 }
 
 #[instrument(skip(db, sync), err)]
-async fn backfill_storage_statistics(
+async fn backfill_volumes(
 	db: &PrismaClient,
 	sync: &SyncManager,
 	device_id: device::id::Type,
 ) -> Result<(), Error> {
-	let Some(stats) = db
-		.storage_statistics()
-		.find_first(vec![storage_statistics::device_id::equals(Some(device_id))])
-		.include(storage_statistics::include!({device: select { pub_id }}))
+	let Some(volume) = db
+		.volume()
+		.find_first(vec![volume::device_id::equals(Some(device_id))])
+		.include(volume::include!({device: select { pub_id }}))
 		.exec()
 		.await?
 	else {
@@ -120,24 +120,29 @@ async fn backfill_storage_statistics(
 
 	db.crdt_operation()
 		.create_many(vec![crdt_op_unchecked_db(&sync.shared_create(
-			prisma_sync::storage_statistics::SyncId {
-				pub_id: stats.pub_id,
+			prisma_sync::volume::SyncId {
+				pub_id: volume.pub_id,
 			},
 			chain_optional_iter(
 				[
-					sync_entry!(stats.total_capacity, storage_statistics::total_capacity),
-					sync_entry!(
-						stats.available_capacity,
-						storage_statistics::available_capacity
-					),
+					sync_entry!(volume.name, volume::name),
+					sync_entry!(volume.mount_type, volume::mount_type),
+					sync_entry!(volume.mount_point, volume::mount_point),
+					sync_entry!(volume.is_mounted, volume::is_mounted),
+					sync_entry!(volume.disk_type, volume::disk_type),
+					sync_entry!(volume.file_system, volume::file_system),
+					sync_entry!(volume.read_only, volume::read_only),
+					sync_entry!(volume.error_status, volume::error_status),
+					sync_entry!(volume.total_bytes_capacity, volume::total_bytes_capacity),
+					sync_entry!(volume.total_bytes_available, volume::total_bytes_available),
 				],
 				[option_sync_entry!(
-					stats.device.map(|device| {
+					volume.device.map(|device| {
 						prisma_sync::device::SyncId {
 							pub_id: device.pub_id,
 						}
 					}),
-					storage_statistics::device
+					volume::device
 				)],
 			),
 		))?])

diff --git a/core/crates/sync/src/manager.rs b/core/crates/sync/src/manager.rs
@@ -372,7 +372,7 @@ impl Manager {
 		total_count += self.ingest_by_model(prisma_sync::device::MODEL_ID).await?;
 
 		total_count += [
-			self.ingest_by_model(prisma_sync::storage_statistics::MODEL_ID),
+			self.ingest_by_model(prisma_sync::volume::MODEL_ID),
 			self.ingest_by_model(prisma_sync::tag::MODEL_ID),
 			self.ingest_by_model(prisma_sync::location::MODEL_ID),
 			self.ingest_by_model(prisma_sync::object::MODEL_ID),

diff --git a/core/prisma/schema.prisma b/core/prisma/schema.prisma
@@ -73,13 +73,13 @@ model Device {
   date_created DateTime? // Not actually NULLABLE, but we have to comply with current sync implementation BS
   date_deleted DateTime?
 
-  StorageStatistics StorageStatistics?
-  Location          Location[]
-  FilePath          FilePath[]
-  Object            Object[]
-  ExifData          ExifData[]
-  TagOnObject       TagOnObject[]
-  LabelOnObject     LabelOnObject[]
+  Location      Location[]
+  FilePath      FilePath[]
+  Object        Object[]
+  ExifData      ExifData[]
+  TagOnObject   TagOnObject[]
+  LabelOnObject LabelOnObject[]
+  Volume        Volume[]
 
   @@map("device")
 }
@@ -138,19 +138,29 @@ model ObjectKindStatistics {
   @@map("object_kind_statistics")
 }
 
-/// @local
+/// @shared(id: pub_id, modelId: 13)
 model Volume {
-  id                    Int      @id @default(autoincrement())
-  name                  String
-  mount_point           String
-  total_bytes_capacity  String   @default("0")
-  total_bytes_available String   @default("0")
+  id     Int   @id @default(autoincrement())
+  pub_id Bytes @unique
+
+  name                  String?
+  mount_point           String?
+  mount_type            String?
+  total_bytes_capacity  String?
+  total_bytes_available String?
   disk_type             String?
-  filesystem            String?
-  is_system             Boolean  @default(false)
-  date_modified         DateTime @default(now())
+  file_system           String?
+  date_modified         DateTime?
+  is_mounted            Boolean?
+  read_speed_mbps       BigInt?
+  write_speed_mbps      BigInt?
+  read_only             Boolean?
+  error_status          String?
+
+  device_id Int?
+  device    Device? @relation(fields: [device_id], references: [id], onDelete: Cascade)
 
-  @@unique([mount_point, name])
+  @@unique([device_id, mount_point, name, total_bytes_capacity, file_system])
   @@map("volume")
 }
 
@@ -599,21 +609,6 @@ model ObjectInSpace {
   @@map("object_in_space")
 }
 
-//// StorageStatistics ////
-/// @shared(id: pub_id, modelId: 11)
-model StorageStatistics {
-  id     Int   @id @default(autoincrement())
-  pub_id Bytes @unique
-
-  total_capacity     BigInt @default(0)
-  available_capacity BigInt @default(0)
-
-  device_id Int?    @unique
-  device    Device? @relation(fields: [device_id], references: [id], onDelete: Cascade)
-
-  @@map("storage_statistics")
-}
-
 //// Job ////
 
 model Job {

diff --git a/core/src/api/cloud/mod.rs b/core/src/api/cloud/mod.rs
@@ -1,4 +1,5 @@
 use crate::{
+	invalidate_query,
 	library::LibraryManagerError,
 	node::{config::NodeConfig, HardwareModel},
 	Node,

diff --git a/core/src/api/libraries.rs b/core/src/api/libraries.rs
@@ -1,7 +1,7 @@
 use crate::{
 	api::CoreEvent,
 	invalidate_query,
-	library::{update_library_statistics, Library, LibraryConfig, LibraryName},
+	library::{Library, LibraryConfig, LibraryName},
 	location::{scan_location, LocationCreateArgs, ScanState},
 	util::MaybeUndefined,
 	Node,
@@ -644,13 +644,13 @@ async fn update_statistics_loop(
 	while let Some(msg) = msg_stream.next().await {
 		match msg {
 			Message::Tick => {
-				if last_received_at.elapsed() < FIVE_MINUTES {
-					if let Err(e) = update_library_statistics(&node, &library).await {
-						error!(?e, "Failed to update library statistics;");
-					} else {
-						invalidate_query!(&library, "library.statistics");
-					}
-				}
+				// if last_received_at.elapsed() < FIVE_MINUTES {
+				// 	if let Err(e) = update_library_statistics(&node, &library).await {
+				// 		error!(?e, "Failed to update library statistics;");
+				// 	} else {
+				// 		invalidate_query!(&library, "library.statistics");
+				// 	}
+				// }
 			}
 			Message::Requested(instant) => {
 				if instant - last_received_at > TWO_MINUTES {