[ENG-1159] Report progress when copying files (#2538)

* Copy ephemeral files by chunks

* Improving buffered writer

* Report progress

* Copy and emit progress as two separated tasks

* Simplify for-if-return with `Iterator::any`

* Docs and use structured tracing

* Simplify code using paths

* wip

* wip

* wip

* wip

* Add report of messages while copying files

* Add info field to job report

* Group paths from OldFileCopierJobStep into a single struct

* Improve progress information

* Remove the need for synchronization

* Error handling for copy

* Clean up frontend

* Make watcher only consider new files

* Fix concurrent renaming of multiple files that have the same name

* Add documentation for file strategist

* Remove editions to ephemeral file copies

* Remove experimental/nightly features from Cargo.toml

* Fix no-case-declarations

* Remove dead comments

* Format code

* Use "Duplicate" instead of "Copy" in the frontend messages

* Remove inline always to make clippy happy

* icons for deleter and copier

* Fix JobManager for copy jobs
 - Fix some types definitions

* Fix Job icon choosing logic
 - Fix Copier job showing two progress bars

* Log which files are not formatted in the Type and style check CI

* fmt

* Forgot an import

* autoformat

---------

Co-authored-by: ameer2468 <[email protected]>
Co-authored-by: Vítor Vasconcellos <[email protected]>

.github/workflows/ci.yml

@@ -47,12 +47,15 @@ jobs:
 
           _files="$(git diff --name-only --cached)"
           if [ -n "$_files" ]; then
-            echo "Some files are not correctly formatted. Please run 'pnpm autoformat' and commit the changes." >&2
+            printf "\e[31mThe following files are not correctly formatted:\e[0m\n" >&2
 
             while IFS= read -r _file || [ -n "$_file" ]; do
               echo "::error file=${_file},title=Incorrectly formatted file::Please run 'pnpm autoformat' and commit the changes."
+              echo " - ${_file}"
             done < <(printf '%s' "$_files")
 
+            printf "\e[31mPlease run 'pnpm autoformat' and commit the changes.\e[0m\n" >&2
+
             exit 1
           fi
 

apps/landing/src/app/page.tsx

@@ -64,10 +64,7 @@ export default async function Page() {
 						.join(' ')}
 				/>
 				<div className="pb-6 xs:pb-24">
-					<div
-						className="xl2:relative z-30 flex h-[255px] w-full px-6
-				 sm:h-[428px] md:mt-[75px] md:h-[428px] lg:h-auto"
-					>
+					<div className="xl2:relative z-30 flex h-[255px] w-full px-6 sm:h-[428px] md:mt-[75px] md:h-[428px] lg:h-auto">
 						<Image
 							loading="eager"
 							className="absolute-horizontal-center animation-delay-2 top-[380px] fade-in xs:top-[180px] md:top-[130px]"

core/crates/file-path-helper/src/isolated_file_path_data.rs

@@ -577,46 +577,35 @@ fn assemble_relative_path(
 	extension: &str,
 	is_dir: bool,
 ) -> String {
-	match (is_dir, extension) {
-		(false, extension) if !extension.is_empty() => {
-			format!("{}{}.{}", &materialized_path[1..], name, extension)
-		}
-		(_, _) => format!("{}{}", &materialized_path[1..], name),
+	if !is_dir && !extension.is_empty() {
+		format!("{}{}.{}", &materialized_path[1..], name, extension)
+	} else {
+		format!("{}{}", &materialized_path[1..], name)
 	}
 }
 
-#[allow(clippy::missing_panics_doc)] // Don't actually panic as we check before `expect`
 pub fn join_location_relative_path(
 	location_path: impl AsRef<Path>,
 	relative_path: impl AsRef<Path>,
 ) -> PathBuf {
 	let relative_path = relative_path.as_ref();
+	let relative_path = relative_path
+		.strip_prefix(MAIN_SEPARATOR_STR)
+		.unwrap_or(relative_path);
 
-	location_path
-		.as_ref()
-		.join(if relative_path.starts_with(MAIN_SEPARATOR_STR) {
-			relative_path
-				.strip_prefix(MAIN_SEPARATOR_STR)
-				.expect("just checked")
-		} else {
-			relative_path
-		})
+	location_path.as_ref().join(relative_path)
 }
 
-#[allow(clippy::missing_panics_doc)] // Don't actually panic as we check before `expect`
 pub fn push_location_relative_path(
 	mut location_path: PathBuf,
 	relative_path: impl AsRef<Path>,
 ) -> PathBuf {
 	let relative_path = relative_path.as_ref();
 
-	location_path.push(if relative_path.starts_with(MAIN_SEPARATOR_STR) {
-		relative_path
-			.strip_prefix(MAIN_SEPARATOR_STR)
-			.expect("just checked")
-	} else {
-		relative_path
-	});
+	let relative_path = relative_path
+		.strip_prefix(MAIN_SEPARATOR_STR)
+		.unwrap_or(relative_path);
+	location_path.push(relative_path);
 
 	location_path
 }

core/crates/heavy-lifting/src/job_system/report.rs

@@ -144,6 +144,7 @@ pub struct Report {
 	pub status: Status,
 	pub task_count: i32,
 	pub completed_task_count: i32,
+	pub info: String,
 
 	pub phase: String,
 	pub message: String,
@@ -177,6 +178,7 @@ impl TryFrom<job::Data> for Report {
 			metadata,
 			parent_id,
 			task_count,
+			info,
 			completed_task_count,
 			date_estimated_completion,
 			date_created,
@@ -208,6 +210,7 @@ impl TryFrom<job::Data> for Report {
 				.expect("corrupted database"),
 			task_count: task_count.unwrap_or(0),
 			completed_task_count: completed_task_count.unwrap_or(0),
+			info: info.unwrap_or_default(),
 			phase: String::new(),
 			message: String::new(),
 			estimated_completion: date_estimated_completion.map_or_else(Utc::now, DateTime::into),
@@ -232,6 +235,7 @@ impl Report {
 			metadata: vec![],
 			parent_id: None,
 			completed_task_count: 0,
+			info: String::new(),
 			phase: String::new(),
 			message: String::new(),
 			estimated_completion: Utc::now(),
@@ -278,6 +282,7 @@ impl Report {
 						job::status::set(Some(self.status as i32)),
 						job::date_started::set(self.started_at.map(Into::into)),
 						job::task_count::set(Some(0)),
+						job::info::set(Some(self.info.clone())),
 						job::completed_task_count::set(Some(0)),
 					],
 					[self
@@ -307,6 +312,7 @@ impl Report {
 					)?)),
 					job::metadata::set(Some(serde_json::to_vec(&self.metadata)?)),
 					job::task_count::set(Some(self.task_count)),
+					job::info::set(Some(self.info.clone())),
 					job::completed_task_count::set(Some(self.completed_task_count)),
 					job::date_started::set(self.started_at.map(Into::into)),
 					job::date_completed::set(self.completed_at.map(Into::into)),
@@ -388,6 +394,7 @@ impl ReportBuilder {
 			metadata: self.metadata,
 			parent_id: self.parent_id,
 			completed_task_count: 0,
+			info: String::new(),
 			phase: String::new(),
 			message: String::new(),
 			estimated_completion: Utc::now(),

core/crates/prisma-helpers/src/lib.rs

@@ -235,6 +235,7 @@ job::select!(job_without_data {
 	date_started
 	date_completed
 	task_count
+	info
 	completed_task_count
 	date_estimated_completion
 });

core/prisma/migrations/20240710015044_add_info_colum_to_job/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "job" ADD COLUMN "info" TEXT;

core/prisma/schema.prisma

@@ -605,6 +605,7 @@ model Job {
 
   task_count                Int?
   completed_task_count      Int?
+  info                      String?
   date_estimated_completion DateTime? // Estimated timestamp that the job will be complete at
 
   date_created   DateTime?

core/src/context.rs

@@ -209,6 +209,7 @@ impl<OuterCtx: OuterContext + NodeContextExt> sd_core_heavy_lifting::JobContext<
 			library_id: library.id,
 			task_count: report.task_count,
 			completed_task_count: report.completed_task_count,
+			info: report.info.clone(),
 			estimated_completion: report.estimated_completion,
 			phase: report.phase.clone(),
 			message: report.message.clone(),

core/src/location/manager/watcher/utils.rs

@@ -339,47 +339,43 @@ async fn inner_create_file(
 		.exec()
 		.await?;
 
-	let (
-		object_ids::Data {
-			id: object_id,
-			pub_id: object_pub_id,
-		},
-		is_new,
-	) = if let Some(object) = existing_object {
-		(object, false)
+	let is_new_file = existing_object.is_none();
+
+	let object_ids::Data {
+		id: object_id,
+		pub_id: object_pub_id,
+	} = if let Some(object) = existing_object {
+		object
 	} else {
 		let pub_id: ObjectPubId = ObjectPubId::new();
 		let date_created: DateTime<FixedOffset> =
 			DateTime::<Local>::from(fs_metadata.created_or_now()).into();
 		let int_kind = kind as i32;
 
-		(
-			sync.write_ops(
-				db,
-				(
-					sync.shared_create(
-						prisma_sync::object::SyncId {
-							pub_id: pub_id.to_db(),
-						},
-						[
-							(object::date_created::NAME, msgpack!(date_created)),
-							(object::kind::NAME, msgpack!(int_kind)),
-						],
-					),
-					db.object()
-						.create(
-							pub_id.into(),
-							vec![
-								object::date_created::set(Some(date_created)),
-								object::kind::set(Some(int_kind)),
-							],
-						)
-						.select(object_ids::select()),
+		sync.write_ops(
+			db,
+			(
+				sync.shared_create(
+					prisma_sync::object::SyncId {
+						pub_id: pub_id.to_db(),
+					},
+					[
+						(object::date_created::NAME, msgpack!(date_created)),
+						(object::kind::NAME, msgpack!(int_kind)),
+					],
 				),
-			)
-			.await?,
-			true,
+				db.object()
+					.create(
+						pub_id.into(),
+						vec![
+							object::date_created::set(Some(date_created)),
+							object::kind::set(Some(int_kind)),
+						],
+					)
+					.select(object_ids::select()),
+			),
 		)
+		.await?
 	};
 
 	sync.write_op(
@@ -403,7 +399,7 @@ async fn inner_create_file(
 	.await?;
 
 	// If the file is a duplicate of an existing file, we don't need to generate thumbnails nor extract media data
-	if is_new
+	if is_new_file
 		&& !extension.is_empty()
 		&& matches!(
 			kind,

core/src/object/fs/mod.rs

@@ -8,6 +8,7 @@ use sd_utils::{
 	db::maybe_missing,
 	error::{FileIOError, NonUtf8PathError},
 };
+use tracing::trace;
 
 use std::{
 	ffi::OsStr,
@@ -43,12 +44,13 @@ pub enum ObjectType {
 	Directory,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct FileData {
 	pub file_path: file_path_with_object::Data,
 	pub full_path: PathBuf,
 }
 
+/// Get the [`FileData`] related to every `file_path_id`
 pub async fn get_many_files_datas(
 	db: &PrismaClient,
 	location_path: impl AsRef<Path>,
@@ -81,7 +83,7 @@ pub async fn get_many_files_datas(
 				})
 			})
 	})
-	.collect::<Result<Vec<_>, _>>()
+	.collect()
 }
 
 pub async fn get_file_data_from_isolated_file_path(
@@ -207,6 +209,7 @@ pub async fn find_available_filename_for_duplicate(
 				continue;
 			}
 			Err(e) if e.kind() == io::ErrorKind::NotFound => {
+				trace!(old_name=?target_path, new_name=?new_file_full_path_candidate, "duplicated file name, file renamed");
 				return Ok(new_file_full_path_candidate);
 			}
 			Err(e) => return Err(FileIOError::from((new_file_full_path_candidate, e)).into()),