Scan location roughly working, a ton of stuff to fix yet

apps/desktop/src/platform.ts

@@ -9,7 +9,6 @@ import { OperatingSystem, Platform } from '@sd/interface';
 
 import { commands, events } from './commands';
 import { env } from './env';
-import { createUpdater } from './updater';
 
 const customUriAuthToken = (window as any).__SD_CUSTOM_SERVER_AUTH_TOKEN__ as string | undefined;
 const customUriServerUrl = (window as any).__SD_CUSTOM_URI_SERVER__ as string[] | undefined;
@@ -46,9 +45,11 @@ function constructServerUrl(urlSuffix: string) {
 
 export const platform = {
 	platform: 'tauri',
-	getThumbnailUrlByThumbKey: (keyParts) =>
+	getThumbnailUrlByThumbKey: (thumbKey) =>
 		constructServerUrl(
-			`/thumbnail/${keyParts.map((i) => encodeURIComponent(i)).join('/')}.webp`
+			`/thumbnail/${encodeURIComponent(
+			thumbKey.base_directory_str
+		)}/${encodeURIComponent(thumbKey.shard_hex)}/${encodeURIComponent(thumbKey.cas_id)}.webp`
 		),
 	getFileUrl: (libraryId, locationLocalId, filePathId) =>
 		constructServerUrl(`/file/${libraryId}/${locationLocalId}/${filePathId}`),

apps/mobile/src/components/explorer/FileThumb.tsx

@@ -1,24 +1,25 @@
 import { DocumentDirectoryPath } from '@dr.pogodin/react-native-fs';
 import { getIcon } from '@sd/assets/util';
+import { Image } from 'expo-image';
+import { useEffect, useLayoutEffect, useMemo, useState, type PropsWithChildren } from 'react';
+import { View } from 'react-native';
 import {
 	getExplorerItemData,
 	getItemFilePath,
 	getItemLocation,
 	isDarkTheme,
+	ThumbKey,
 	type ExplorerItem
 } from '@sd/client';
-import { Image } from 'expo-image';
-import { useEffect, useLayoutEffect, useMemo, useState, type PropsWithChildren } from 'react';
-import { View } from 'react-native';
 import { flattenThumbnailKey, useExplorerStore } from '~/stores/explorerStore';
 
 import { tw } from '../../lib/tailwind';
 
 // NOTE: `file://` is required for Android to load local files!
-export const getThumbnailUrlByThumbKey = (thumbKey: string[]) => {
-	return `file://${DocumentDirectoryPath}/thumbnails/${thumbKey
-		.map((i) => encodeURIComponent(i))
-		.join('/')}.webp`;
+export const getThumbnailUrlByThumbKey = (thumbKey: ThumbKey) => {
+	return `file://${DocumentDirectoryPath}/thumbnails/${encodeURIComponent(
+		thumbKey.base_directory_str
+	)}/${encodeURIComponent(thumbKey.shard_hex)}/${encodeURIComponent(thumbKey.cas_id)}.webp`;
 };
 
 const FileThumbWrapper = ({ children, size = 1 }: PropsWithChildren<{ size: number }>) => (

apps/mobile/src/stores/explorerStore.ts

@@ -1,6 +1,6 @@
+import { ThumbKey, resetStore } from '@sd/client';
 import { proxy, useSnapshot } from 'valtio';
 import { proxySet } from 'valtio/utils';
-import { resetStore, type Ordering } from '@sd/client';
 
 export type ExplorerLayoutMode = 'list' | 'grid' | 'media';
 
@@ -26,14 +26,14 @@ const state = {
 	orderDirection: 'Asc' as 'Asc' | 'Desc'
 };
 
-export function flattenThumbnailKey(thumbKey: string[]) {
-	return thumbKey.join('/');
+export function flattenThumbnailKey(thumbKey: ThumbKey) {
+	return `${thumbKey.base_directory_str}/${thumbKey.shard_hex}/${thumbKey.cas_id}`;
 }
 
 const store = proxy({
 	...state,
 	reset: () => resetStore(store, state),
-	addNewThumbnail: (thumbKey: string[]) => {
+	addNewThumbnail: (thumbKey: ThumbKey) => {
 		store.newThumbnails.add(flattenThumbnailKey(thumbKey));
 	},
 	// this should be done when the explorer query is refreshed

apps/web/src/App.tsx

@@ -42,8 +42,10 @@ const spacedriveURL = (() => {
 
 const platform: Platform = {
 	platform: 'web',
-	getThumbnailUrlByThumbKey: (keyParts) =>
-		`${spacedriveURL}/thumbnail/${keyParts.map((i) => encodeURIComponent(i)).join('/')}.webp`,
+	getThumbnailUrlByThumbKey: (thumbKey) =>
+		`${spacedriveURL}/thumbnail/${encodeURIComponent(
+			thumbKey.base_directory_str
+		)}/${encodeURIComponent(thumbKey.shard_hex)}/${encodeURIComponent(thumbKey.cas_id)}.webp`,
 	getFileUrl: (libraryId, locationLocalId, filePathId) =>
 		`${spacedriveURL}/file/${encodeURIComponent(libraryId)}/${encodeURIComponent(
 			locationLocalId

core/crates/heavy-lifting/src/file_identifier/job.rs

@@ -35,7 +35,7 @@ use futures_concurrency::future::TryJoin;
 use serde::{Deserialize, Serialize};
 use serde_json::json;
 use tokio::time::Instant;
-use tracing::warn;
+use tracing::{trace, warn};
 
 use super::{
 	orphan_path_filters_deep, orphan_path_filters_shallow,
@@ -430,6 +430,8 @@ impl FileIdentifier {
 				.exec()
 				.await?;
 
+			trace!("Found {} orphan paths", orphan_paths.len());
+
 			if orphan_paths.is_empty() {
 				break;
 			}

core/crates/heavy-lifting/src/file_identifier/mod.rs

@@ -141,7 +141,7 @@ fn orphan_path_filters_shallow(
 			)),
 			file_path::size_in_bytes_bytes::not(Some(0u64.to_be_bytes().to_vec())),
 		],
-		[file_path_id.map(file_path::id::gte)],
+		[file_path_id.map(file_path::id::gt)],
 	)
 }
 
@@ -162,7 +162,7 @@ fn orphan_path_filters_deep(
 		],
 		[
 			// this is a workaround for the cursor not working properly
-			file_path_id.map(file_path::id::gte),
+			file_path_id.map(file_path::id::gt),
 			maybe_sub_iso_file_path.as_ref().map(|sub_iso_file_path| {
 				file_path::materialized_path::starts_with(
 					sub_iso_file_path

core/crates/heavy-lifting/src/file_identifier/shallow.rs

@@ -32,8 +32,8 @@ use super::{
 pub async fn shallow(
 	location: location::Data,
 	sub_path: impl AsRef<Path> + Send,
-	dispatcher: BaseTaskDispatcher<Error>,
-	ctx: impl OuterContext,
+	dispatcher: &BaseTaskDispatcher<Error>,
+	ctx: &impl OuterContext,
 ) -> Result<Vec<NonCriticalError>, Error> {
 	let sub_path = sub_path.as_ref();
 	let db = ctx.db();
@@ -87,7 +87,7 @@ pub async fn shallow(
 		orphans_count += orphan_paths.len() as u64;
 		last_orphan_file_path_id = Some(last_orphan.id);
 
-		pending_running_tasks.insert(CancelTaskOnDrop(
+		pending_running_tasks.insert(CancelTaskOnDrop::new(
 			dispatcher
 				.dispatch(ExtractFileMetadataTask::new(
 					Arc::clone(&location),
@@ -115,8 +115,8 @@ pub async fn shallow(
 
 async fn process_tasks(
 	pending_running_tasks: FutureGroup<CancelTaskOnDrop<Error>>,
-	dispatcher: BaseTaskDispatcher<Error>,
-	ctx: impl OuterContext,
+	dispatcher: &BaseTaskDispatcher<Error>,
+	ctx: &impl OuterContext,
 ) -> Result<Vec<NonCriticalError>, Error> {
 	let mut pending_running_tasks = pending_running_tasks.lend_mut();
 
@@ -141,7 +141,7 @@ async fn process_tasks(
 					errors.extend(more_errors);
 
 					if !identified_files.is_empty() {
-						pending_running_tasks.insert(CancelTaskOnDrop(
+						pending_running_tasks.insert(CancelTaskOnDrop::new(
 							dispatcher
 								.dispatch(ObjectProcessorTask::new(
 									identified_files,

core/crates/heavy-lifting/src/file_identifier/tasks/extract_file_metadata.rs

@@ -21,7 +21,7 @@ use futures::stream::{self, FuturesUnordered, StreamExt};
 use futures_concurrency::stream::Merge;
 use serde::{Deserialize, Serialize};
 use tokio::time::Instant;
-use tracing::error;
+use tracing::{error, trace};
 use uuid::Uuid;
 
 use super::IdentifiedFile;
@@ -95,6 +95,7 @@ impl Task<Error> for ExtractFileMetadataTask {
 		}
 
 		let Self {
+			id,
 			location,
 			location_path,
 			file_paths_by_id,
@@ -139,6 +140,8 @@ impl Task<Error> for ExtractFileMetadataTask {
 							.remove(&file_path_pub_id)
 							.expect("file_path must be here");
 
+						trace!("Processed file <file_path_pub_id='{file_path_pub_id}'>, {} files remaining", file_paths_by_id.len());
+
 						match res {
 							Ok(FileMetadata { cas_id, kind, .. }) => {
 								identified_files.insert(
@@ -168,6 +171,7 @@ impl Task<Error> for ExtractFileMetadataTask {
 					}
 
 					StreamMessage::Interrupt(kind) => {
+						trace!("Task received interrupt {kind:?}: <task_id={id}>");
 						*extract_metadata_time += start_time.elapsed();
 						return Ok(match kind {
 							InterruptionKind::Pause => ExecStatus::Paused,

core/crates/heavy-lifting/src/indexer/mod.rs

@@ -27,7 +27,7 @@ use std::{
 };
 
 use itertools::Itertools;
-use prisma_client_rust::{operator::or, Select};
+use prisma_client_rust::{operator::or, QueryError, Select};
 use rspc::ErrorCode;
 use serde::{Deserialize, Serialize};
 use specta::Type;
@@ -53,8 +53,8 @@ pub enum Error {
 	SubPath(#[from] sub_path::Error),
 
 	// Internal Errors
-	#[error("database Error: {0}")]
-	Database(#[from] prisma_client_rust::QueryError),
+	#[error("database error: {0}")]
+	Database(#[from] QueryError),
 	#[error(transparent)]
 	FileIO(#[from] FileIOError),
 	#[error(transparent)]
@@ -241,7 +241,7 @@ async fn remove_non_existing_file_paths(
 }
 
 #[allow(clippy::missing_panics_doc)] // Can't actually panic as we only deal with directories
-async fn reverse_update_directories_sizes(
+pub async fn reverse_update_directories_sizes(
 	base_path: impl AsRef<Path> + Send,
 	location_id: location::id::Type,
 	location_path: impl AsRef<Path> + Send,
@@ -345,7 +345,7 @@ async fn compute_sizes(
 	pub_id_by_ancestor_materialized_path: &mut HashMap<String, (file_path::pub_id::Type, u64)>,
 	db: &PrismaClient,
 	errors: &mut Vec<crate::NonCriticalError>,
-) -> Result<(), Error> {
+) -> Result<(), QueryError> {
 	db.file_path()
 		.find_many(vec![
 			file_path::location_id::equals(Some(location_id)),

core/crates/heavy-lifting/src/indexer/shallow.rs

@@ -33,8 +33,8 @@ use super::{
 pub async fn shallow(
 	location: location_with_indexer_rules::Data,
 	sub_path: impl AsRef<Path> + Send,
-	dispatcher: BaseTaskDispatcher<Error>,
-	ctx: impl OuterContext,
+	dispatcher: &BaseTaskDispatcher<Error>,
+	ctx: &impl OuterContext,
 ) -> Result<Vec<NonCriticalError>, Error> {
 	let sub_path = sub_path.as_ref();
 	let db = ctx.db();
@@ -64,7 +64,7 @@ pub async fn shallow(
 		Arc::clone(&location_path),
 		Arc::clone(&to_walk_path),
 		Arc::clone(db),
-		&dispatcher,
+		dispatcher,
 	)
 	.await?
 	else {
@@ -82,7 +82,7 @@ pub async fn shallow(
 		to_update,
 		Arc::clone(db),
 		Arc::clone(sync),
-		&dispatcher,
+		dispatcher,
 	)
 	.await?
 	else {
@@ -109,7 +109,7 @@ pub async fn shallow(
 			.await?;
 		}
 
-		update_location_size(location.id, db, &ctx).await?;
+		update_location_size(location.id, db, ctx).await?;
 	}
 
 	if indexed_count > 0 || removed_count > 0 {
@@ -222,7 +222,7 @@ async fn save_and_update(
 		.dispatch_many_boxed(save_and_update_tasks)
 		.await
 		.into_iter()
-		.map(CancelTaskOnDrop)
+		.map(CancelTaskOnDrop::new)
 		.collect::<Vec<_>>()
 		.try_join()
 		.await?