Skip to content

Commit

Permalink
Enhance files query performance
Browse files Browse the repository at this point in the history
  • Loading branch information
@ismartcoding
ismartcoding committed Jun 18, 2024
1 parent 6e4d37a commit bfeaf00
Show file tree
Hide file tree
Showing 8 changed files with 116 additions and 24 deletions.
23 changes: 23 additions & 0 deletions app/src/main/java/com/ismartcoding/plain/features/file/FileSortBy.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,4 +59,27 @@ enum class FileSortBy {
}
}
}

fun toFileSortBy(): SortBy {
return when (this) {
NAME_ASC -> {
SortBy(MediaStore.MediaColumns.DISPLAY_NAME, SortDirection.ASC)
}
NAME_DESC -> {
SortBy(MediaStore.MediaColumns.DISPLAY_NAME, SortDirection.DESC)
}
DATE_ASC -> {
SortBy(MediaStore.MediaColumns.DATE_MODIFIED, SortDirection.ASC)
}
DATE_DESC -> {
SortBy(MediaStore.MediaColumns.DATE_MODIFIED, SortDirection.DESC)
}
SIZE_ASC -> {
SortBy(MediaStore.MediaColumns.SIZE, SortDirection.ASC)
}
SIZE_DESC -> {
SortBy(MediaStore.MediaColumns.SIZE, SortDirection.DESC)
}
}
}
}
6 changes: 0 additions & 6 deletions app/src/main/java/com/ismartcoding/plain/features/file/FileSystemHelper.kt
View file
Original file line number Diff line number Diff line change
@@ -1,18 +1,12 @@
package com.ismartcoding.plain.features.file

import android.content.ContentResolver
import android.content.Context
import android.os.Environment
import android.os.StatFs
import android.os.storage.StorageManager
import android.provider.MediaStore
import android.text.TextUtils
import androidx.core.os.bundleOf
import com.ismartcoding.lib.extensions.forEach
import com.ismartcoding.lib.extensions.getDirectChildrenCount
import com.ismartcoding.lib.extensions.getLongValue
import com.ismartcoding.lib.extensions.getStringValue
import com.ismartcoding.lib.extensions.getTimeValue
import com.ismartcoding.lib.isRPlus
import com.ismartcoding.plain.R
import com.ismartcoding.plain.extensions.sorted
Expand Down
66 changes: 57 additions & 9 deletions app/src/main/java/com/ismartcoding/plain/features/media/FileMediaStoreHelper.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,14 @@ import android.net.Uri
import android.provider.MediaStore
import com.ismartcoding.lib.content.ContentWhere
import com.ismartcoding.lib.extensions.*
import com.ismartcoding.plain.MainApp
import com.ismartcoding.plain.enums.FileType
import com.ismartcoding.plain.extensions.sorted
import com.ismartcoding.plain.features.file.DFile
import com.ismartcoding.plain.features.file.FileSortBy
import com.ismartcoding.plain.features.file.FileSystemHelper
import com.ismartcoding.plain.helpers.QueryHelper
import java.io.File
import java.util.Locale

object FileMediaStoreHelper : BaseContentHelper() {
Expand Down Expand Up @@ -55,13 +58,30 @@ object FileMediaStoreHelper : BaseContentHelper() {
override suspend fun buildWhereAsync(query: String): ContentWhere {
val where = ContentWhere()
if (query.isNotEmpty()) {
var showHidden = false
QueryHelper.parseAsync(query).forEach {
if (it.name == "text") {
where.add("${MediaStore.Files.FileColumns.DISPLAY_NAME} LIKE ?", "%${it.value}%")
} else if (it.name == "ids") {
where.addIn(MediaStore.Files.FileColumns._ID, it.value.split(","))
when (it.name) {
"text" -> {
where.add("${MediaStore.Files.FileColumns.DISPLAY_NAME} LIKE ?", "%${it.value}%")
}
"parent" -> {
where.add("${MediaStore.Files.FileColumns.PARENT} = ?", getIdByPathAsync(MainApp.instance, it.value) ?: "-1")
}
"type" -> {
where.add("${MediaStore.Files.FileColumns.MIME_TYPE} = ?", it.value)
}
"show_hidden" -> {
showHidden = it.value.toBoolean()
}
"ids" -> {
where.addIn(MediaStore.Files.FileColumns._ID, it.value.split(","))
}
}
}

if (!showHidden) {
where.addNotStartsWith(MediaStore.Files.FileColumns.DISPLAY_NAME, ".")
}
}
return where
}
Expand All @@ -73,12 +93,17 @@ object FileMediaStoreHelper : BaseContentHelper() {
offset: Int,
sortBy: FileSortBy,
): List<DFile> {
return context.contentResolver.getPagingCursor(
val items = context.contentResolver.getPagingCursor(
uriExternal, getProjection(), buildWhereAsync(query),
limit, offset, sortBy.toSortBy()
limit, offset, sortBy.toFileSortBy()
)?.map { cursor, cache ->
cursorToFile(cursor, cache)
} ?: emptyList()
val folderIds = items.filter { it.isDir }.map { it.mediaId }
val counts = getChildrenCountAsync(context, folderIds)
return items.map {
it.copy(children = counts[it.mediaId] ?: 0)
}
}

fun getByIdAsync(context: Context, id: String): DFile? {
Expand All @@ -88,6 +113,29 @@ object FileMediaStoreHelper : BaseContentHelper() {
}
}

private fun getChildrenCountAsync(context: Context, folderIds: List<String>): Map<String, Int> {
val counts = mutableMapOf<String, Int>()
if (folderIds.isEmpty()) {
return counts
}
val where = ContentWhere()
where.addIn(MediaStore.Files.FileColumns.PARENT, folderIds)
context.contentResolver
.queryCursor(uriExternal, arrayOf(MediaStore.Files.FileColumns.PARENT), where.toSelection(), where.args.toTypedArray())?.forEach { cursor, cache ->
val parentId = cursor.getStringValue(MediaStore.Files.FileColumns.PARENT, cache)
counts[parentId] = counts.getOrDefault(parentId, 0) + 1
}

return counts
}

fun getIdByPathAsync(context: Context, path: String): String? {
return context.contentResolver
.queryCursor(uriExternal, arrayOf(MediaStore.Files.FileColumns._ID), "${MediaStore.Files.FileColumns.DATA} = ?", arrayOf(path))?.find { cursor, cache ->
cursor.getStringValue(MediaStore.Files.FileColumns._ID, cache)
}
}

fun getAllByFileTypeAsync(
context: Context, volumeName: String,
fileType: FileType, sortBy: FileSortBy,
Expand Down Expand Up @@ -162,7 +210,7 @@ object FileMediaStoreHelper : BaseContentHelper() {
suspend fun getRecentFilesAsync(context: Context, query: String): List<DFile> {
return context.contentResolver.getPagingCursor(
uriExternal, getProjection(), buildWhereAsync(query),
100, 0, FileSortBy.DATE_DESC.toSortBy()
100, 0, FileSortBy.DATE_DESC.toFileSortBy()
)?.map { cursor, cache ->
cursorToFile(cursor, cache)
}?.filter { !it.isDir }?.take(50) ?: emptyList()
Expand All @@ -175,7 +223,7 @@ object FileMediaStoreHelper : BaseContentHelper() {
val path = cursor.getStringValue(MediaStore.Files.FileColumns.DATA, cache)
val createdAt = cursor.getTimeSecondsValue(MediaStore.Files.FileColumns.DATE_ADDED, cache)
val updatedAt = cursor.getTimeSecondsValue(MediaStore.Files.FileColumns.DATE_MODIFIED, cache)

val mimeType = cursor.getString(cursor.getColumnIndexOrThrow(MediaStore.Files.FileColumns.MIME_TYPE))
val mediaType = cursor.getIntValue(MediaStore.Files.FileColumns.MEDIA_TYPE, cache)
return DFile(
title,
Expand All @@ -184,7 +232,7 @@ object FileMediaStoreHelper : BaseContentHelper() {
createdAt,
updatedAt,
size,
mediaType == MediaStore.Files.FileColumns.MEDIA_TYPE_NONE,
mediaType == MediaStore.Files.FileColumns.MEDIA_TYPE_NONE && mimeType == null && size == 0L,
0,
id
)
Expand Down
4 changes: 4 additions & 0 deletions app/src/main/java/com/ismartcoding/plain/web/HttpModule.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -401,6 +401,10 @@ object HttpModule {
call.respond(HttpStatusCode.NotFound)
return@get
}
if (file.isDirectory) {
call.respond(HttpStatusCode.BadRequest)
return@get
}

val fileName = (q["name"] ?: file.name).urlEncode().replace("+", "%20")
if (q["dl"] == "1") {
Expand Down
20 changes: 18 additions & 2 deletions app/src/main/java/com/ismartcoding/plain/web/SXGraphQL.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ import com.ismartcoding.plain.helpers.AppHelper
import com.ismartcoding.plain.helpers.DeviceInfoHelper
import com.ismartcoding.plain.helpers.ExchangeHelper
import com.ismartcoding.plain.helpers.FileHelper
import com.ismartcoding.plain.helpers.QueryHelper
import com.ismartcoding.plain.helpers.TempHelper
import com.ismartcoding.plain.preference.DeveloperModePreference
import com.ismartcoding.plain.preference.DeviceNamePreference
Expand Down Expand Up @@ -510,14 +511,29 @@ class SXGraphQL(val schema: Schema) {
FileMediaStoreHelper.getRecentFilesAsync(context, "").map { it.toModel() }
}
}
query("files") {
query("files2") {
resolver { dir: String, showHidden: Boolean, sortBy: FileSortBy ->
val context = MainApp.instance
Permission.WRITE_EXTERNAL_STORAGE.checkAsync(context)
val files = FileSystemHelper.getFilesList(dir, showHidden, sortBy).map { it.toModel() }
Files(dir, files)
}
}
query("files") {
resolver { offset: Int, limit: Int, query: String, sortBy: FileSortBy ->
val context = MainApp.instance
Permission.WRITE_EXTERNAL_STORAGE.checkAsync(context)
val filterFields = QueryHelper.parseAsync(query)
val appFolder = context.getExternalFilesDir(null)?.path ?: ""
val parent = filterFields.find { it.name == "parent" }
if (parent?.value?.startsWith(appFolder) == true) {
val showHidden = filterFields.find { it.name == "show_hidden" }?.value?.toBoolean() ?: false
FileSystemHelper.getFilesList(parent.value, showHidden, sortBy).map { it.toModel() }
} else {
FileMediaStoreHelper.searchAsync(MainApp.instance, query, limit, offset, sortBy).map { it.toModel() }
}
}
}
query("fileInfo") {
resolver { id: ID, path: String ->
val context = MainApp.instance
Expand Down Expand Up @@ -1079,7 +1095,7 @@ class SXGraphQL(val schema: Schema) {
val allTags = TagHelper.getAll(DataType.NOTE)
val map = TagHelper.getTagRelationsByKeys(keys.toSet(), DataType.NOTE).groupBy { it.key }
jsonEncode(items.map {
val tagIds = map[it.id]?.map { t -> t.tagId } ?: emptyList()
val tagIds = map[it.id]?.map { t -> t.tagId } ?: emptyList()
it.toExportModel(if (tagIds.isNotEmpty()) allTags.filter { tagIds.contains(it.id) }.map { t -> t.toModel() } else emptyList())
})
}
Expand Down
3 changes: 2 additions & 1 deletion app/src/main/java/com/ismartcoding/plain/web/models/File.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,12 @@ data class File(
val updatedAt: Instant,
val size: Long,
val isDir: Boolean,
val children: Int,
val mediaId: String
)

fun DFile.toModel(): File {
return File(name, path, permission, createdAt, updatedAt, size, isDir, mediaId)
return File(name, path, permission, createdAt, updatedAt, size, isDir, children, mediaId)
}

data class Files(val dir: String, val items: List<File>)
7 changes: 7 additions & 0 deletions lib/src/main/java/com/ismartcoding/lib/content/ContentWhere.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,13 @@ data class ContentWhere(private val selections: MutableList<String> = mutableLis
add("$field LIKE '%' || ? || '%'", value)
}

fun addNotStartsWith(
field: String,
value: String,
) {
add("$field NOT LIKE ? || '%'", value)
}

fun addLikes(
fields: List<String>,
values: List<String>,
Expand Down
11 changes: 5 additions & 6 deletions lib/src/main/java/com/ismartcoding/lib/extensions/File.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,11 @@ import java.io.ByteArrayOutputStream
import java.io.File

fun File.getDirectChildrenCount(countHiddenItems: Boolean): Int {
return listFiles()?.filter {
if (countHiddenItems) {
true
} else {
!it.name.startsWith('.')
}
if (countHiddenItems) {
return list()?.size ?: 0
}
return list()?.filter {
!it.startsWith('.')
}?.size ?: 0
}

Expand Down

0 comments on commit bfeaf00

Please sign in to comment.