autodownload: add danbooru1 and 2 collections support

lib/api/preset/autodownload/multi/image_boards.dart

@@ -1,26 +1,50 @@
 part of preset;
 
-// e926/e621: same idea as danbooru 2, if you add .json at the end of the post url, it'll return the JSON of that post
-Future<VirtualPresetCollection> e621ToCollectionPreset(Uri uri) async {
+// danbooru2 and e621/e926 share the same api endpoints when it comes to pools
+// danbooru 2: if you add .json at the end of the post url, it'll return the JSON of that post
+Future<VirtualPresetCollection> _danbooru2LikeAPIs(Uri uri, Function(Uri uri, {HandleChunk handleChunk}) importer) async {
     final res = await http.get(Uri.parse("${[uri.origin, uri.path].join("/")}.json"));
     final json = jsonDecode(res.body);
 
     final presets = await multiImageDownloader(
         postsToIterate: List<int>.from(json["post_ids"]),
         getter: (post, handler) {
-            return e621ToPresetImage(Uri.parse([uri.origin, "posts", post].join("/")), handleChunk: handler,);
+            return importer(Uri.parse([uri.origin, "posts", post].join("/")), handleChunk: handler,);
         },
     );
 
     return VirtualPresetCollection(
         name: (json["name"] as String).replaceAll("_", " "),
-        pages: presets.map((preset) {
-            final pool = [uri.origin, uri.path].join("/");
-            if(preset.sources == null) preset.sources = [pool];
-            else {
-                preset.sources!.add(pool);
-            }
-            return preset;
-        },).toList()
+        pages: addSourceToAllPresets(presets, [uri.origin, uri.path].join("/"))
     );
+}
+Future<VirtualPresetCollection> danbooru2ToCollectionPreset(Uri uri) => _danbooru2LikeAPIs(uri, danbooru2ToPresetImage);
+Future<VirtualPresetCollection> e621ToCollectionPreset(Uri uri) => _danbooru2LikeAPIs(uri, e621ToPresetImage);
+
+// danbooru 1: /pool/show.xml?id=(id) returns all of the posts already parsed
+Future<VirtualPresetCollection> danbooru1ToCollectionPreset(Uri uri) async {
+    final id = uri.pathSegments.last;
+    final res = await http.get(Uri.parse("${uri.origin}/pool/show.json?id=$id"));
+    final json = jsonDecode(res.body);
+
+    final presets = await multiImageDownloader(
+        postsToIterate: List<Map<String, dynamic>>.from(json["posts"]),
+        getter: (post, handler) {
+            return anyURLToPresetImage(post["file_url"], handleChunk: handler);
+        },
+    );
+
+    return VirtualPresetCollection(
+        name: (json["name"] as String).replaceAll("_", " "),
+        pages: addSourceToAllPresets(presets, [uri.origin, uri.path].join("/"))
+    );
+}
+
+
+List<PresetImage> addSourceToAllPresets(List<PresetImage> presets, String source) {
+    return presets.map((preset) {
+        if(preset.sources == null) preset.sources = [source];
+        else preset.sources!.add(source);
+        return preset;
+    },).toList();
 }

lib/api/preset/autodownload/single/image_boards.dart

@@ -1,11 +1,11 @@
 part of "../../index.dart";
 
 // danbooru 2: if you add .json at the end of the post url, it'll return the JSON of that post
-Future<PresetImage> danbooru2ToPresetImage(Uri uri) async {
+Future<PresetImage> danbooru2ToPresetImage(Uri uri, {HandleChunk? handleChunk}) async {
     final res = await http.get(Uri.parse("${[uri.origin, uri.path].join("/")}.json"));
     final bodyRes = jsonDecode(res.body);
 
-    final downloadedFileInfo = await downloadFile(Uri.parse(bodyRes["file_url"]));
+    final downloadedFileInfo = await downloadFile(Uri.parse(bodyRes["file_url"]), handleChunk: handleChunk);
 
     return PresetImage(
         image: downloadedFileInfo,
@@ -21,12 +21,12 @@ Future<PresetImage> danbooru2ToPresetImage(Uri uri) async {
 
 // danbooru 1/moebooru: you can ask danbooru to do a search with the id: meta-tag. for obtaining the tag types, only webcrawling
 // as we cant obtain tag types in bulk, nor does post.json returns tag types in its response like danbooru 2
-Future<PresetImage> danbooru1ToPresetImage(Uri uri) async {
+Future<PresetImage> danbooru1ToPresetImage(Uri uri, {HandleChunk? handleChunk}) async {
     final postID = uri.pathSegments[2];
     final res = await http.get(Uri.parse([uri.origin, "post/index.json?tags=id:$postID"].join("/")));
     final post = jsonDecode(res.body)[0];
 
-    final downloadedFileInfo = await downloadFile(Uri.parse(post["file_url"]));
+    final downloadedFileInfo = await downloadFile(Uri.parse(post["file_url"]), handleChunk: handleChunk);
 
     final webpage = await http.get(uri);
     final document = parse(webpage.body);

lib/api/preset/autodownload/single/other.dart

@@ -1,8 +1,8 @@
 part of preset;
 
 // any: assumes that it returns an image
-Future<PresetImage> anyURLToPresetImage(String url) async {
-    final downloadedFileInfo = await downloadFile(Uri.parse(url));
+Future<PresetImage> anyURLToPresetImage(String url, {HandleChunk? handleChunk}) async {
+    final downloadedFileInfo = await downloadFile(Uri.parse(url), handleChunk: handleChunk);
 
     final mime = lookupMimeType(downloadedFileInfo.path)!;
 

lib/api/preset/collection.dart

@@ -58,6 +58,8 @@ class VirtualPresetCollection extends VirtualPreset {
 
         final VirtualPresetCollection preset = switch (website) {
             ServiceWebsites.e621 => await e621ToCollectionPreset(uri),
+            ServiceWebsites.danbooru2 => await danbooru2ToCollectionPreset(uri),
+            ServiceWebsites.danbooru1 => await danbooru1ToCollectionPreset(uri),
             // Websites.instagram => await instagramToSinglePreset(url),
             _ => throw "Could not identify"
         };

test/import/download/multiple_test.dart

@@ -26,7 +26,7 @@ const String kExternalStoragePath = 'externalStoragePath';
 
 void main() {
     void updateFunc() {
-        debugPrint("Progress: ${importListener.progress}");
+        debugPrint("Progress: ${importListener.progress.toStringAsFixed(3)}");
     }
 
     group("download", () {
@@ -41,8 +41,22 @@ void main() {
             final res = await VirtualPresetCollection.urlToPreset(Collections.e621.toString());
             expect(res, isA<VirtualPresetCollection>());
             expect(res.pages, isA<List<PresetImage>>());
-            // expect(await res.image!.length(), isNot(0));
         }, tags: "e621");
+        test("danbooru2", () async {
+            final res = await VirtualPresetCollection.urlToPreset(Collections.danbooru2.toString());
+            expect(res, isA<VirtualPresetCollection>());
+            expect(res.pages, isA<List<PresetImage>>());
+        }, tags: "danbooru2");
+        test("moebooru", () async {
+            final res = await VirtualPresetCollection.urlToPreset(Collections.moebooru.toString());
+            expect(res, isA<VirtualPresetCollection>());
+            expect(res.pages, isA<List<PresetImage>>());
+        }, tags: "moebooru");
+        test("danbooru1", () async {
+            final res = await VirtualPresetCollection.urlToPreset(Collections.danbooru1.toString());
+            expect(res, isA<VirtualPresetCollection>());
+            expect(res.pages, isA<List<PresetImage>>());
+        }, tags: "danbooru1");
     });
 }
 

test/import/download/single_test.dart

@@ -26,7 +26,7 @@ const String kExternalStoragePath = 'externalStoragePath';
 
 void main() {
     void updateFunc() {
-        debugPrint("Progress: ${importListener.progress}");
+        debugPrint("Progress: ${importListener.progress.toStringAsFixed(3)}");
     }
 
     group("download", () {

test/shared.dart

@@ -13,11 +13,11 @@ class Posts {
 }
 
 class Collections {
-    static Uri danbooru1 = Uri.parse("http://behoimi.org/pool/show/47");
-    static Uri moebooru = Uri.parse("https://konachan.com/pool/show/542");
-    static Uri danbooru2 = Uri.parse("https://danbooru.donmai.us/pools/14957");
+    static Uri danbooru1 = Uri.parse("http://behoimi.org/pool/show/37");
+    static Uri moebooru = Uri.parse("https://konachan.com/pool/show/394");
+    static Uri danbooru2 = Uri.parse("https://danbooru.donmai.us/pools/23046");
     static Uri e621 = Uri.parse("https://e926.net/pools/42095");
     // static Uri e621 = Uri.parse("https://e926.net/pools/41798"); // this pool has a deleted post, can be useful
-    static Uri gelbooru025 = Uri.parse("https://gelbooru.com/index.php?page=pool&s=show&id=64318");
+    static Uri gelbooru025 = Uri.parse("https://gelbooru.com/index.php?page=pool&s=show&id=64104");
     static Uri gelbooru020 = Uri.parse("https://safebooru.org/index.php?page=pool&s=show&id=694");
 }