download: inform custom user agent

Signed-off-by: abUwUser <[email protected]>
resucutie · Aug 17, 2024 · 81f870f · 81f870f
1 parent 6924a78
commit 81f870f
Show file tree

Hide file tree

Showing 9 changed files with 66 additions and 24 deletions.
diff --git a/lib/api/preset/autodownload/multi/image_boards.dart b/lib/api/preset/autodownload/multi/image_boards.dart
@@ -3,7 +3,7 @@ part of preset;
 // danbooru2 and e621/e926 share the same api endpoints when it comes to pools
 // danbooru 2: if you add .json at the end of the post url, it'll return the JSON of that post
 Future<VirtualPresetCollection> _danbooru2LikeAPIs(Uri uri, Function(Uri uri, {HandleChunk handleChunk}) importer) async {
-    final res = await http.get(Uri.parse("${[uri.origin, uri.path].join("/")}.json"));
+    final res = await lbHttp.get(Uri.parse("${[uri.origin, uri.path].join("/")}.json"));
     final json = jsonDecode(res.body);
 
     final presets = await multiImageDownloader(
@@ -24,7 +24,7 @@ Future<VirtualPresetCollection> e621ToCollectionPreset(Uri uri) => _danbooru2Lik
 // danbooru 1: /pool/show.xml?id=(id) returns all of the posts already parsed
 Future<VirtualPresetCollection> danbooru1ToCollectionPreset(Uri uri) async {
     final id = uri.pathSegments.last;
-    final res = await http.get(Uri.parse("${uri.origin}/pool/show.json?id=$id"));
+    final res = await lbHttp.get(Uri.parse("${uri.origin}/pool/show.json?id=$id"));
     final json = jsonDecode(res.body);
 
     final presets = await multiImageDownloader(

diff --git a/lib/api/preset/autodownload/single/art_directed.dart b/lib/api/preset/autodownload/single/art_directed.dart
@@ -4,9 +4,9 @@ part of preset;
 // obtain its image. the url nor fxraffinity's embed gives any clue about the poster, but furryaffinity's website title, as well as its
 // embed title gives, so we just fetch those (and also bypasses the nsfw sign up wall)
 Future<PresetImage> furaffinityToPresetImage(Uri uri) async {
-    final fxReq = http.Request("Get", Uri.parse(["https://fxraffinity.net", uri.path, "?full"].join()))..followRedirects = false;
-    final res = await http.Response.fromStream(await http.Client().send(fxReq));
-    final websiteRes = await http.get(Uri.parse(["https://furaffinity.net", uri.path].join()));
+    final fxReq = Request("Get", Uri.parse(["https://fxraffinity.net", uri.path, "?full"].join()))..followRedirects = false;
+    final res = await Response.fromStream(await lbHttp.send(fxReq));
+    final websiteRes = await lbHttp.get(Uri.parse(["https://furaffinity.net", uri.path].join()));
 
     final fileUrl = getMetaProperty(parse(res.body), property: "og:image");
     if(fileUrl == null) throw "Could not grab image";
@@ -26,7 +26,7 @@ Future<PresetImage> furaffinityToPresetImage(Uri uri) async {
 
 // devianart: use their oEmbed API
 Future<PresetImage> deviantartToPresetImage(String url) async {
-    final res = await http.get(Uri.parse(["https://backend.deviantart.com/oembed?url=", url].join()));
+    final res = await lbHttp.get(Uri.parse(["https://backend.deviantart.com/oembed?url=", url].join()));
     final json = jsonDecode(res.body);
 
     final downloadedFileInfo = await downloadFile(Uri.parse(json["url"]));

diff --git a/lib/api/preset/autodownload/single/generic.dart b/lib/api/preset/autodownload/single/generic.dart
@@ -17,10 +17,11 @@ Future<PresetImage> twitterToPresetImage(Uri uri) async {
 
 // twitter: instafix offers a url to give only the image. getting the artist is as easy as reading the first path segment
 Future<PresetImage> instagramToPresetImage(Uri uri) async {
-    final fxReq = http.Request("Get", Uri.parse(["https://ddinstagram.com", uri.path].join()))..followRedirects = false;
-    final title = getMetaProperty(parse(fxReq.body), property: "twitter:title");
+    final fxReq = Request("Get", Uri.parse(["https://ddinstagram.com", uri.path].join()))..followRedirects = false;
+    final response = await Response.fromStream(await lbHttp.send(fxReq));
+    final title = getMetaProperty(parse(response.body), property: "twitter:title");
 
-    debugPrint(fxReq.body);
+    debugPrint(response.body);
 
     final downloadedFileInfo = await downloadFile(Uri.parse(["https://d.ddinstagram.com", uri.path].join()));
 

diff --git a/lib/api/preset/autodownload/single/image_boards.dart b/lib/api/preset/autodownload/single/image_boards.dart
@@ -2,7 +2,7 @@ part of "../../index.dart";
 
 // danbooru 2: if you add .json at the end of the post url, it'll return the JSON of that post
 Future<PresetImage> danbooru2ToPresetImage(Uri uri, {HandleChunk? handleChunk}) async {
-    final res = await http.get(Uri.parse("${[uri.origin, uri.path].join("/")}.json"));
+    final res = await lbHttp.get(Uri.parse("${[uri.origin, uri.path].join("/")}.json"));
     final bodyRes = jsonDecode(res.body);
 
     final downloadedFileInfo = await downloadFile(Uri.parse(bodyRes["file_url"]), handleChunk: handleChunk);
@@ -23,12 +23,12 @@ Future<PresetImage> danbooru2ToPresetImage(Uri uri, {HandleChunk? handleChunk})
 // as we cant obtain tag types in bulk, nor does post.json returns tag types in its response like danbooru 2
 Future<PresetImage> danbooru1ToPresetImage(Uri uri, {HandleChunk? handleChunk}) async {
     final postID = uri.pathSegments[2];
-    final res = await http.get(Uri.parse([uri.origin, "post/index.json?tags=id:$postID"].join("/")));
+    final res = await lbHttp.get(Uri.parse([uri.origin, "post/index.json?tags=id:$postID"].join("/")));
     final post = jsonDecode(res.body)[0];
 
     final downloadedFileInfo = await downloadFile(Uri.parse(post["file_url"]), handleChunk: handleChunk);
 
-    final webpage = await http.get(uri);
+    final webpage = await lbHttp.get(uri);
     final document = parse(webpage.body);
 
     final tagsElements = document.getElementsByClassName("tag-link");
@@ -61,7 +61,7 @@ Future<PresetImage> danbooru1ToPresetImage(Uri uri, {HandleChunk? handleChunk})
 
 // e926/e621: same idea as danbooru 2, if you add .json at the end of the post url, it'll return the JSON of that post
 Future<PresetImage> e621ToPresetImage(Uri uri, {HandleChunk? handleChunk}) async {
-    final res = await http.get(Uri.parse("${[uri.origin, uri.path].join("/")}.json"));
+    final res = await lbHttp.get(Uri.parse("${[uri.origin, uri.path].join("/")}.json"));
     final postRes = jsonDecode(res.body)["post"];
 
     final downloadedFileInfo = await downloadFile(Uri.parse(postRes["file"]["url"]), handleChunk: handleChunk);
@@ -94,7 +94,7 @@ final Map<int, String> gelbooruTagMap = {
 Future<PresetImage> gelbooruToPresetImage(Uri uri) async {
     final String imageID = uri.queryParameters["id"]!;
 
-    final res = await http.get(Uri.parse([uri.origin, "index.php?page=dapi&s=post&q=index&json=1&id=$imageID"].join("/")));
+    final res = await lbHttp.get(Uri.parse([uri.origin, "index.php?page=dapi&s=post&q=index&json=1&id=$imageID"].join("/")));
     final json = jsonDecode(res.body);
     final bool is020 = json is List;
     final Map<String, dynamic> post = !is020 ? json["post"][0] : json[0]; // api differences, first one 0.2.5, second 0.2.0
@@ -110,7 +110,7 @@ Future<PresetImage> gelbooruToPresetImage(Uri uri) async {
     String imageURL;
     if(is020) { // probably 0.2.0, grab html documents
         // sadly it doesn't have an api to obtain tag types, or i couldn't find one
-        final webpage = await http.get(Uri.parse([uri.origin, "index.php?page=post&s=view&id=$imageID"].join("/")));
+        final webpage = await lbHttp.get(Uri.parse([uri.origin, "index.php?page=post&s=view&id=$imageID"].join("/")));
         final document = parse(webpage.body);
 
         final tagsElements = document.getElementsByClassName("tag");
@@ -129,7 +129,7 @@ Future<PresetImage> gelbooruToPresetImage(Uri uri) async {
         imageURL = imageElement!.attributes["src"]!;
 
     } else { // probably 0.2.5, use their api instead
-        final tagTypesRes = await http.get(Uri.parse([uri.origin, "index.php?page=dapi&s=tag&q=index&json=1&names=$tags"].join("/")));
+        final tagTypesRes = await lbHttp.get(Uri.parse([uri.origin, "index.php?page=dapi&s=tag&q=index&json=1&names=$tags"].join("/")));
 
         final List<dynamic> tagTypes = jsonDecode(tagTypesRes.body)["tag"];
 

diff --git a/lib/api/preset/getter/website/accurate.dart b/lib/api/preset/getter/website/accurate.dart
@@ -16,18 +16,18 @@ Future<Websites?> accurateGetWebsite(Uri uri) async {
 Future<Websites?> _determineSPByAPIFetch(Uri uri) async {
     Response res;
 
-    res = await http.get(Uri.parse("${uri.origin}/posts.json"));
+    res = await lbHttp.get(Uri.parse("${uri.origin}/posts.json"));
     if(res.statusCode == 200) {
-        res = await http.get(Uri.parse("${uri.origin}/status.json"));
+        res = await lbHttp.get(Uri.parse("${uri.origin}/status.json"));
         if(res.statusCode == 200) return ServiceWebsites.danbooru2; // e621 does not support /status.json, but supports almost all API endpoints
         else return ServiceWebsites.e621; // e621 does not support /status.json, but supports almost all API endpoints
     }
 
 
-    res = await http.get(Uri.parse("${uri.origin}/post/index.xml"));
+    res = await lbHttp.get(Uri.parse("${uri.origin}/post/index.xml"));
     if(res.statusCode == 200) return ServiceWebsites.danbooru1;
 
-    res = await http.get(Uri.parse("${uri.origin}/index.php?page=dapi&s=user&q=index"));
+    res = await lbHttp.get(Uri.parse("${uri.origin}/index.php?page=dapi&s=user&q=index"));
     if(res.statusCode == 200) {
         if(res.body.isEmpty) return ServiceWebsites.gelbooru020;
         else return ServiceWebsites.gelbooru025;
@@ -37,7 +37,7 @@ Future<Websites?> _determineSPByAPIFetch(Uri uri) async {
 }
 
 Future<Websites?> _determineWebsiteByWebcrawl(Uri uri) async {
-    final webpage = await http.get(uri);
+    final webpage = await lbHttp.get(uri);
     final Document document = parse(webpage.body);
 
     final head = document.head;

diff --git a/lib/api/preset/index.dart b/lib/api/preset/index.dart
@@ -1,17 +1,18 @@
 library preset;
 
+import 'dart:async';
 import 'dart:convert';
 import 'dart:io';
 
 import 'package:collection/collection.dart';
 import 'package:flutter/material.dart';
 import 'package:flutter_svg/svg.dart';
 import 'package:html/dom.dart';
-import 'package:http/http.dart' as http;
 import 'package:http/http.dart';
 import 'package:localbooru/api/index.dart';
 import 'package:html/parser.dart' show parse;
 import 'package:localbooru/api/preset/autodownload/multi/index.dart';
+import 'package:localbooru/utils/http_client.dart' show lbHttp;
 import 'package:localbooru/utils/download_image.dart';
 import 'package:localbooru/utils/get_meta_property.dart';
 import 'package:mime/mime.dart';

diff --git a/lib/utils/download_image.dart b/lib/utils/download_image.dart
@@ -1,6 +1,7 @@
 import 'dart:io';
 
 import 'package:http/http.dart' as http;
+import 'package:localbooru/utils/http_client.dart';
 import 'package:localbooru/utils/listeners.dart';
 import 'package:path/path.dart' as p;
 import 'package:path_provider/path_provider.dart';
@@ -13,7 +14,7 @@ Future<File> downloadFile(Uri uri, {HandleChunk? handleChunk}) async {
     final file = File(p.join(downloadDir.path, uri.pathSegments.last));
 
     final request = http.Request("GET", uri);
-    final response = await request.send();
+    final response = await lbHttp.send(request);
     final sink = file.openWrite();
 
     await response.stream.map((chunk) {

diff --git a/lib/utils/http_client.dart b/lib/utils/http_client.dart
@@ -0,0 +1,39 @@
+import 'dart:async';
+import 'dart:io';
+
+import 'package:http/http.dart' as http;
+import 'package:package_info_plus/package_info_plus.dart';
+
+class LocalBooruHttpClient extends http.BaseClient {
+    LocalBooruHttpClient(this._inner, {
+        this.withCustomUserAgent = true,
+        this.manipulateRequest
+    });
+
+    final http.Client _inner;
+    final bool withCustomUserAgent;
+    final FutureOr<http.BaseRequest> Function(http.BaseRequest request)? manipulateRequest;
+
+    @override
+    Future<http.StreamedResponse> send(http.BaseRequest request) async {
+        if(withCustomUserAgent) {
+            if(!Platform.environment.containsKey('FLUTTER_TEST')) {
+                final package = await PackageInfo.fromPlatform();
+                request.headers['user-agent'] = "LocalBooru/${package.version}"; //didn't even test, dont know how i'll be sure it is sending headers
+            } else {
+                request.headers['user-agent'] = "LocalBooru/TEST_ENV";
+            }
+        }
+        if(manipulateRequest != null) request = await manipulateRequest!(request);
+        return await _inner.send(request);
+    }
+}
+
+class LoggedHttpClients {
+    static LocalBooruHttpClient e621Cleint = LocalBooruHttpClient(http.Client());
+}
+
+final lbHttp = LocalBooruHttpClient(http.Client());
+// final e6lbHttp = LocalBooruHttpClient(http.Client(), manipulateRequest: (request) {
+
+// },);
diff --git a/test/import/identify/determine_collection_test.dart b/test/import/identify/determine_collection_test.dart
@@ -1,8 +1,8 @@
 @Tags(['import', 'identify', 'multi'])
 library;
 
+import 'package:flutter_test/flutter_test.dart';
 import 'package:localbooru/api/preset/index.dart';
-import 'package:test/test.dart';
 
 import '../../shared.dart';