diff --git a/lib/api/preset/autodownload/multi/image_boards.dart b/lib/api/preset/autodownload/multi/image_boards.dart index 552c40d..888c05a 100644 --- a/lib/api/preset/autodownload/multi/image_boards.dart +++ b/lib/api/preset/autodownload/multi/image_boards.dart @@ -3,7 +3,7 @@ part of preset; // danbooru2 and e621/e926 share the same api endpoints when it comes to pools // danbooru 2: if you add .json at the end of the post url, it'll return the JSON of that post Future _danbooru2LikeAPIs(Uri uri, Function(Uri uri, {HandleChunk handleChunk}) importer) async { - final res = await http.get(Uri.parse("${[uri.origin, uri.path].join("/")}.json")); + final res = await lbHttp.get(Uri.parse("${[uri.origin, uri.path].join("/")}.json")); final json = jsonDecode(res.body); final presets = await multiImageDownloader( @@ -24,7 +24,7 @@ Future e621ToCollectionPreset(Uri uri) => _danbooru2Lik // danbooru 1: /pool/show.xml?id=(id) returns all of the posts already parsed Future danbooru1ToCollectionPreset(Uri uri) async { final id = uri.pathSegments.last; - final res = await http.get(Uri.parse("${uri.origin}/pool/show.json?id=$id")); + final res = await lbHttp.get(Uri.parse("${uri.origin}/pool/show.json?id=$id")); final json = jsonDecode(res.body); final presets = await multiImageDownloader( diff --git a/lib/api/preset/autodownload/single/art_directed.dart b/lib/api/preset/autodownload/single/art_directed.dart index ddf751d..5de98bb 100644 --- a/lib/api/preset/autodownload/single/art_directed.dart +++ b/lib/api/preset/autodownload/single/art_directed.dart @@ -4,9 +4,9 @@ part of preset; // obtain its image. the url nor fxraffinity's embed gives any clue about the poster, but furryaffinity's website title, as well as its // embed title gives, so we just fetch those (and also bypasses the nsfw sign up wall) Future furaffinityToPresetImage(Uri uri) async { - final fxReq = http.Request("Get", Uri.parse(["https://fxraffinity.net", uri.path, "?full"].join()))..followRedirects = false; - final res = await http.Response.fromStream(await http.Client().send(fxReq)); - final websiteRes = await http.get(Uri.parse(["https://furaffinity.net", uri.path].join())); + final fxReq = Request("Get", Uri.parse(["https://fxraffinity.net", uri.path, "?full"].join()))..followRedirects = false; + final res = await Response.fromStream(await lbHttp.send(fxReq)); + final websiteRes = await lbHttp.get(Uri.parse(["https://furaffinity.net", uri.path].join())); final fileUrl = getMetaProperty(parse(res.body), property: "og:image"); if(fileUrl == null) throw "Could not grab image"; @@ -26,7 +26,7 @@ Future furaffinityToPresetImage(Uri uri) async { // devianart: use their oEmbed API Future deviantartToPresetImage(String url) async { - final res = await http.get(Uri.parse(["https://backend.deviantart.com/oembed?url=", url].join())); + final res = await lbHttp.get(Uri.parse(["https://backend.deviantart.com/oembed?url=", url].join())); final json = jsonDecode(res.body); final downloadedFileInfo = await downloadFile(Uri.parse(json["url"])); diff --git a/lib/api/preset/autodownload/single/generic.dart b/lib/api/preset/autodownload/single/generic.dart index 9fe697c..306a41a 100644 --- a/lib/api/preset/autodownload/single/generic.dart +++ b/lib/api/preset/autodownload/single/generic.dart @@ -17,10 +17,11 @@ Future twitterToPresetImage(Uri uri) async { // twitter: instafix offers a url to give only the image. getting the artist is as easy as reading the first path segment Future instagramToPresetImage(Uri uri) async { - final fxReq = http.Request("Get", Uri.parse(["https://ddinstagram.com", uri.path].join()))..followRedirects = false; - final title = getMetaProperty(parse(fxReq.body), property: "twitter:title"); + final fxReq = Request("Get", Uri.parse(["https://ddinstagram.com", uri.path].join()))..followRedirects = false; + final response = await Response.fromStream(await lbHttp.send(fxReq)); + final title = getMetaProperty(parse(response.body), property: "twitter:title"); - debugPrint(fxReq.body); + debugPrint(response.body); final downloadedFileInfo = await downloadFile(Uri.parse(["https://d.ddinstagram.com", uri.path].join())); diff --git a/lib/api/preset/autodownload/single/image_boards.dart b/lib/api/preset/autodownload/single/image_boards.dart index d002e09..932012b 100644 --- a/lib/api/preset/autodownload/single/image_boards.dart +++ b/lib/api/preset/autodownload/single/image_boards.dart @@ -2,7 +2,7 @@ part of "../../index.dart"; // danbooru 2: if you add .json at the end of the post url, it'll return the JSON of that post Future danbooru2ToPresetImage(Uri uri, {HandleChunk? handleChunk}) async { - final res = await http.get(Uri.parse("${[uri.origin, uri.path].join("/")}.json")); + final res = await lbHttp.get(Uri.parse("${[uri.origin, uri.path].join("/")}.json")); final bodyRes = jsonDecode(res.body); final downloadedFileInfo = await downloadFile(Uri.parse(bodyRes["file_url"]), handleChunk: handleChunk); @@ -23,12 +23,12 @@ Future danbooru2ToPresetImage(Uri uri, {HandleChunk? handleChunk}) // as we cant obtain tag types in bulk, nor does post.json returns tag types in its response like danbooru 2 Future danbooru1ToPresetImage(Uri uri, {HandleChunk? handleChunk}) async { final postID = uri.pathSegments[2]; - final res = await http.get(Uri.parse([uri.origin, "post/index.json?tags=id:$postID"].join("/"))); + final res = await lbHttp.get(Uri.parse([uri.origin, "post/index.json?tags=id:$postID"].join("/"))); final post = jsonDecode(res.body)[0]; final downloadedFileInfo = await downloadFile(Uri.parse(post["file_url"]), handleChunk: handleChunk); - final webpage = await http.get(uri); + final webpage = await lbHttp.get(uri); final document = parse(webpage.body); final tagsElements = document.getElementsByClassName("tag-link"); @@ -61,7 +61,7 @@ Future danbooru1ToPresetImage(Uri uri, {HandleChunk? handleChunk}) // e926/e621: same idea as danbooru 2, if you add .json at the end of the post url, it'll return the JSON of that post Future e621ToPresetImage(Uri uri, {HandleChunk? handleChunk}) async { - final res = await http.get(Uri.parse("${[uri.origin, uri.path].join("/")}.json")); + final res = await lbHttp.get(Uri.parse("${[uri.origin, uri.path].join("/")}.json")); final postRes = jsonDecode(res.body)["post"]; final downloadedFileInfo = await downloadFile(Uri.parse(postRes["file"]["url"]), handleChunk: handleChunk); @@ -94,7 +94,7 @@ final Map gelbooruTagMap = { Future gelbooruToPresetImage(Uri uri) async { final String imageID = uri.queryParameters["id"]!; - final res = await http.get(Uri.parse([uri.origin, "index.php?page=dapi&s=post&q=index&json=1&id=$imageID"].join("/"))); + final res = await lbHttp.get(Uri.parse([uri.origin, "index.php?page=dapi&s=post&q=index&json=1&id=$imageID"].join("/"))); final json = jsonDecode(res.body); final bool is020 = json is List; final Map post = !is020 ? json["post"][0] : json[0]; // api differences, first one 0.2.5, second 0.2.0 @@ -110,7 +110,7 @@ Future gelbooruToPresetImage(Uri uri) async { String imageURL; if(is020) { // probably 0.2.0, grab html documents // sadly it doesn't have an api to obtain tag types, or i couldn't find one - final webpage = await http.get(Uri.parse([uri.origin, "index.php?page=post&s=view&id=$imageID"].join("/"))); + final webpage = await lbHttp.get(Uri.parse([uri.origin, "index.php?page=post&s=view&id=$imageID"].join("/"))); final document = parse(webpage.body); final tagsElements = document.getElementsByClassName("tag"); @@ -129,7 +129,7 @@ Future gelbooruToPresetImage(Uri uri) async { imageURL = imageElement!.attributes["src"]!; } else { // probably 0.2.5, use their api instead - final tagTypesRes = await http.get(Uri.parse([uri.origin, "index.php?page=dapi&s=tag&q=index&json=1&names=$tags"].join("/"))); + final tagTypesRes = await lbHttp.get(Uri.parse([uri.origin, "index.php?page=dapi&s=tag&q=index&json=1&names=$tags"].join("/"))); final List tagTypes = jsonDecode(tagTypesRes.body)["tag"]; diff --git a/lib/api/preset/getter/website/accurate.dart b/lib/api/preset/getter/website/accurate.dart index 419c615..eb96b11 100644 --- a/lib/api/preset/getter/website/accurate.dart +++ b/lib/api/preset/getter/website/accurate.dart @@ -16,18 +16,18 @@ Future accurateGetWebsite(Uri uri) async { Future _determineSPByAPIFetch(Uri uri) async { Response res; - res = await http.get(Uri.parse("${uri.origin}/posts.json")); + res = await lbHttp.get(Uri.parse("${uri.origin}/posts.json")); if(res.statusCode == 200) { - res = await http.get(Uri.parse("${uri.origin}/status.json")); + res = await lbHttp.get(Uri.parse("${uri.origin}/status.json")); if(res.statusCode == 200) return ServiceWebsites.danbooru2; // e621 does not support /status.json, but supports almost all API endpoints else return ServiceWebsites.e621; // e621 does not support /status.json, but supports almost all API endpoints } - res = await http.get(Uri.parse("${uri.origin}/post/index.xml")); + res = await lbHttp.get(Uri.parse("${uri.origin}/post/index.xml")); if(res.statusCode == 200) return ServiceWebsites.danbooru1; - res = await http.get(Uri.parse("${uri.origin}/index.php?page=dapi&s=user&q=index")); + res = await lbHttp.get(Uri.parse("${uri.origin}/index.php?page=dapi&s=user&q=index")); if(res.statusCode == 200) { if(res.body.isEmpty) return ServiceWebsites.gelbooru020; else return ServiceWebsites.gelbooru025; @@ -37,7 +37,7 @@ Future _determineSPByAPIFetch(Uri uri) async { } Future _determineWebsiteByWebcrawl(Uri uri) async { - final webpage = await http.get(uri); + final webpage = await lbHttp.get(uri); final Document document = parse(webpage.body); final head = document.head; diff --git a/lib/api/preset/index.dart b/lib/api/preset/index.dart index 44c05eb..a42f62c 100644 --- a/lib/api/preset/index.dart +++ b/lib/api/preset/index.dart @@ -1,5 +1,6 @@ library preset; +import 'dart:async'; import 'dart:convert'; import 'dart:io'; @@ -7,11 +8,11 @@ import 'package:collection/collection.dart'; import 'package:flutter/material.dart'; import 'package:flutter_svg/svg.dart'; import 'package:html/dom.dart'; -import 'package:http/http.dart' as http; import 'package:http/http.dart'; import 'package:localbooru/api/index.dart'; import 'package:html/parser.dart' show parse; import 'package:localbooru/api/preset/autodownload/multi/index.dart'; +import 'package:localbooru/utils/http_client.dart' show lbHttp; import 'package:localbooru/utils/download_image.dart'; import 'package:localbooru/utils/get_meta_property.dart'; import 'package:mime/mime.dart'; diff --git a/lib/utils/download_image.dart b/lib/utils/download_image.dart index ab5521f..a0462ef 100644 --- a/lib/utils/download_image.dart +++ b/lib/utils/download_image.dart @@ -1,6 +1,7 @@ import 'dart:io'; import 'package:http/http.dart' as http; +import 'package:localbooru/utils/http_client.dart'; import 'package:localbooru/utils/listeners.dart'; import 'package:path/path.dart' as p; import 'package:path_provider/path_provider.dart'; @@ -13,7 +14,7 @@ Future downloadFile(Uri uri, {HandleChunk? handleChunk}) async { final file = File(p.join(downloadDir.path, uri.pathSegments.last)); final request = http.Request("GET", uri); - final response = await request.send(); + final response = await lbHttp.send(request); final sink = file.openWrite(); await response.stream.map((chunk) { diff --git a/lib/utils/http_client.dart b/lib/utils/http_client.dart new file mode 100644 index 0000000..13cf99f --- /dev/null +++ b/lib/utils/http_client.dart @@ -0,0 +1,39 @@ +import 'dart:async'; +import 'dart:io'; + +import 'package:http/http.dart' as http; +import 'package:package_info_plus/package_info_plus.dart'; + +class LocalBooruHttpClient extends http.BaseClient { + LocalBooruHttpClient(this._inner, { + this.withCustomUserAgent = true, + this.manipulateRequest + }); + + final http.Client _inner; + final bool withCustomUserAgent; + final FutureOr Function(http.BaseRequest request)? manipulateRequest; + + @override + Future send(http.BaseRequest request) async { + if(withCustomUserAgent) { + if(!Platform.environment.containsKey('FLUTTER_TEST')) { + final package = await PackageInfo.fromPlatform(); + request.headers['user-agent'] = "LocalBooru/${package.version}"; //didn't even test, dont know how i'll be sure it is sending headers + } else { + request.headers['user-agent'] = "LocalBooru/TEST_ENV"; + } + } + if(manipulateRequest != null) request = await manipulateRequest!(request); + return await _inner.send(request); + } +} + +class LoggedHttpClients { + static LocalBooruHttpClient e621Cleint = LocalBooruHttpClient(http.Client()); +} + +final lbHttp = LocalBooruHttpClient(http.Client()); +// final e6lbHttp = LocalBooruHttpClient(http.Client(), manipulateRequest: (request) { + +// },); \ No newline at end of file diff --git a/test/import/identify/determine_collection_test.dart b/test/import/identify/determine_collection_test.dart index 7c4b077..10a28e5 100644 --- a/test/import/identify/determine_collection_test.dart +++ b/test/import/identify/determine_collection_test.dart @@ -1,8 +1,8 @@ @Tags(['import', 'identify', 'multi']) library; +import 'package:flutter_test/flutter_test.dart'; import 'package:localbooru/api/preset/index.dart'; -import 'package:test/test.dart'; import '../../shared.dart';