From da8c9a4c34cb1ad5c6e3b024c5dcbdd93c4f0930 Mon Sep 17 00:00:00 2001 From: DatLag Date: Wed, 19 Apr 2023 22:31:53 +0200 Subject: [PATCH] fixed video scraping --- .../datlag/burningseries/module/PlatformModule.kt | 12 +++++++++--- .../dev/datlag/burningseries/module/NetworkModule.kt | 1 + .../datlag/burningseries/module/PlatformModule.kt | 12 +++++++++--- network/build.gradle.kts | 9 ++------- scraper/build.gradle.kts | 4 +++- .../burningseries/scraper}/video/VideoScraper.kt | 10 +++++----- .../scraper}/video/VideoStreamManipulation.kt | 2 +- .../burningseries/scraper}/video/hoster/Mixdrop.kt | 4 ++-- .../burningseries/scraper}/video/hoster/StreamZZ.kt | 4 ++-- 9 files changed, 34 insertions(+), 24 deletions(-) rename {network/src/jvmMain/kotlin/dev/datlag/burningseries/network => scraper/src/main/kotlin/dev/datlag/burningseries/scraper}/video/VideoScraper.kt (92%) rename {network/src/jvmMain/kotlin/dev/datlag/burningseries/network => scraper/src/main/kotlin/dev/datlag/burningseries/scraper}/video/VideoStreamManipulation.kt (88%) rename {network/src/jvmMain/kotlin/dev/datlag/burningseries/network => scraper/src/main/kotlin/dev/datlag/burningseries/scraper}/video/hoster/Mixdrop.kt (91%) rename {network/src/jvmMain/kotlin/dev/datlag/burningseries/network => scraper/src/main/kotlin/dev/datlag/burningseries/scraper}/video/hoster/StreamZZ.kt (81%) diff --git a/app/src/androidMain/java/dev/datlag/burningseries/module/PlatformModule.kt b/app/src/androidMain/java/dev/datlag/burningseries/module/PlatformModule.kt index 617b60c0..cd4a8b79 100644 --- a/app/src/androidMain/java/dev/datlag/burningseries/module/PlatformModule.kt +++ b/app/src/androidMain/java/dev/datlag/burningseries/module/PlatformModule.kt @@ -4,8 +4,10 @@ import android.content.Context import androidx.datastore.dataStoreFile import dev.datlag.burningseries.database.DriverFactory import dev.datlag.burningseries.datastore.CryptoManager +import dev.datlag.burningseries.model.HosterStream +import dev.datlag.burningseries.model.VideoStream import dev.datlag.burningseries.network.video.Scraper -import dev.datlag.burningseries.network.video.VideoScraper +import dev.datlag.burningseries.scraper.video.VideoScraper import org.kodein.di.* import java.io.File @@ -49,8 +51,12 @@ actual object PlatformModule { returnFile } - bindSingleton { - VideoScraper + bindSingleton { + object : Scraper { + override suspend fun scrapeVideosFrom(hosterStream: HosterStream): VideoStream? { + return VideoScraper.scrapeVideosFrom(hosterStream) + } + } } } diff --git a/app/src/commonMain/kotlin/dev/datlag/burningseries/module/NetworkModule.kt b/app/src/commonMain/kotlin/dev/datlag/burningseries/module/NetworkModule.kt index 16e3da0f..1e75d744 100644 --- a/app/src/commonMain/kotlin/dev/datlag/burningseries/module/NetworkModule.kt +++ b/app/src/commonMain/kotlin/dev/datlag/burningseries/module/NetworkModule.kt @@ -7,6 +7,7 @@ import dev.datlag.burningseries.network.GitHub import dev.datlag.burningseries.network.JsonBase import dev.datlag.burningseries.network.converter.FlowerResponseConverter import dev.datlag.burningseries.network.repository.* +import dev.datlag.burningseries.network.video.Scraper import dev.datlag.burningseries.other.EasyDns import dev.datlag.burningseries.other.MultiDoH import io.ktor.client.* diff --git a/app/src/desktopMain/kotlin/dev/datlag/burningseries/module/PlatformModule.kt b/app/src/desktopMain/kotlin/dev/datlag/burningseries/module/PlatformModule.kt index 062beb96..20a7a072 100644 --- a/app/src/desktopMain/kotlin/dev/datlag/burningseries/module/PlatformModule.kt +++ b/app/src/desktopMain/kotlin/dev/datlag/burningseries/module/PlatformModule.kt @@ -4,9 +4,11 @@ import dev.datlag.burningseries.common.createWithParents import dev.datlag.burningseries.database.BurningSeriesDB import dev.datlag.burningseries.database.DriverFactory import dev.datlag.burningseries.datastore.CryptoManager +import dev.datlag.burningseries.model.HosterStream +import dev.datlag.burningseries.model.VideoStream import dev.datlag.burningseries.network.video.Scraper -import dev.datlag.burningseries.network.video.VideoScraper import dev.datlag.burningseries.other.AppIO +import dev.datlag.burningseries.scraper.video.VideoScraper import net.harawata.appdirs.AppDirs import net.harawata.appdirs.AppDirsFactory import org.kodein.di.* @@ -41,8 +43,12 @@ actual object PlatformModule { AppIO.getFolderInSiteDataDir("images") } - bindSingleton { - VideoScraper + bindSingleton { + object : Scraper { + override suspend fun scrapeVideosFrom(hosterStream: HosterStream): VideoStream? { + return VideoScraper.scrapeVideosFrom(hosterStream) + } + } } } diff --git a/network/build.gradle.kts b/network/build.gradle.kts index a173b51d..899e00da 100644 --- a/network/build.gradle.kts +++ b/network/build.gradle.kts @@ -10,7 +10,6 @@ group = "dev.datlag.burningseries.network" val flower = "3.0.0" val ktorfit = "1.0.1" -val jsunpacker = "1.0.1" val jsoup = "1.15.4" val coroutines = "1.6.4" @@ -34,17 +33,13 @@ kotlin { val jvmMain by getting { dependencies { - implementation("dev.datlag.jsunpacker:jsunpacker:$jsunpacker") - implementation("org.jsoup:jsoup:$jsoup") - implementation(project(":scraper")) + api(project(":scraper")) } } val androidMain by getting { dependencies { - implementation("dev.datlag.jsunpacker:jsunpacker:$jsunpacker") - implementation("org.jsoup:jsoup:$jsoup") - implementation(project(":scraper")) + api(project(":scraper")) } } } diff --git a/scraper/build.gradle.kts b/scraper/build.gradle.kts index d436dd95..b2ac2a68 100644 --- a/scraper/build.gradle.kts +++ b/scraper/build.gradle.kts @@ -8,10 +8,12 @@ group = "dev.datlag.burningseries.scraper" val jsoup = "1.15.4" val coroutines = "1.6.4" val ktor = "2.2.3" +val jsunpacker = "1.0.1" dependencies { implementation(project(":model")) implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutines") implementation("io.ktor:ktor-client-core-jvm:$ktor") - implementation("org.jsoup:jsoup:$jsoup") + api("org.jsoup:jsoup:$jsoup") + implementation("dev.datlag.jsunpacker:jsunpacker:$jsunpacker") } diff --git a/network/src/jvmMain/kotlin/dev/datlag/burningseries/network/video/VideoScraper.kt b/scraper/src/main/kotlin/dev/datlag/burningseries/scraper/video/VideoScraper.kt similarity index 92% rename from network/src/jvmMain/kotlin/dev/datlag/burningseries/network/video/VideoScraper.kt rename to scraper/src/main/kotlin/dev/datlag/burningseries/scraper/video/VideoScraper.kt index c26ddd14..b3c50503 100644 --- a/network/src/jvmMain/kotlin/dev/datlag/burningseries/network/video/VideoScraper.kt +++ b/scraper/src/main/kotlin/dev/datlag/burningseries/scraper/video/VideoScraper.kt @@ -1,22 +1,22 @@ -package dev.datlag.burningseries.network.video +package dev.datlag.burningseries.scraper.video import dev.datlag.burningseries.model.HosterStream import dev.datlag.burningseries.model.VideoStream import dev.datlag.burningseries.scraper.common.getSources -import dev.datlag.burningseries.network.video.hoster.Mixdrop -import dev.datlag.burningseries.network.video.hoster.StreamZZ +import dev.datlag.burningseries.scraper.video.hoster.Mixdrop +import dev.datlag.burningseries.scraper.video.hoster.StreamZZ import dev.datlag.jsunpacker.JsUnpacker import org.jsoup.Jsoup import org.jsoup.nodes.Document -object VideoScraper : Scraper { +object VideoScraper { val MANIPULATION_LIST = listOf( StreamZZ(), Mixdrop() ) - override suspend fun scrapeVideosFrom(hosterStream: HosterStream): VideoStream? { + suspend fun scrapeVideosFrom(hosterStream: HosterStream): VideoStream? { val doc = try { Jsoup.connect(hosterStream.url).get() } catch (ignored: Throwable) { diff --git a/network/src/jvmMain/kotlin/dev/datlag/burningseries/network/video/VideoStreamManipulation.kt b/scraper/src/main/kotlin/dev/datlag/burningseries/scraper/video/VideoStreamManipulation.kt similarity index 88% rename from network/src/jvmMain/kotlin/dev/datlag/burningseries/network/video/VideoStreamManipulation.kt rename to scraper/src/main/kotlin/dev/datlag/burningseries/scraper/video/VideoStreamManipulation.kt index 8e873534..f00f8190 100644 --- a/network/src/jvmMain/kotlin/dev/datlag/burningseries/network/video/VideoStreamManipulation.kt +++ b/scraper/src/main/kotlin/dev/datlag/burningseries/scraper/video/VideoStreamManipulation.kt @@ -1,4 +1,4 @@ -package dev.datlag.burningseries.network.video +package dev.datlag.burningseries.scraper.video import dev.datlag.burningseries.model.HosterStream import org.jsoup.nodes.Document diff --git a/network/src/jvmMain/kotlin/dev/datlag/burningseries/network/video/hoster/Mixdrop.kt b/scraper/src/main/kotlin/dev/datlag/burningseries/scraper/video/hoster/Mixdrop.kt similarity index 91% rename from network/src/jvmMain/kotlin/dev/datlag/burningseries/network/video/hoster/Mixdrop.kt rename to scraper/src/main/kotlin/dev/datlag/burningseries/scraper/video/hoster/Mixdrop.kt index a1640480..df0cb431 100644 --- a/network/src/jvmMain/kotlin/dev/datlag/burningseries/network/video/hoster/Mixdrop.kt +++ b/scraper/src/main/kotlin/dev/datlag/burningseries/scraper/video/hoster/Mixdrop.kt @@ -1,7 +1,7 @@ -package dev.datlag.burningseries.network.video.hoster +package dev.datlag.burningseries.scraper.video.hoster import dev.datlag.burningseries.model.HosterStream -import dev.datlag.burningseries.network.video.VideoStreamManipulation +import dev.datlag.burningseries.scraper.video.VideoStreamManipulation import dev.datlag.jsunpacker.JsUnpacker import org.jsoup.nodes.Document diff --git a/network/src/jvmMain/kotlin/dev/datlag/burningseries/network/video/hoster/StreamZZ.kt b/scraper/src/main/kotlin/dev/datlag/burningseries/scraper/video/hoster/StreamZZ.kt similarity index 81% rename from network/src/jvmMain/kotlin/dev/datlag/burningseries/network/video/hoster/StreamZZ.kt rename to scraper/src/main/kotlin/dev/datlag/burningseries/scraper/video/hoster/StreamZZ.kt index fa8d38a2..4d95904f 100644 --- a/network/src/jvmMain/kotlin/dev/datlag/burningseries/network/video/hoster/StreamZZ.kt +++ b/scraper/src/main/kotlin/dev/datlag/burningseries/scraper/video/hoster/StreamZZ.kt @@ -1,7 +1,7 @@ -package dev.datlag.burningseries.network.video.hoster +package dev.datlag.burningseries.scraper.video.hoster import dev.datlag.burningseries.model.HosterStream -import dev.datlag.burningseries.network.video.VideoStreamManipulation +import dev.datlag.burningseries.scraper.video.VideoStreamManipulation import org.jsoup.nodes.Document class StreamZZ : VideoStreamManipulation {