From 25a70d7fc77a3b587a02f11404ce9ee36312305e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 28 Sep 2023 09:24:37 +0000 Subject: [PATCH 01/16] fix(deps): update dependency io.sentry:sentry to v6.30.0 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 7ab5ba1..d674d42 100644 --- a/build.gradle +++ b/build.gradle @@ -41,7 +41,7 @@ dependencies { implementation(platform("com.squareup.okhttp3:okhttp-bom:4.11.0")) implementation 'com.squareup.okhttp3:okhttp' implementation 'com.squareup.okhttp3:okhttp-brotli' - implementation 'io.sentry:sentry:6.28.0' + implementation 'io.sentry:sentry:6.30.0' implementation 'rocks.kavin:reqwest4j:1.0.12' implementation 'io.minio:minio:8.5.6' } From 11d289cb5bfb42995c885bb757bf3228774ea970 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 29 Sep 2023 22:23:57 +0000 Subject: [PATCH 02/16] fix(deps): update dependency commons-io:commons-io to v2.14.0 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index d674d42..caa03ff 100644 --- a/build.gradle +++ b/build.gradle @@ -13,7 +13,7 @@ repositories { dependencies { implementation 'org.apache.commons:commons-lang3:3.13.0' implementation 'org.apache.commons:commons-text:1.10.0' - implementation 'commons-io:commons-io:2.13.0' + implementation 'commons-io:commons-io:2.14.0' implementation 'it.unimi.dsi:fastutil-core:8.5.12' implementation 'commons-codec:commons-codec:1.16.0' implementation 'org.bouncycastle:bcprov-jdk15on:1.70' From c936595a50d8fbeb9eac2c3d3f6362cd691634fb Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 4 Oct 2023 23:23:37 +0000 Subject: [PATCH 03/16] chore(deps): update dependency gradle to v8.4 --- gradle/wrapper/gradle-wrapper.properties | 2 +- gradlew | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 1a38ecc..a456744 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://downloads.gradle.org/distributions/gradle-8.3-bin.zip +distributionUrl=https\://downloads.gradle.org/distributions/gradle-8.4-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index 0adc8e1..1aa94a4 100755 --- a/gradlew +++ b/gradlew @@ -145,7 +145,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC2039,SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac @@ -153,7 +153,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then '' | soft) :;; #( *) # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC2039,SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -202,11 +202,11 @@ fi # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' -# Collect all arguments for the java command; -# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of -# shell script including quotes and variable substitutions, so put them in -# double quotes to make sure that they get re-expanded; and -# * put everything else in single quotes, so that it's not re-expanded. +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ From 5f2043859e37412813ffe7f182b8fd2246aff5d8 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 8 Oct 2023 15:06:44 +0000 Subject: [PATCH 04/16] chore(deps): update plugin io.freefair.lombok to v8.4 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index caa03ff..a82c231 100644 --- a/build.gradle +++ b/build.gradle @@ -1,7 +1,7 @@ plugins { id "com.github.johnrengelman.shadow" version "8.1.1" id "java" - id "io.freefair.lombok" version "8.3" + id "io.freefair.lombok" version "8.4" id "eclipse" } From f25ad497d218fbd1f863914d5ae9f754f9a83ac4 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Tue, 19 Sep 2023 23:57:19 +0100 Subject: [PATCH 05/16] Use oracle's new container registry for docker images. --- Dockerfile.graalvm-jvm | 2 +- Dockerfile.graalvm-jvm.ci | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile.graalvm-jvm b/Dockerfile.graalvm-jvm index a085b0d..c217048 100644 --- a/Dockerfile.graalvm-jvm +++ b/Dockerfile.graalvm-jvm @@ -1,4 +1,4 @@ -FROM ghcr.io/graalvm/native-image:latest as build +FROM container-registry.oracle.com/graalvm/native-image:latest as build WORKDIR /app/ diff --git a/Dockerfile.graalvm-jvm.ci b/Dockerfile.graalvm-jvm.ci index 53be7ba..6140fe4 100644 --- a/Dockerfile.graalvm-jvm.ci +++ b/Dockerfile.graalvm-jvm.ci @@ -1,4 +1,4 @@ -FROM ghcr.io/graalvm/native-image:latest as build +FROM container-registry.oracle.com/graalvm/native-image:latest as build RUN jlink \ --add-modules java.base,java.logging,java.sql,java.management,java.xml,java.naming,java.desktop,jdk.crypto.ec \ From 922ef589c9b55d9666af4596b302af3a3092311c Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Tue, 19 Sep 2023 23:57:44 +0100 Subject: [PATCH 06/16] actions: use java 21 in CI --- .github/workflows/ci.yml | 2 +- .github/workflows/docker-migrations-build-test.yml | 4 ++-- .github/workflows/fat-build.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e65a91f..a60da25 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - java: [ 17 ] + java: [ 21 ] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/docker-migrations-build-test.yml b/.github/workflows/docker-migrations-build-test.yml index f31d437..ee91379 100644 --- a/.github/workflows/docker-migrations-build-test.yml +++ b/.github/workflows/docker-migrations-build-test.yml @@ -15,10 +15,10 @@ jobs: - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.base.sha }} - - name: set up JDK 17 + - name: set up JDK 21 uses: actions/setup-java@v3 with: - java-version: 17 + java-version: 21 distribution: temurin cache: "gradle" - name: Run Build diff --git a/.github/workflows/fat-build.yml b/.github/workflows/fat-build.yml index 26be8ed..b79c5e8 100644 --- a/.github/workflows/fat-build.yml +++ b/.github/workflows/fat-build.yml @@ -9,10 +9,10 @@ jobs: steps: - uses: actions/checkout@v4 - - name: set up JDK 17 + - name: set up JDK 21 uses: actions/setup-java@v3 with: - java-version: 17 + java-version: 21 distribution: temurin cache: "gradle" - name: Run Build From b3fd4f2ce9a9d14dfc031a85da32f911ba056ae1 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Tue, 19 Sep 2023 23:58:20 +0100 Subject: [PATCH 07/16] Compile and target java 21. --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index a82c231..50486a8 100644 --- a/build.gradle +++ b/build.gradle @@ -60,5 +60,5 @@ jar { group = 'me.kavin.piped' version = '1.0' -sourceCompatibility = JavaVersion.VERSION_17 -targetCompatibility = JavaVersion.VERSION_17 +sourceCompatibility = JavaVersion.VERSION_21 +targetCompatibility = JavaVersion.VERSION_21 From 52229ed8c73b42ef412ec53ea8dbd1a554de4467 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Wed, 20 Sep 2023 00:45:07 +0100 Subject: [PATCH 08/16] Use virtual threads. --- src/main/java/me/kavin/piped/Main.java | 4 ++-- src/main/java/me/kavin/piped/utils/Multithreading.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/me/kavin/piped/Main.java b/src/main/java/me/kavin/piped/Main.java index bf6ab03..fa6403e 100644 --- a/src/main/java/me/kavin/piped/Main.java +++ b/src/main/java/me/kavin/piped/Main.java @@ -53,11 +53,11 @@ public class Main { System.exit(1); } - Multithreading.runAsync(() -> new Thread(new SyncRunner( + Multithreading.runAsync(() -> Thread.ofVirtual().start(new SyncRunner( new OkHttpClient.Builder().readTimeout(60, TimeUnit.SECONDS).build(), MATRIX_SERVER, MatrixHelper.MATRIX_TOKEN) - ).start()); + )); new Timer().scheduleAtFixedRate(new TimerTask() { @Override diff --git a/src/main/java/me/kavin/piped/utils/Multithreading.java b/src/main/java/me/kavin/piped/utils/Multithreading.java index 2e4efd4..82c0b5b 100644 --- a/src/main/java/me/kavin/piped/utils/Multithreading.java +++ b/src/main/java/me/kavin/piped/utils/Multithreading.java @@ -5,7 +5,7 @@ import java.util.function.Supplier; public class Multithreading { - private static final ExecutorService es = Executors.newCachedThreadPool(); + private static final ExecutorService es = Executors.newVirtualThreadPerTaskExecutor(); private static final ExecutorService esLimited = Executors .newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 8); private static final ExecutorService esLimitedPubSub = Executors From 4d4bfdc21dfbc255330b45cd1bdbfd9b8fc2e687 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Wed, 20 Sep 2023 00:54:58 +0100 Subject: [PATCH 09/16] actions: use zulu jre distribution --- .github/workflows/ci.yml | 2 +- .github/workflows/docker-migrations-build-test.yml | 2 +- .github/workflows/fat-build.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a60da25..5edb614 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: ${{ matrix.java }} - distribution: temurin + distribution: zulu cache: "gradle" - name: Run Build run: ./gradlew build diff --git a/.github/workflows/docker-migrations-build-test.yml b/.github/workflows/docker-migrations-build-test.yml index ee91379..3e440cd 100644 --- a/.github/workflows/docker-migrations-build-test.yml +++ b/.github/workflows/docker-migrations-build-test.yml @@ -19,7 +19,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: 21 - distribution: temurin + distribution: zulu cache: "gradle" - name: Run Build run: ./gradlew shadowJar diff --git a/.github/workflows/fat-build.yml b/.github/workflows/fat-build.yml index b79c5e8..6d3d85b 100644 --- a/.github/workflows/fat-build.yml +++ b/.github/workflows/fat-build.yml @@ -13,7 +13,7 @@ jobs: uses: actions/setup-java@v3 with: java-version: 21 - distribution: temurin + distribution: zulu cache: "gradle" - name: Run Build run: ./gradlew shadowJar From 9cfd3a2556e7a95956e3e03fa60c6f2376066fda Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Wed, 27 Sep 2023 04:53:41 +0100 Subject: [PATCH 10/16] Use lombok directly without an unofficial lombok plugin. --- build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 50486a8..cbc7e4c 100644 --- a/build.gradle +++ b/build.gradle @@ -1,7 +1,6 @@ plugins { id "com.github.johnrengelman.shadow" version "8.1.1" id "java" - id "io.freefair.lombok" version "8.4" id "eclipse" } @@ -44,6 +43,8 @@ dependencies { implementation 'io.sentry:sentry:6.30.0' implementation 'rocks.kavin:reqwest4j:1.0.12' implementation 'io.minio:minio:8.5.6' + compileOnly 'org.projectlombok:lombok:1.18.30' + annotationProcessor 'org.projectlombok:lombok:1.18.30' } shadowJar { From 5a0554ad8c293d333f66977c83f6e6866f470bb3 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Tue, 10 Oct 2023 22:52:26 +0100 Subject: [PATCH 11/16] Don't build openj9 and adoptium for now. --- .github/workflows/docker-build-test.yml | 4 ++-- .github/workflows/docker-build.yml | 8 ++++---- .github/workflows/docker-migrations-build-test.yml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker-build-test.yml b/.github/workflows/docker-build-test.yml index 26423c4..ae9f905 100644 --- a/.github/workflows/docker-build-test.yml +++ b/.github/workflows/docker-build-test.yml @@ -22,9 +22,9 @@ jobs: - testing/docker-compose.cockroachdb.yml - testing/docker-compose.yugabytedb.yml dockerfile: - - Dockerfile.ci + #- Dockerfile.ci - Dockerfile.azul.ci - - Dockerfile.openj9.ci + #- Dockerfile.openj9.ci - Dockerfile.graalvm-jvm.ci include: - sleep: 20 diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 800f644..a7ea2ba 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -17,10 +17,10 @@ jobs: strategy: matrix: include: - - image: 1337kavin/piped:openj9 - dockerfile: ./Dockerfile.openj9.ci - - image: 1337kavin/piped:hotspot - dockerfile: ./Dockerfile.ci +# - image: 1337kavin/piped:openj9 +# dockerfile: ./Dockerfile.openj9.ci +# - image: 1337kavin/piped:hotspot +# dockerfile: ./Dockerfile.ci - image: 1337kavin/piped:latest,1337kavin/piped:azul-zulu dockerfile: ./Dockerfile.azul.ci - image: 1337kavin/piped:graalvm-jvm diff --git a/.github/workflows/docker-migrations-build-test.yml b/.github/workflows/docker-migrations-build-test.yml index 3e440cd..8290e96 100644 --- a/.github/workflows/docker-migrations-build-test.yml +++ b/.github/workflows/docker-migrations-build-test.yml @@ -39,7 +39,7 @@ jobs: - testing/docker-compose.cockroachdb.yml - testing/docker-compose.yugabytedb.yml dockerfile: - - Dockerfile.ci + - Dockerfile.azul.ci include: - sleep: 20 - docker-compose-file: testing/docker-compose.cockroachdb.yml From 0cda3836d58d8931aac7bb45e967e5f5c562b229 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Tue, 10 Oct 2023 22:43:36 +0100 Subject: [PATCH 12/16] Update NPE. --- build.gradle | 2 +- src/main/java/me/kavin/piped/Main.java | 6 +++--- .../kavin/piped/server/handlers/ChannelHandlers.java | 10 +++++----- .../kavin/piped/server/handlers/PlaylistHandlers.java | 9 ++++----- .../me/kavin/piped/server/handlers/StreamHandlers.java | 7 +++---- .../server/handlers/auth/AuthPlaylistHandlers.java | 6 +++--- .../java/me/kavin/piped/utils/CollectionUtils.java | 10 +++++----- src/main/java/me/kavin/piped/utils/DatabaseHelper.java | 2 +- src/main/java/me/kavin/piped/utils/URLUtils.java | 6 ++++++ src/main/java/me/kavin/piped/utils/VideoHelpers.java | 4 ++-- 10 files changed, 33 insertions(+), 29 deletions(-) diff --git a/build.gradle b/build.gradle index cbc7e4c..b87f662 100644 --- a/build.gradle +++ b/build.gradle @@ -16,7 +16,7 @@ dependencies { implementation 'it.unimi.dsi:fastutil-core:8.5.12' implementation 'commons-codec:commons-codec:1.16.0' implementation 'org.bouncycastle:bcprov-jdk15on:1.70' - implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:5518112dce594bb4cd07d8ed3391bcc5e688f829' + implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:48beff184a9792c4787cfa05fce577c3adf89f56' implementation 'com.github.FireMasterK:nanojson:9f4af3b739cc13f3d0d9d4b758bbe2b2ae7119d7' implementation 'com.fasterxml.jackson.core:jackson-core:2.15.2' implementation 'com.fasterxml.jackson.core:jackson-annotations:2.15.2' diff --git a/src/main/java/me/kavin/piped/Main.java b/src/main/java/me/kavin/piped/Main.java index fa6403e..b80d31f 100644 --- a/src/main/java/me/kavin/piped/Main.java +++ b/src/main/java/me/kavin/piped/Main.java @@ -18,7 +18,7 @@ import org.hibernate.StatelessSession; import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; -import org.schabi.newpipe.extractor.services.youtube.YoutubeThrottlingDecrypter; +import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptPlayerManager; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; import java.util.*; @@ -62,8 +62,8 @@ public class Main { new Timer().scheduleAtFixedRate(new TimerTask() { @Override public void run() { - System.out.printf("ThrottlingCache: %o entries%n", YoutubeThrottlingDecrypter.getCacheSize()); - YoutubeThrottlingDecrypter.clearCache(); + System.out.printf("ThrottlingCache: %o entries%n", YoutubeJavaScriptPlayerManager.getThrottlingParametersCacheSize()); + YoutubeJavaScriptPlayerManager.clearThrottlingParametersCache(); } }, 0, TimeUnit.MINUTES.toMillis(60)); diff --git a/src/main/java/me/kavin/piped/server/handlers/ChannelHandlers.java b/src/main/java/me/kavin/piped/server/handlers/ChannelHandlers.java index 92117ca..1e43bc0 100644 --- a/src/main/java/me/kavin/piped/server/handlers/ChannelHandlers.java +++ b/src/main/java/me/kavin/piped/server/handlers/ChannelHandlers.java @@ -32,7 +32,7 @@ import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE; import static me.kavin.piped.consts.Constants.mapper; import static me.kavin.piped.utils.CollectionUtils.collectPreloadedTabs; import static me.kavin.piped.utils.CollectionUtils.collectRelatedItems; -import static me.kavin.piped.utils.URLUtils.rewriteURL; +import static me.kavin.piped.utils.URLUtils.getLastThumbnail; public class ChannelHandlers { public static byte[] channelResponse(String channelPath) throws Exception { @@ -77,7 +77,7 @@ public class ChannelHandlers { Multithreading.runAsync(() -> { try { MatrixHelper.sendEvent("video.piped.channel.info", new FederatedChannelInfo( - info.getId(), StringUtils.abbreviate(info.getName(), 100), info.getAvatarUrl(), info.isVerified()) + info.getId(), StringUtils.abbreviate(info.getName(), 100), info.getAvatars().isEmpty() ? null : info.getAvatars().getLast().getUrl(), info.isVerified()) ); } catch (IOException e) { throw new RuntimeException(e); @@ -93,7 +93,7 @@ public class ChannelHandlers { if (channel != null) { - ChannelHelpers.updateChannel(s, channel, StringUtils.abbreviate(info.getName(), 100), info.getAvatarUrl(), info.isVerified()); + ChannelHelpers.updateChannel(s, channel, StringUtils.abbreviate(info.getName(), 100), info.getAvatars().isEmpty() ? null : info.getAvatars().getLast().getUrl(), info.isVerified()); Set ids = tabInfo.getRelatedItems() .stream() @@ -159,8 +159,8 @@ public class ChannelHandlers { } }).toList(); - final Channel channel = new Channel(info.getId(), info.getName(), rewriteURL(info.getAvatarUrl()), - rewriteURL(info.getBannerUrl()), info.getDescription(), info.getSubscriberCount(), info.isVerified(), + final Channel channel = new Channel(info.getId(), info.getName(), getLastThumbnail(info.getAvatars()), + getLastThumbnail(info.getBanners()), info.getDescription(), info.getSubscriberCount(), info.isVerified(), nextpage, relatedStreams, tabs); return mapper.writeValueAsBytes(channel); diff --git a/src/main/java/me/kavin/piped/server/handlers/PlaylistHandlers.java b/src/main/java/me/kavin/piped/server/handlers/PlaylistHandlers.java index 94d771e..6b358b1 100644 --- a/src/main/java/me/kavin/piped/server/handlers/PlaylistHandlers.java +++ b/src/main/java/me/kavin/piped/server/handlers/PlaylistHandlers.java @@ -30,8 +30,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE; import static me.kavin.piped.consts.Constants.mapper; import static me.kavin.piped.utils.CollectionUtils.collectRelatedItems; -import static me.kavin.piped.utils.URLUtils.rewriteURL; -import static me.kavin.piped.utils.URLUtils.substringYouTube; +import static me.kavin.piped.utils.URLUtils.*; public class PlaylistHandlers { public static byte[] playlistResponse(String playlistId) throws Exception { @@ -60,10 +59,10 @@ public class PlaylistHandlers { nextpage = mapper.writeValueAsString(page); } - final Playlist playlist = new Playlist(info.getName(), rewriteURL(info.getThumbnailUrl()), - info.getDescription().getContent(), rewriteURL(info.getBannerUrl()), nextpage, + final Playlist playlist = new Playlist(info.getName(), getLastThumbnail(info.getThumbnails()), + info.getDescription().getContent(), getLastThumbnail(info.getBanners()), nextpage, info.getUploaderName().isEmpty() ? null : info.getUploaderName(), - substringYouTube(info.getUploaderUrl()), rewriteURL(info.getUploaderAvatarUrl()), + substringYouTube(info.getUploaderUrl()), getLastThumbnail(info.getUploaderAvatars()), (int) info.getStreamCount(), relatedStreams); return mapper.writeValueAsBytes(playlist); diff --git a/src/main/java/me/kavin/piped/server/handlers/StreamHandlers.java b/src/main/java/me/kavin/piped/server/handlers/StreamHandlers.java index a6e50e3..c7de8be 100644 --- a/src/main/java/me/kavin/piped/server/handlers/StreamHandlers.java +++ b/src/main/java/me/kavin/piped/server/handlers/StreamHandlers.java @@ -36,8 +36,7 @@ import java.util.concurrent.TimeoutException; import static java.nio.charset.StandardCharsets.UTF_8; import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE; import static me.kavin.piped.consts.Constants.mapper; -import static me.kavin.piped.utils.URLUtils.rewriteURL; -import static me.kavin.piped.utils.URLUtils.substringYouTube; +import static me.kavin.piped.utils.URLUtils.*; import static org.schabi.newpipe.extractor.NewPipe.getPreferredContentCountry; import static org.schabi.newpipe.extractor.NewPipe.getPreferredLocalization; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; @@ -342,7 +341,7 @@ public class StreamHandlers { if (comment.getReplies() != null) repliespage = mapper.writeValueAsString(comment.getReplies()); - comments.add(new Comment(comment.getUploaderName(), rewriteURL(comment.getUploaderAvatarUrl()), + comments.add(new Comment(comment.getUploaderName(), getLastThumbnail(comment.getUploaderAvatars()), comment.getCommentId(), Optional.ofNullable(comment.getCommentText()).map(Description::getContent).orElse(null), comment.getTextualUploadDate(), substringYouTube(comment.getUploaderUrl()), repliespage, comment.getLikeCount(), comment.getReplyCount(), comment.isHeartedByUploader(), comment.isPinned(), comment.isUploaderVerified())); @@ -380,7 +379,7 @@ public class StreamHandlers { if (comment.getReplies() != null) repliespage = mapper.writeValueAsString(comment.getReplies()); - comments.add(new Comment(comment.getUploaderName(), rewriteURL(comment.getUploaderAvatarUrl()), + comments.add(new Comment(comment.getUploaderName(), getLastThumbnail(comment.getUploaderAvatars()), comment.getCommentId(), Optional.ofNullable(comment.getCommentText()).map(Description::getContent).orElse(null), comment.getTextualUploadDate(), substringYouTube(comment.getUploaderUrl()), repliespage, comment.getLikeCount(), comment.getReplyCount(), comment.isHeartedByUploader(), comment.isPinned(), comment.isUploaderVerified())); diff --git a/src/main/java/me/kavin/piped/server/handlers/auth/AuthPlaylistHandlers.java b/src/main/java/me/kavin/piped/server/handlers/auth/AuthPlaylistHandlers.java index 4cdffb9..03829d9 100644 --- a/src/main/java/me/kavin/piped/server/handlers/auth/AuthPlaylistHandlers.java +++ b/src/main/java/me/kavin/piped/server/handlers/auth/AuthPlaylistHandlers.java @@ -268,7 +268,7 @@ public class AuthPlaylistHandlers { channel = DatabaseHelper.saveChannel(channelId); } - video = new PlaylistVideo(videoId, info.getName(), info.getThumbnailUrl(), info.getDuration(), channel); + video = new PlaylistVideo(videoId, info.getName(), info.getThumbnails().getLast().getUrl(), info.getDuration(), channel); var tr = s.beginTransaction(); try { @@ -402,7 +402,7 @@ public class AuthPlaylistHandlers { PlaylistInfo info = PlaylistInfo.getInfo(url); - var playlist = new me.kavin.piped.utils.obj.db.Playlist(info.getName(), user, info.getThumbnailUrl()); + var playlist = new me.kavin.piped.utils.obj.db.Playlist(info.getName(), user, info.getThumbnails().getLast().getUrl()); List videos = new ObjectArrayList<>(info.getRelatedItems()); @@ -451,7 +451,7 @@ public class AuthPlaylistHandlers { var channel = channelMap.get(channelId); - playlist.getVideos().add(videoMap.computeIfAbsent(videoId, (key) -> new PlaylistVideo(videoId, video.getName(), video.getThumbnailUrl(), video.getDuration(), channel))); + playlist.getVideos().add(videoMap.computeIfAbsent(videoId, (key) -> new PlaylistVideo(videoId, video.getName(), video.getThumbnails().getLast().getUrl(), video.getDuration(), channel))); }); var tr = s.beginTransaction(); diff --git a/src/main/java/me/kavin/piped/utils/CollectionUtils.java b/src/main/java/me/kavin/piped/utils/CollectionUtils.java index bd09132..ea4efe8 100644 --- a/src/main/java/me/kavin/piped/utils/CollectionUtils.java +++ b/src/main/java/me/kavin/piped/utils/CollectionUtils.java @@ -71,7 +71,7 @@ public class CollectionUtils { return new Streams(info.getName(), info.getDescription().getContent(), info.getTextualUploadDate(), info.getUploaderName(), substringYouTube(info.getUploaderUrl()), - rewriteURL(info.getUploaderAvatarUrl()), rewriteURL(info.getThumbnailUrl()), info.getDuration(), + getLastThumbnail(info.getUploaderAvatars()), getLastThumbnail(info.getThumbnails()), info.getDuration(), info.getViewCount(), info.getLikeCount(), info.getDislikeCount(), info.getUploaderSubscriberCount(), info.isUploaderVerified(), audioStreams, videoStreams, relatedStreams, subtitles, livestream, rewriteVideoURL(info.getHlsUrl()), rewriteVideoURL(info.getDashMpdUrl()), null, info.getCategory(), info.getLicence(), @@ -101,9 +101,9 @@ public class CollectionUtils { StreamInfoItem item = (StreamInfoItem) o; return new StreamItem(substringYouTube(item.getUrl()), item.getName(), - rewriteURL(item.getThumbnailUrl()), + getLastThumbnail(item.getThumbnails()), item.getUploaderName(), substringYouTube(item.getUploaderUrl()), - rewriteURL(item.getUploaderAvatarUrl()), item.getTextualUploadDate(), + getLastThumbnail(item.getUploaderAvatars()), item.getTextualUploadDate(), item.getShortDescription(), item.getDuration(), item.getViewCount(), item.getUploadDate() != null ? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli() : -1, @@ -115,7 +115,7 @@ public class CollectionUtils { PlaylistInfoItem item = (PlaylistInfoItem) o; return new PlaylistItem(substringYouTube(item.getUrl()), item.getName(), - rewriteURL(item.getThumbnailUrl()), + getLastThumbnail(item.getThumbnails()), item.getUploaderName(), substringYouTube(item.getUploaderUrl()), item.isUploaderVerified(), item.getPlaylistType().name(), item.getStreamCount()); @@ -126,7 +126,7 @@ public class CollectionUtils { ChannelInfoItem item = (ChannelInfoItem) o; return new ChannelItem(substringYouTube(item.getUrl()), item.getName(), - rewriteURL(item.getThumbnailUrl()), + getLastThumbnail(item.getThumbnails()), item.getDescription(), item.getSubscriberCount(), item.getStreamCount(), item.isVerified()); } diff --git a/src/main/java/me/kavin/piped/utils/DatabaseHelper.java b/src/main/java/me/kavin/piped/utils/DatabaseHelper.java index 93563cc..a4af627 100644 --- a/src/main/java/me/kavin/piped/utils/DatabaseHelper.java +++ b/src/main/java/me/kavin/piped/utils/DatabaseHelper.java @@ -192,7 +192,7 @@ public class DatabaseHelper { } var channel = new Channel(channelId, StringUtils.abbreviate(info.getName(), 100), - info.getAvatarUrl(), info.isVerified()); + info.getAvatars().isEmpty() ? null : info.getAvatars().getLast().getUrl(), info.isVerified()); try (StatelessSession s = DatabaseSessionFactory.createStatelessSession()) { var tr = s.beginTransaction(); diff --git a/src/main/java/me/kavin/piped/utils/URLUtils.java b/src/main/java/me/kavin/piped/utils/URLUtils.java index 2ec0430..a36f3e9 100644 --- a/src/main/java/me/kavin/piped/utils/URLUtils.java +++ b/src/main/java/me/kavin/piped/utils/URLUtils.java @@ -2,12 +2,14 @@ package me.kavin.piped.utils; import me.kavin.piped.consts.Constants; import org.apache.commons.lang3.StringUtils; +import org.schabi.newpipe.extractor.Image; import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; +import java.util.List; public class URLUtils { @@ -37,6 +39,10 @@ public class URLUtils { return rewriteURL(old, Constants.IMAGE_PROXY_PART); } + public static String getLastThumbnail(final List thumbnails) { + return thumbnails.isEmpty() ? null : rewriteURL(thumbnails.getLast().getUrl()); + } + public static String rewriteVideoURL(final String old) { return rewriteURL(old, Constants.PROXY_PART); } diff --git a/src/main/java/me/kavin/piped/utils/VideoHelpers.java b/src/main/java/me/kavin/piped/utils/VideoHelpers.java index c1dc5cb..a89c413 100644 --- a/src/main/java/me/kavin/piped/utils/VideoHelpers.java +++ b/src/main/java/me/kavin/piped/utils/VideoHelpers.java @@ -50,7 +50,7 @@ public class VideoHelpers { if (!DatabaseHelper.doesVideoExist(s, info.getId())) { Video video = new Video(info.getId(), info.getName(), info.getViewCount(), info.getDuration(), - Math.max(infoTime, time), info.getThumbnailUrl(), info.isShortFormContent(), channel); + Math.max(infoTime, time), info.getThumbnails().getLast().getUrl(), info.isShortFormContent(), channel); insertVideo(video); return; @@ -81,7 +81,7 @@ public class VideoHelpers { boolean isShort = extractor.isShortFormContent() || isShort(extractor.getId()); Video video = new Video(extractor.getId(), extractor.getName(), extractor.getViewCount(), extractor.getLength(), - Math.max(infoTime, time), extractor.getThumbnailUrl(), isShort, channel); + Math.max(infoTime, time), extractor.getThumbnails().getLast().getUrl(), isShort, channel); insertVideo(video); From 3a1ea8bf07dc36386cd00b299532c9ea4a655bbd Mon Sep 17 00:00:00 2001 From: FineFindus Date: Mon, 9 Oct 2023 20:11:52 +0200 Subject: [PATCH 13/16] feat(comments): add creator reply field --- .../java/me/kavin/piped/server/handlers/StreamHandlers.java | 4 ++-- src/main/java/me/kavin/piped/utils/obj/Comment.java | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/me/kavin/piped/server/handlers/StreamHandlers.java b/src/main/java/me/kavin/piped/server/handlers/StreamHandlers.java index c7de8be..9dd6c6e 100644 --- a/src/main/java/me/kavin/piped/server/handlers/StreamHandlers.java +++ b/src/main/java/me/kavin/piped/server/handlers/StreamHandlers.java @@ -344,7 +344,7 @@ public class StreamHandlers { comments.add(new Comment(comment.getUploaderName(), getLastThumbnail(comment.getUploaderAvatars()), comment.getCommentId(), Optional.ofNullable(comment.getCommentText()).map(Description::getContent).orElse(null), comment.getTextualUploadDate(), substringYouTube(comment.getUploaderUrl()), repliespage, comment.getLikeCount(), comment.getReplyCount(), - comment.isHeartedByUploader(), comment.isPinned(), comment.isUploaderVerified())); + comment.isHeartedByUploader(), comment.isPinned(), comment.isUploaderVerified(), comment.hasCreatorReply())); } catch (JsonProcessingException e) { ExceptionHandler.handle(e); } @@ -382,7 +382,7 @@ public class StreamHandlers { comments.add(new Comment(comment.getUploaderName(), getLastThumbnail(comment.getUploaderAvatars()), comment.getCommentId(), Optional.ofNullable(comment.getCommentText()).map(Description::getContent).orElse(null), comment.getTextualUploadDate(), substringYouTube(comment.getUploaderUrl()), repliespage, comment.getLikeCount(), comment.getReplyCount(), - comment.isHeartedByUploader(), comment.isPinned(), comment.isUploaderVerified())); + comment.isHeartedByUploader(), comment.isPinned(), comment.isUploaderVerified(), comment.hasCreatorReply())); } catch (JsonProcessingException e) { ExceptionHandler.handle(e); } diff --git a/src/main/java/me/kavin/piped/utils/obj/Comment.java b/src/main/java/me/kavin/piped/utils/obj/Comment.java index ab68b6c..53bdda2 100644 --- a/src/main/java/me/kavin/piped/utils/obj/Comment.java +++ b/src/main/java/me/kavin/piped/utils/obj/Comment.java @@ -4,10 +4,10 @@ public class Comment { public String author, thumbnail, commentId, commentText, commentedTime, commentorUrl, repliesPage; public int likeCount, replyCount; - public boolean hearted, pinned, verified; + public boolean hearted, pinned, verified, creatorReplied; public Comment(String author, String thumbnail, String commentId, String commentText, String commentedTime, - String commentorUrl, String repliesPage, int likeCount, int replyCount, boolean hearted, boolean pinned, boolean verified) { + String commentorUrl, String repliesPage, int likeCount, int replyCount, boolean hearted, boolean pinned, boolean verified, boolean creatorReplied) { this.author = author; this.thumbnail = thumbnail; this.commentId = commentId; @@ -20,5 +20,6 @@ public class Comment { this.hearted = hearted; this.pinned = pinned; this.verified = verified; + this.creatorReplied = creatorReplied; } } From f1f4fd18b343ed6f30ed82109284d1db92d9e08b Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Thu, 12 Oct 2023 18:07:42 +0100 Subject: [PATCH 14/16] Re-enable eclipse temurin docker builds. See https://github.com/adoptium/temurin/issues/8 --- .github/workflows/docker-build-test.yml | 2 +- .github/workflows/docker-build.yml | 4 ++-- Dockerfile | 4 ++-- Dockerfile.ci | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker-build-test.yml b/.github/workflows/docker-build-test.yml index ae9f905..f311630 100644 --- a/.github/workflows/docker-build-test.yml +++ b/.github/workflows/docker-build-test.yml @@ -22,7 +22,7 @@ jobs: - testing/docker-compose.cockroachdb.yml - testing/docker-compose.yugabytedb.yml dockerfile: - #- Dockerfile.ci + - Dockerfile.ci - Dockerfile.azul.ci #- Dockerfile.openj9.ci - Dockerfile.graalvm-jvm.ci diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index a7ea2ba..79ebb29 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -19,8 +19,8 @@ jobs: include: # - image: 1337kavin/piped:openj9 # dockerfile: ./Dockerfile.openj9.ci -# - image: 1337kavin/piped:hotspot -# dockerfile: ./Dockerfile.ci + - image: 1337kavin/piped:hotspot + dockerfile: ./Dockerfile.ci - image: 1337kavin/piped:latest,1337kavin/piped:azul-zulu dockerfile: ./Dockerfile.azul.ci - image: 1337kavin/piped:graalvm-jvm diff --git a/Dockerfile b/Dockerfile index 26146e3..039a721 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM eclipse-temurin:17-jdk AS build +FROM eclipse-temurin:21-jdk AS build WORKDIR /app/ @@ -7,7 +7,7 @@ COPY . /app/ RUN --mount=type=cache,target=/root/.gradle/caches/ \ ./gradlew shadowJar -FROM eclipse-temurin:17-jre +FROM eclipse-temurin:21-jre RUN --mount=type=cache,target=/var/cache/apt/ \ apt-get update && \ diff --git a/Dockerfile.ci b/Dockerfile.ci index adc4949..43ce2e6 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,4 +1,4 @@ -FROM eclipse-temurin:17-jre +FROM eclipse-temurin:21-jre RUN --mount=type=cache,target=/var/cache/apt/ \ apt-get update && \ From 57276a823c7b7f6d4950f606f5f22850dca36f58 Mon Sep 17 00:00:00 2001 From: FineFindus Date: Wed, 25 Oct 2023 14:08:32 +0200 Subject: [PATCH 15/16] feat(comments): add isChannelOwner field --- .../java/me/kavin/piped/server/handlers/StreamHandlers.java | 4 ++-- src/main/java/me/kavin/piped/utils/obj/Comment.java | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/me/kavin/piped/server/handlers/StreamHandlers.java b/src/main/java/me/kavin/piped/server/handlers/StreamHandlers.java index 9dd6c6e..4354265 100644 --- a/src/main/java/me/kavin/piped/server/handlers/StreamHandlers.java +++ b/src/main/java/me/kavin/piped/server/handlers/StreamHandlers.java @@ -344,7 +344,7 @@ public class StreamHandlers { comments.add(new Comment(comment.getUploaderName(), getLastThumbnail(comment.getUploaderAvatars()), comment.getCommentId(), Optional.ofNullable(comment.getCommentText()).map(Description::getContent).orElse(null), comment.getTextualUploadDate(), substringYouTube(comment.getUploaderUrl()), repliespage, comment.getLikeCount(), comment.getReplyCount(), - comment.isHeartedByUploader(), comment.isPinned(), comment.isUploaderVerified(), comment.hasCreatorReply())); + comment.isHeartedByUploader(), comment.isPinned(), comment.isUploaderVerified(), comment.hasCreatorReply(), comment.isChannelOwner())); } catch (JsonProcessingException e) { ExceptionHandler.handle(e); } @@ -382,7 +382,7 @@ public class StreamHandlers { comments.add(new Comment(comment.getUploaderName(), getLastThumbnail(comment.getUploaderAvatars()), comment.getCommentId(), Optional.ofNullable(comment.getCommentText()).map(Description::getContent).orElse(null), comment.getTextualUploadDate(), substringYouTube(comment.getUploaderUrl()), repliespage, comment.getLikeCount(), comment.getReplyCount(), - comment.isHeartedByUploader(), comment.isPinned(), comment.isUploaderVerified(), comment.hasCreatorReply())); + comment.isHeartedByUploader(), comment.isPinned(), comment.isUploaderVerified(), comment.hasCreatorReply(), comment.isChannelOwner())); } catch (JsonProcessingException e) { ExceptionHandler.handle(e); } diff --git a/src/main/java/me/kavin/piped/utils/obj/Comment.java b/src/main/java/me/kavin/piped/utils/obj/Comment.java index 53bdda2..d6f42aa 100644 --- a/src/main/java/me/kavin/piped/utils/obj/Comment.java +++ b/src/main/java/me/kavin/piped/utils/obj/Comment.java @@ -4,10 +4,10 @@ public class Comment { public String author, thumbnail, commentId, commentText, commentedTime, commentorUrl, repliesPage; public int likeCount, replyCount; - public boolean hearted, pinned, verified, creatorReplied; + public boolean hearted, pinned, verified, creatorReplied, channelOwner; public Comment(String author, String thumbnail, String commentId, String commentText, String commentedTime, - String commentorUrl, String repliesPage, int likeCount, int replyCount, boolean hearted, boolean pinned, boolean verified, boolean creatorReplied) { + String commentorUrl, String repliesPage, int likeCount, int replyCount, boolean hearted, boolean pinned, boolean verified, boolean creatorReplied, boolean channelOwner) { this.author = author; this.thumbnail = thumbnail; this.commentId = commentId; @@ -21,5 +21,6 @@ public class Comment { this.pinned = pinned; this.verified = verified; this.creatorReplied = creatorReplied; + this.channelOwner = channelOwner; } } From fb13256c96a0948ea9681840e63a5f7ae5884e49 Mon Sep 17 00:00:00 2001 From: Kavin <20838718+FireMasterK@users.noreply.github.com> Date: Wed, 25 Oct 2023 14:32:20 +0100 Subject: [PATCH 16/16] Bump extractor version. --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index b87f662..5a58433 100644 --- a/build.gradle +++ b/build.gradle @@ -16,7 +16,7 @@ dependencies { implementation 'it.unimi.dsi:fastutil-core:8.5.12' implementation 'commons-codec:commons-codec:1.16.0' implementation 'org.bouncycastle:bcprov-jdk15on:1.70' - implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:48beff184a9792c4787cfa05fce577c3adf89f56' + implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:8cf9a4aef0919df2ef1baafd30ab5bfefefc0844' implementation 'com.github.FireMasterK:nanojson:9f4af3b739cc13f3d0d9d4b758bbe2b2ae7119d7' implementation 'com.fasterxml.jackson.core:jackson-core:2.15.2' implementation 'com.fasterxml.jackson.core:jackson-annotations:2.15.2'