and though bugs are the bane of my existence, rest assured the wretched thing will get the best of care here

...
 
Commits (81)
......@@ -32,7 +32,9 @@ jobs:
- name: Build and run Tests
run: |
if [[ $GITHUB_EVENT_NAME == 'schedule' ]]; then
echo running with real downloader
./gradlew check --stacktrace -Ddownloader=REAL
else
echo running with mock downloader
./gradlew check --stacktrace -Ddownloader=MOCK
fi
......@@ -11,7 +11,7 @@ NewPipe Extractor is available at JitPack's Maven repo.
If you're using Gradle, you could add NewPipe Extractor as a dependency with the following steps:
1. Add `maven { url 'https://jitpack.io' }` to the `repositories` in your `build.gradle`.
2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.10'`the `dependencies` in your `build.gradle`. Replace `v0.20.10` with the latest release.
2. Add `implementation 'com.github.TeamNewPipe:NewPipeExtractor:v0.20.11'`the `dependencies` in your `build.gradle`. Replace `v0.20.11` with the latest release.
**Note:** To use NewPipe Extractor in projects with a `minSdkVersion` below 26, [API desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) is required.
......
......@@ -2,10 +2,13 @@ allprojects {
apply plugin: 'java-library'
apply plugin: 'maven'
compileJava.options.encoding = 'UTF-8'
compileTestJava.options.encoding = 'UTF-8'
sourceCompatibility = 1.8
targetCompatibility = 1.8
version 'v0.20.10'
version 'v0.20.11'
group 'com.github.TeamNewPipe'
repositories {
......
......@@ -15,6 +15,6 @@ dependencies {
implementation 'org.nibor.autolink:autolink:0.10.0'
testImplementation 'junit:junit:4.13.1'
testImplementation "com.squareup.okhttp3:okhttp:3.12.11"
testImplementation "com.squareup.okhttp3:okhttp:3.12.13"
testImplementation 'com.google.code.gson:gson:2.8.6'
}
......@@ -40,4 +40,6 @@ public abstract class ChannelExtractor extends ListExtractor<StreamInfoItem> {
public abstract String getParentChannelName() throws ParsingException;
public abstract String getParentChannelUrl() throws ParsingException;
public abstract String getParentChannelAvatarUrl() throws ParsingException;
public abstract boolean isVerified() throws ParsingException;
}
......@@ -27,7 +27,7 @@ public class ChannelInfoItem extends InfoItem {
private String description;
private long subscriberCount = -1;
private long streamCount = -1;
private boolean verified = false;
public ChannelInfoItem(int serviceId, String url, String name) {
super(InfoType.CHANNEL, serviceId, url, name);
......@@ -56,4 +56,12 @@ public class ChannelInfoItem extends InfoItem {
public void setStreamCount(long stream_count) {
this.streamCount = stream_count;
}
public boolean isVerified() {
return verified;
}
public void setVerified(boolean verified) {
this.verified = verified;
}
}
......@@ -27,5 +27,8 @@ public interface ChannelInfoItemExtractor extends InfoItemExtractor {
String getDescription() throws ParsingException;
long getSubscriberCount() throws ParsingException;
long getStreamCount() throws ParsingException;
boolean isVerified() throws ParsingException;
}
......@@ -59,6 +59,12 @@ public class ChannelInfoItemsCollector extends InfoItemsCollector<ChannelInfoIte
} catch (Exception e) {
addError(e);
}
try {
resultItem.setVerified(extractor.isVerified());
} catch (Exception e) {
addError(e);
}
return resultItem;
}
}
......@@ -12,6 +12,7 @@ public class CommentsInfoItem extends InfoItem {
private String uploaderName;
private String uploaderAvatarUrl;
private String uploaderUrl;
private boolean uploaderVerified;
private String textualUploadDate;
@Nullable
private DateWrapper uploadDate;
......@@ -103,4 +104,12 @@ public class CommentsInfoItem extends InfoItem {
public void setPinned(boolean pinned) {
this.pinned = pinned;
}
public void setUploaderVerified(boolean uploaderVerified) {
this.uploaderVerified = uploaderVerified;
}
public boolean isUploaderVerified() {
return uploaderVerified;
}
}
......@@ -53,4 +53,9 @@ public interface CommentsInfoItemExtractor extends InfoItemExtractor {
* Whether the comment is pinned
*/
boolean isPinned() throws ParsingException;
/**
* Whether the uploader is verified by the service
*/
boolean isUploaderVerified() throws ParsingException;
}
......@@ -14,6 +14,8 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class KioskList {
public interface KioskExtractorFactory {
......@@ -70,7 +72,7 @@ public class KioskList {
public KioskExtractor getDefaultKioskExtractor(Page nextPage, Localization localization)
throws ExtractionException, IOException {
if (defaultKiosk != null && !defaultKiosk.equals("")) {
if (!isNullOrEmpty(defaultKiosk)) {
return getExtractorById(defaultKiosk, nextPage, localization);
} else {
if (!kioskList.isEmpty()) {
......
......@@ -5,6 +5,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.util.ArrayList;
import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory {
///////////////////////////////////
......@@ -12,7 +14,7 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory {
///////////////////////////////////
@Override
public abstract String getUrl(String querry, List<String> contentFilter, String sortFilter) throws ParsingException;
public abstract String getUrl(String query, List<String> contentFilter, String sortFilter) throws ParsingException;
public String getSearchString(String url) {
return "";
......@@ -28,14 +30,14 @@ public abstract class SearchQueryHandlerFactory extends ListLinkHandlerFactory {
}
@Override
public SearchQueryHandler fromQuery(String querry,
public SearchQueryHandler fromQuery(String query,
List<String> contentFilter,
String sortFilter) throws ParsingException {
return new SearchQueryHandler(super.fromQuery(querry, contentFilter, sortFilter));
return new SearchQueryHandler(super.fromQuery(query, contentFilter, sortFilter));
}
public SearchQueryHandler fromQuery(String querry) throws ParsingException {
return fromQuery(querry, new ArrayList<String>(0), "");
public SearchQueryHandler fromQuery(String query) throws ParsingException {
return fromQuery(query, new ArrayList<String>(0), EMPTY_STRING);
}
/**
......
......@@ -20,6 +20,7 @@ public abstract class PlaylistExtractor extends ListExtractor<StreamInfoItem> {
public abstract String getUploaderUrl() throws ParsingException;
public abstract String getUploaderName() throws ParsingException;
public abstract String getUploaderAvatarUrl() throws ParsingException;
public abstract boolean isUploaderVerified() throws ParsingException;
public abstract long getStreamCount() throws ParsingException;
......
......@@ -4,7 +4,6 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
......@@ -17,9 +16,8 @@ import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConfe
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import java.io.IOException;
import javax.annotation.Nonnull;
import java.io.IOException;
public class MediaCCCConferenceExtractor extends ChannelExtractor {
private JsonObject conferenceData;
......@@ -69,6 +67,11 @@ public class MediaCCCConferenceExtractor extends ChannelExtractor {
return "";
}
@Override
public boolean isVerified() throws ParsingException {
return false;
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() {
......
......@@ -126,6 +126,11 @@ public class MediaCCCLiveStreamExtractor extends StreamExtractor {
return conference.getString("conference");
}
@Override
public boolean isUploaderVerified() throws ParsingException {
return false;
}
@Nonnull
@Override
public String getUploaderAvatarUrl() {
......@@ -180,7 +185,7 @@ public class MediaCCCLiveStreamExtractor extends StreamExtractor {
for (int s = 0; s < room.getArray("streams").size(); s++) {
final JsonObject stream = room.getArray("streams").getObject(s);
if (stream.getString("type").equals("audio")) {
for (final String type :stream.getObject("urls").keySet()) {
for (final String type : stream.getObject("urls").keySet()) {
final JsonObject url = stream.getObject("urls").getObject(type);
audioStreams.add(new AudioStream(url.getString("url"), MediaFormat.getFromSuffix(type), -1));
}
......@@ -197,7 +202,7 @@ public class MediaCCCLiveStreamExtractor extends StreamExtractor {
if (stream.getString("type").equals("video")) {
final String resolution = stream.getArray("videoSize").getInt(0) + "x"
+ stream.getArray("videoSize").getInt(1);
for (final String type :stream.getObject("urls").keySet()) {
for (final String type : stream.getObject("urls").keySet()) {
if (!type.equals("hls")) {
final JsonObject url = stream.getObject("urls").getObject(type);
videoStreams.add(new VideoStream(
......@@ -218,7 +223,7 @@ public class MediaCCCLiveStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public List<SubtitlesStream> getSubtitlesDefault(){
public List<SubtitlesStream> getSubtitlesDefault() {
return Collections.emptyList();
}
......
......@@ -73,6 +73,11 @@ public class MediaCCCLiveStreamKioskExtractor implements StreamInfoItemExtractor
return "https://media.ccc.de/c/" + conferenceInfo.getString("slug");
}
@Override
public boolean isUploaderVerified() throws ParsingException {
return false;
}
@Nullable
@Override
public String getTextualUploadDate() throws ParsingException {
......
......@@ -68,6 +68,11 @@ public class MediaCCCRecentKioskExtractor implements StreamInfoItemExtractor {
.getUrl(); // web URL
}
@Override
public boolean isUploaderVerified() throws ParsingException {
return false;
}
@Nullable
@Override
public String getTextualUploadDate() throws ParsingException {
......
......@@ -142,6 +142,11 @@ public class MediaCCCSearchExtractor extends SearchExtractor {
return item.getStreamCount();
}
@Override
public boolean isVerified() throws ParsingException {
return false;
}
@Override
public String getName() {
return item.getName();
......
......@@ -4,7 +4,6 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.StreamingService;
......@@ -13,26 +12,14 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCStreamLinkHandlerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import org.schabi.newpipe.extractor.stream.*;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.*;
public class MediaCCCStreamExtractor extends StreamExtractor {
private JsonObject data;
......@@ -109,6 +96,11 @@ public class MediaCCCStreamExtractor extends StreamExtractor {
.replaceFirst("https://(api\\.)?media\\.ccc\\.de/public/conferences/", "");
}
@Override
public boolean isUploaderVerified() throws ParsingException {
return false;
}
@Nonnull
@Override
public String getUploaderAvatarUrl() {
......
......@@ -28,6 +28,11 @@ public class MediaCCCConferenceInfoItemExtractor implements ChannelInfoItemExtra
return ListExtractor.ITEM_COUNT_UNKNOWN;
}
@Override
public boolean isVerified() throws ParsingException {
return false;
}
@Override
public String getName() throws ParsingException {
return conference.getString("title");
......
......@@ -46,6 +46,11 @@ public class MediaCCCStreamInfoItemExtractor implements StreamInfoItemExtractor
return event.getString("conference_url");
}
@Override
public boolean isUploaderVerified() throws ParsingException {
return false;
}
@Nullable
@Override
public String getTextualUploadDate() {
......
......@@ -7,6 +7,8 @@ import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
public static final String ALL = "all";
public static final String CONFERENCES = "conferences";
......@@ -31,7 +33,7 @@ public class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory
final String sortFilter) throws ParsingException {
try {
return "https://media.ccc.de/public/events/search?q="
+ URLEncoder.encode(query, "UTF-8");
+ URLEncoder.encode(query, UTF_8);
} catch (UnsupportedEncodingException e) {
throw new ParsingException("Could not create search string with query: " + query, e);
}
......
......@@ -83,6 +83,11 @@ public class PeertubeAccountExtractor extends ChannelExtractor {
return "";
}
@Override
public boolean isVerified() throws ParsingException {
return false;
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
......
......@@ -90,6 +90,11 @@ public class PeertubeChannelExtractor extends ChannelExtractor {
return baseUrl + value;
}
@Override
public boolean isVerified() throws ParsingException {
return false;
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
......
package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.ServiceList;
......@@ -13,6 +12,8 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import java.util.Objects;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
private final JsonObject item;
private final String url;
......@@ -68,7 +69,7 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
final Document doc = Jsoup.parse(htmlText);
return doc.body().text();
} catch (Exception e) {
return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", "");
return htmlText.replaceAll("(?s)<[^>]*>(\\s*<[^>]*>)*", EMPTY_STRING);
}
}
......@@ -98,6 +99,11 @@ public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtrac
return false;
}
@Override
public boolean isUploaderVerified() throws ParsingException {
return false;
}
@Override
public String getUploaderName() throws ParsingException {
return JsonUtils.getString(item, "account.name") + "@" + JsonUtils.getString(item, "account.host");
......
......@@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.peertube.extractors;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
......@@ -17,14 +16,10 @@ import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.collectStreamsFrom;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class PeertubePlaylistExtractor extends PlaylistExtractor {
......@@ -59,6 +54,11 @@ public class PeertubePlaylistExtractor extends PlaylistExtractor {
return getBaseUrl() + playlistInfo.getObject("ownerAccount").getObject("avatar").getString("path");
}
@Override
public boolean isUploaderVerified() throws ParsingException {
return false;
}
@Override
public long getStreamCount() {
return playlistInfo.getLong("videosLength");
......
......@@ -18,15 +18,7 @@ import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeSearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
......@@ -40,6 +32,8 @@ import java.util.Collections;
import java.util.List;
import java.util.Locale;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class PeertubeStreamExtractor extends StreamExtractor {
private final String baseUrl;
private JsonObject json;
......@@ -154,6 +148,11 @@ public class PeertubeStreamExtractor extends StreamExtractor {
return JsonUtils.getString(json, "account.displayName");
}
@Override
public boolean isUploaderVerified() throws ParsingException {
return false;
}
@Nonnull
@Override
public String getUploaderAvatarUrl() {
......@@ -322,7 +321,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
params.append("start=0&count=8&sort=-createdAt");
for (final String tag : tags) {
params.append("&tagsOneOf=");
params.append(URLEncoder.encode(tag, "UTF-8"));
params.append(URLEncoder.encode(tag, UTF_8));
}
return url + "?" + params.toString();
}
......
......@@ -54,6 +54,11 @@ public class PeertubeStreamInfoItemExtractor implements StreamInfoItemExtractor
.fromId("accounts/" + name + "@" + host, baseUrl).getUrl();
}
@Override
public boolean isUploaderVerified() throws ParsingException {
return false;
}
@Override
public String getUploaderName() throws ParsingException {
return JsonUtils.getString(item, "account.displayName");
......
......@@ -8,9 +8,10 @@ import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
public static final String CHARSET_UTF_8 = "UTF-8";
public static final String VIDEOS = "videos";
public static final String SEPIA_VIDEOS = "sepia_videos"; // sepia is the global index
public static final String SEPIA_BASE_URL = "https://sepiasearch.org";
......@@ -35,7 +36,7 @@ public class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
public String getUrl(String searchString, List<String> contentFilters, String sortFilter, String baseUrl) throws ParsingException {
try {
final String url = baseUrl + SEARCH_ENDPOINT
+ "?search=" + URLEncoder.encode(searchString, CHARSET_UTF_8);
+ "?search=" + URLEncoder.encode(searchString, UTF_8);
return url;
} catch (UnsupportedEncodingException e) {
......
......@@ -37,12 +37,10 @@ import java.util.List;
import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
import static org.schabi.newpipe.extractor.utils.Utils.*;
public class SoundcloudParsingHelper {
private static final String HARDCODED_CLIENT_ID = "H2c34Q0E7hftqnuDHGsk88DbNqhYpgMm"; // Updated on 24/06/20
private static final String HARDCODED_CLIENT_ID = "Kl9G8jQT22DxqatQk09IjWRujGlut5Vd"; // Updated on 04/03/21
private static String clientId;
private SoundcloudParsingHelper() {
......@@ -69,7 +67,7 @@ public class SoundcloudParsingHelper {
Collections.reverse(possibleScripts);
final HashMap<String, List<String>> headers = new HashMap<>();
headers.put("Range", singletonList("bytes=0-16384"));
headers.put("Range", singletonList("bytes=0-50000"));
for (Element element : possibleScripts) {
final String srcUrl = element.attr("src");
......@@ -117,7 +115,7 @@ public class SoundcloudParsingHelper {
*/
public static JsonObject resolveFor(Downloader downloader, String url) throws IOException, ExtractionException {
String apiUrl = "https://api-v2.soundcloud.com/resolve"
+ "?url=" + URLEncoder.encode(url, "UTF-8")
+ "?url=" + URLEncoder.encode(url, UTF_8)
+ "&client_id=" + clientId();
try {
......@@ -136,7 +134,7 @@ public class SoundcloudParsingHelper {
public static String resolveUrlWithEmbedPlayer(String apiUrl) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
+ URLEncoder.encode(apiUrl, "UTF-8"), SoundCloud.getLocalization()).responseBody();
+ URLEncoder.encode(apiUrl, UTF_8), SoundCloud.getLocalization()).responseBody();
return Jsoup.parse(response).select("link[rel=\"canonical\"]").first().attr("abs:href");
}
......@@ -148,17 +146,17 @@ public class SoundcloudParsingHelper {
*/
public static String resolveIdWithEmbedPlayer(String urlString) throws IOException, ReCaptchaException, ParsingException {
// Remove the tailing slash from URLs due to issues with the SoundCloud API
if (urlString.charAt(urlString.length() -1) == '/') urlString = urlString.substring(0, urlString.length()-1);
if (urlString.charAt(urlString.length() - 1) == '/') urlString = urlString.substring(0, urlString.length() - 1);
URL url;
try {
url = Utils.stringToURL(urlString);
} catch (MalformedURLException e){
} catch (MalformedURLException e) {
throw new IllegalArgumentException("The given URL is not valid");
}
String response = NewPipe.getDownloader().get("https://w.soundcloud.com/player/?url="
+ URLEncoder.encode(url.toString(), "UTF-8"), SoundCloud.getLocalization()).responseBody();
+ URLEncoder.encode(url.toString(), UTF_8), SoundCloud.getLocalization()).responseBody();
// handle playlists / sets different and get playlist id via uir field in JSON
if (url.getPath().contains("/sets/") && !url.getPath().endsWith("/sets"))
return Parser.matchGroup1("\"uri\":\\s*\"https:\\/\\/api\\.soundcloud\\.com\\/playlists\\/((\\d)*?)\"", response);
......@@ -240,10 +238,14 @@ public class SoundcloudParsingHelper {
* @return the next streams url, empty if don't have
*/
public static String getStreamsFromApi(StreamInfoItemsCollector collector, String apiUrl, boolean charts) throws IOException, ReCaptchaException, ParsingException {
String response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization()).responseBody();
final Response response = NewPipe.getDownloader().get(apiUrl, SoundCloud.getLocalization());
if (response.responseCode() >= 400) {
throw new IOException("Could not get streams from API, HTTP " + response.responseCode());
}
JsonObject responseObject;
try {
responseObject = JsonParser.object().from(response);
responseObject = JsonParser.object().from(response.responseBody());
} catch (JsonParserException e) {
throw new ParsingException("Could not parse json response", e);
}
......
......@@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
......@@ -15,11 +14,10 @@ import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import java.io.IOException;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@SuppressWarnings("WeakerAccess")
......@@ -98,6 +96,11 @@ public class SoundcloudChannelExtractor extends ChannelExtractor {
return "";
}
@Override
public boolean isVerified() throws ParsingException {
return user.getBoolean("verified");
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
......
......@@ -3,7 +3,7 @@ package org.schabi.newpipe.extractor.services.soundcloud.extractors;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor {
......@@ -40,6 +40,11 @@ public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtrac
return itemObject.getLong("track_count");
}
@Override
public boolean isVerified() {
return itemObject.getBoolean("verified");
}
@Override
public String getDescription() {
return itemObject.getString("description", EMPTY_STRING);
......
......@@ -6,13 +6,13 @@ import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.kiosk.KioskExtractor;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.ContentCountry;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import java.io.IOException;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
......@@ -61,10 +61,20 @@ public class SoundcloudChartsExtractor extends KioskExtractor<StreamInfoItem> {
apiUrl += "&kind=trending";
}
final String contentCountry = SoundCloud.getContentCountry().getCountryCode();
apiUrl += "&region=soundcloud:regions:" + contentCountry;
final ContentCountry contentCountry = SoundCloud.getContentCountry();
String apiUrlWithRegion = null;
if (getService().getSupportedCountries().contains(contentCountry)) {
apiUrlWithRegion = apiUrl + "&region=soundcloud:regions:" + contentCountry.getCountryCode();
}
final String nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true);
String nextPageUrl;
try {
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrlWithRegion == null ? apiUrl : apiUrlWithRegion, true);
} catch (IOException e) {
// Request to other region may be geo-restricted. See https://github.com/TeamNewPipe/NewPipeExtractor/issues/537
// we retry without the specified region.
nextPageUrl = SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl, true);
}
return new InfoItemsPage<>(collector, new Page(nextPageUrl));
}
......
......@@ -6,9 +6,8 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import java.util.Objects;
import javax.annotation.Nullable;
import java.util.Objects;
public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
private JsonObject json;
......@@ -49,6 +48,11 @@ public class SoundcloudCommentsInfoItemExtractor implements CommentsInfoItemExtr
return false;
}
@Override
public boolean isUploaderVerified() throws ParsingException {
return json.getObject("user").getBoolean("verified");
}
@Override
public String getUploaderUrl() {
return json.getObject("user").getString("permalink_url");
......
......@@ -111,6 +111,11 @@ public class SoundcloudPlaylistExtractor extends PlaylistExtractor {
return SoundcloudParsingHelper.getAvatarUrl(playlist);
}
@Override
public boolean isUploaderVerified() throws ParsingException {
return playlist.getObject("user").getBoolean("verified");
}
@Override
public long getStreamCount() {
return playlist.getLong("track_count");
......
......@@ -4,7 +4,7 @@ import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
......
......@@ -4,13 +4,7 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.InfoItemExtractor;
import org.schabi.newpipe.extractor.InfoItemsCollector;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
......@@ -19,6 +13,7 @@ import org.schabi.newpipe.extractor.search.InfoItemsSearchCollector;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.utils.Parser;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
......@@ -26,10 +21,8 @@ import java.net.URL;
import java.util.Collections;
import java.util.List;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudSearchQueryHandlerFactory.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class SoundcloudSearchExtractor extends SearchExtractor {
......@@ -50,7 +43,7 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
return false;
}
@Nonnull
@Nonnull
@Override
public List<MetaInfo> getMetaInfo() {
return Collections.emptyList();
......
......@@ -4,7 +4,6 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.NewPipe;
......@@ -17,15 +16,10 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.stream.*;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
......@@ -34,11 +28,7 @@ import java.util.Collections;
import java.util.List;
import java.util.Locale;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.utils.Utils.*;
public class SoundcloudStreamExtractor extends StreamExtractor {
private JsonObject track;
......@@ -60,7 +50,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Nonnull
@Override
public String getId() {
return track.getInt("id") + "";
return track.getInt("id") + EMPTY_STRING;
}
@Nonnull
......@@ -73,8 +63,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Override
public String getTextualUploadDate() {
return track.getString("created_at")
.replace("T"," ")
.replace("Z", "");
.replace("T", " ")
.replace("Z", EMPTY_STRING);
}
@Nonnull
......@@ -141,6 +131,11 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
return SoundcloudParsingHelper.getUploaderName(track);
}
@Override
public boolean isUploaderVerified() throws ParsingException {
return track.getObject("user").getBoolean("verified");
}
@Nonnull
@Override
public String getUploaderAvatarUrl() {
......@@ -232,7 +227,7 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
private static String urlEncode(String value) {
try {
return URLEncoder.encode(value, "UTF-8");
return URLEncoder.encode(value, UTF_8);
} catch (UnsupportedEncodingException e) {
throw new IllegalStateException(e);
}
......
......@@ -7,7 +7,7 @@ import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;
public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtractor {
......@@ -43,6 +43,11 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto
return replaceHttpWithHttps(itemObject.getObject("user").getString("permalink_url"));
}
@Override
public boolean isUploaderVerified() throws ParsingException {
return itemObject.getObject("user").getBoolean("verified");
}
@Override
public String getTextualUploadDate() {
return itemObject.getString("created_at");
......@@ -64,8 +69,7 @@ public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtracto
if (artworkUrl.isEmpty()) {
artworkUrl = itemObject.getObject("user").getString("avatar_url");
}
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
return artworkUrlBetterResolution;
return artworkUrl.replace("large.jpg", "crop.jpg");
}
@Override
......
......@@ -17,9 +17,9 @@ import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public static final String CHARSET_UTF_8 = "UTF-8";
public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
public SoundcloudSuggestionExtractor(StreamingService service) {
super(service);
......@@ -32,7 +32,7 @@ public class SoundcloudSuggestionExtractor extends SuggestionExtractor {
Downloader dl = NewPipe.getDownloader();
String url = "https://api-v2.soundcloud.com/search/queries"
+ "?q=" + URLEncoder.encode(query, CHARSET_UTF_8)
+ "?q=" + URLEncoder.encode(query, UTF_8)
+ "&client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=10";
......
......@@ -11,8 +11,9 @@ import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
public static final String CHARSET_UTF_8 = "UTF-8";
public static final String TRACKS = "tracks";
public static final String USERS = "users";
......@@ -43,7 +44,7 @@ public class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFacto
}
}
return url + "?q=" + URLEncoder.encode(id, CHARSET_UTF_8)
return url + "?q=" + URLEncoder.encode(id, UTF_8)
+ "&client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=" + ITEMS_PER_PAGE
+ "&offset=0";
......
......@@ -8,7 +8,7 @@ import static org.schabi.newpipe.extractor.services.youtube.ItagItem.ItagType.*;
public class ItagItem {
/**
* List can be found here https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L360
* List can be found here https://github.com/ytdl-org/youtube-dl/blob/9fc5eafb8e384453a49f7cfe73147be491f0b19d/youtube_dl/extractor/youtube.py#L1071
*/
private static final ItagItem[] ITAG_LIST = {
/////////////////////////////////////////////////////
......@@ -155,4 +155,86 @@ public class ItagItem {
public String resolutionString;
public int fps = -1;
// Fields for Dash
private int bitrate;
private int width;
private int height;
private int initStart;
private int initEnd;
private int indexStart;
private int indexEnd;
private String quality;
private String codec;
public int getBitrate() {
return bitrate;
}
public void setBitrate(int bitrate) {
this.bitrate = bitrate;
}
public int getWidth() {
return width;
}
public void setWidth(int width) {
this.width = width;
}
public int getHeight() {
return height;
}
public void setHeight(int height) {
this.height = height;
}
public int getInitStart() {
return initStart;
}
public void setInitStart(int initStart) {
this.initStart = initStart;
}
public int getInitEnd() {
return initEnd;
}
public void setInitEnd(int initEnd) {
this.initEnd = initEnd;
}
public int getIndexStart() {
return indexStart;
}
public void setIndexStart(int indexStart) {
this.indexStart = indexStart;
}
public int getIndexEnd() {
return indexEnd;
}
public void setIndexEnd(int indexEnd) {
this.indexEnd = indexEnd;
}
public String getQuality() {
return quality;
}
public void setQuality(String quality) {
this.quality = quality;
}
public String getCodec() {
return codec;
}
public void setCodec(String codec) {
this.codec = codec;
}
}
package org.schabi.newpipe.extractor.services.youtube;
import com.grack.nanojson.*;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.downloader.Response;
......@@ -10,11 +15,10 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
......@@ -24,11 +28,23 @@ import java.time.LocalDate;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeParseException;
import java.util.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.NewPipe.getDownloader;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.*;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.utils.Utils.join;
/*
* Created by Christian Schabesberger on 02.03.16.
......@@ -118,6 +134,7 @@ public class YoutubeParsingHelper {
/**
* Parses the duration string of the video expecting ":" or "." as separators
*
* @return the duration in seconds
* @throws ParsingException when more than 3 separators are found
*/
......@@ -187,6 +204,7 @@ public class YoutubeParsingHelper {
/**
* Checks if the given playlist id is a YouTube Mix (auto-generated playlist)
* Ids from a YouTube Mix start with "RD"
*
* @param playlistId
* @return Whether given id belongs to a YouTube Mix
*/
......@@ -197,15 +215,18 @@ public class YoutubeParsingHelper {
/**
* Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist)
* Ids from a YouTube Music Mix start with "RDAMVM" or "RDCLAK"
*
* @param playlistId
* @return Whether given id belongs to a YouTube Music Mix
*/
public static boolean isYoutubeMusicMixId(final String playlistId) {
return playlistId.startsWith("RDAMVM") || playlistId.startsWith("RDCLAK");
}
/**
* Checks if the given playlist id is a YouTube Channel Mix (auto-generated playlist)
* Ids from a YouTube channel Mix start with "RDCM"
*
* @return Whether given id belongs to a YouTube Channel Mix
*/
public static boolean isYoutubeChannelMixId(final String playlistId) {
......@@ -214,6 +235,7 @@ public class YoutubeParsingHelper {
/**
* Extracts the video id from the playlist id for Mixes.
*
* @throws ParsingException If the playlistId is a Channel Mix or not a mix.
*/
public static String extractVideoIdFromMixId(final String playlistId) throws ParsingException {
......@@ -305,7 +327,8 @@ public class YoutubeParsingHelper {
clientVersion = contextClientVersion;
break;
}
} catch (Parser.RegexException ignored) { }
} catch (Parser.RegexException ignored) {
}
}
if (!isNullOrEmpty(clientVersion) && !isNullOrEmpty(shortClientVersion)) {
......@@ -317,7 +340,8 @@ public class YoutubeParsingHelper {
} catch (Parser.RegexException e) {
try {
key = Parser.matchGroup1("innertubeApiKey\":\"([0-9a-zA-Z_-]+?)\"", html);
} catch (Parser.RegexException ignored) { }
} catch (Parser.RegexException ignored) {
}
}
}
......@@ -345,11 +369,21 @@ public class YoutubeParsingHelper {
}
/**
* Only use in tests.
* <p>
* <b>Only use in tests.</b>
* </p>
*
* <p>
* Quick-and-dirty solution to reset global state in between test classes.
* </p>
* <p>
* This is needed for the mocks because in order to reach that state a network request has to
* be made. If the global state is not reset and the RecordingDownloader is used,
* then only the first test class has that request recorded. Meaning running the other
* tests with mocks will fail, because the mock is missing.
* </p>
*/
static void resetClientVersionAndKey() {
public static void resetClientVersionAndKey() {
clientVersion = null;
key = null;
}
......@@ -384,7 +418,7 @@ public class YoutubeParsingHelper {
.end()
.value("query", "test")
.value("params", "Eg-KAQwIARAAGAAgACgAMABqChAEEAUQAxAKEAk%3D")
.end().done().getBytes("UTF-8");
.end().done().getBytes(UTF_8);
// @formatter:on
final Map<String, List<String>> headers = new HashMap<>();
......@@ -448,7 +482,7 @@ public class YoutubeParsingHelper {
if (param.split("=")[0].equals("q")) {
String url;
try {
url = URLDecoder.decode(param.split("=")[1], "UTF-8");
url = URLDecoder.decode(param.split("=")[1], UTF_8);
} catch (UnsupportedEncodingException e) {
return null;
}
......@@ -496,6 +530,7 @@ public class YoutubeParsingHelper {
/**
* Get the text from a JSON object that has either a simpleText or a runs array.
*
* @param textObject JSON object to get the text from
* @param html whether to return HTML, by parsing the navigationEndpoint
* @return text in the JSON object or {@code null}
......@@ -614,7 +649,7 @@ public class YoutubeParsingHelper {
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
final Response response = getDownloader().get(url, headers, localization);
return toJsonArray(getValidJsonResponseBody(response));
return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
}
public static JsonArray getJsonResponse(final Page page, final Localization localization)
......@@ -628,15 +663,7 @@ public class YoutubeParsingHelper {
final Response response = getDownloader().get(page.getUrl(), headers, localization);
return toJsonArray(getValidJsonResponseBody(response));
}
public static JsonArray toJsonArray(final String responseBody) throws ParsingException {
try {
return JsonParser.array().from(responseBody);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse JSON", e);
}
return JsonUtils.toJsonArray(getValidJsonResponseBody(response));
}
/**
......@@ -723,7 +750,7 @@ public class YoutubeParsingHelper {
final String title = YoutubeParsingHelper.getTextFromObject(clarificationRenderer.getObject("contentTitle"));
final String text = YoutubeParsingHelper.getTextFromObject(clarificationRenderer.getObject("text"));
if (title == null || text == null) {
if (title == null || text == null) {
throw new ParsingException("Could not extract clarification renderer content");
}
metaInfo.setTitle(title);
......@@ -767,6 +794,7 @@ public class YoutubeParsingHelper {
/**
* Sometimes, YouTube provides URLs which use Google's cache. They look like
* {@code https://webcache.googleusercontent.com/search?q=cache:CACHED_URL}
*
* @param url the URL which might refer to the Google's webcache
* @return the URL which is referring to the original site
*/
......@@ -779,4 +807,31 @@ public class YoutubeParsingHelper {
}
return url;
}
public static boolean isVerified(final JsonArray badges) {
if (Utils.isNullOrEmpty(badges)) {
return false;
}
for (Object badge : badges) {
final String style = ((JsonObject) badge).getObject("metadataBadgeRenderer")
.getString("style");
if (style != null && (style.equals("BADGE_STYLE_TYPE_VERIFIED")
|| style.equals("BADGE_STYLE_TYPE_VERIFIED_ARTIST"))) {
return true;
}
}
return false;
}
public static String unescapeDocument(final String doc) {
return doc
.replaceAll("\\\\x22", "\"")
.replaceAll("\\\\x7b", "{")
.replaceAll("\\\\x7d", "}")
.replaceAll("\\\\x5b", "[")
.replaceAll("\\\\x5d", "]");
}
}
......@@ -2,10 +2,13 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ContentNotSupportedException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
......@@ -15,13 +18,20 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/*
......@@ -216,9 +226,17 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
return "";
}
@Override
public boolean isVerified() throws ParsingException {
final JsonArray badges = initialData.getObject("header").getObject("c4TabbedHeaderRenderer")
.getArray("badges");
return YoutubeParsingHelper.isVerified(badges);
}
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ExtractionException {
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
Page nextPage = null;
......@@ -246,27 +264,44 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
// as they don't deliver enough information on their own (the channel name, for example).
fetchPage();
// @formatter:off
byte[] json = JsonWriter.string()
.object()
.object("context")
.object("client")
.value("clientName", "1")
.value("clientVersion", getClientVersion())
.end()
.end()
.value("continuation", page.getId())
.end()
.done()
.getBytes(UTF_8);
// @formatter:on
StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization());
final Response response = getDownloader().post(page.getUrl(), null, json, getExtractorLocalization());
final JsonObject ajaxJson = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
JsonObject sectionListContinuation = ajaxJson.getObject(1).getObject("response")
.getArray("onResponseReceivedActions").getObject(0).getObject("appendContinuationItemsAction");
JsonObject sectionListContinuation = ajaxJson.getArray("onResponseReceivedActions")
.getObject(0)
.getObject("appendContinuationItemsAction");
final JsonObject continuation = collectStreamsFrom(collector, sectionListContinuation.getArray("continuationItems"));
return new InfoItemsPage<>(collector, getNextPageFrom(continuation));
}
private Page getNextPageFrom(final JsonObject continuations) {
private Page getNextPageFrom(final JsonObject continuations) throws IOException, ExtractionException {
if (isNullOrEmpty(continuations)) {
return null;
}
final JsonObject continuationEndpoint = continuations.getObject("continuationEndpoint");
final String continuation = continuationEndpoint.getObject("continuationCommand").getString("token");
final String clickTrackingParams = continuationEndpoint.getString("clickTrackingParams");
return new Page("https://www.youtube.com/browse_ajax?ctoken=" + continuation
+ "&continuation=" + continuation + "&itct=" + clickTrackingParams);
return new Page("https://www.youtube.com/youtubei/v1/browse?key=" + getKey(),
continuation);
}
/**
......
......@@ -5,6 +5,7 @@ import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.utils.Utils;
......@@ -97,6 +98,11 @@ public class YoutubeChannelInfoItemExtractor implements ChannelInfoItemExtractor
}
}
@Override
public boolean isVerified() throws ParsingException {
return YoutubeParsingHelper.isVerified(channelInfoItem.getArray("ownerBadges"));
}
@Override
public String getDescription() throws ParsingException {
try {
......
......@@ -3,7 +3,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
......@@ -16,9 +15,11 @@ import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
......@@ -27,9 +28,8 @@ import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import javax.annotation.Nonnull;
import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
public class YoutubeCommentsExtractor extends CommentsExtractor {
......@@ -47,7 +47,10 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
@Override
public InfoItemsPage<CommentsInfoItem> getInitialPage() throws IOException, ExtractionException {
final String commentsTokenInside = findValue(responseBody, "commentSectionRenderer", "}");
String commentsTokenInside = findValue(responseBody, "sectionListRenderer", "}");
if (!commentsTokenInside.contains("continuation\":\"")) {
commentsTokenInside = findValue(responseBody, "commentSectionRenderer", "}");
}
final String commentsToken = findValue(commentsTokenInside, "continuation\":\"", "\"");
return getPage(getNextPage(commentsToken));
}
......@@ -129,7 +132,7 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
final Map<String, List<String>> requestHeaders = new HashMap<>();
requestHeaders.put("User-Agent", singletonList(USER_AGENT));
final Response response = downloader.get(getUrl(), requestHeaders, getExtractorLocalization());
responseBody = response.responseBody();
responseBody = YoutubeParsingHelper.unescapeDocument(response.responseBody());
ytClientVersion = findValue(responseBody, "INNERTUBE_CONTEXT_CLIENT_VERSION\":\"", "\"");
ytClientName = Parser.matchGroup1(YT_CLIENT_NAME_PATTERN, responseBody);
}
......@@ -152,23 +155,16 @@ public class YoutubeCommentsExtractor extends CommentsExtractor {
first = false;
else
result.append("&");
result.append(URLEncoder.encode(entry.getKey(), "UTF-8"));
result.append(URLEncoder.encode(entry.getKey(), UTF_8));
result.append("=");
result.append(URLEncoder.encode(entry.getValue(), "UTF-8"));
result.append(URLEncoder.encode(entry.getValue(), UTF_8));
}
return result.toString();
}
private String findValue(String doc, String start, String end) {
final String unescaped = doc
.replaceAll("\\\\x22", "\"")
.replaceAll("\\\\x7b", "{")
.replaceAll("\\\\x7d", "}")
.replaceAll("\\\\x5b", "[")
.replaceAll("\\\\x5d", "]");
final int beginIndex = unescaped.indexOf(start) + start.length();
final int endIndex = unescaped.indexOf(end, beginIndex);
return unescaped.substring(beginIndex, endIndex);
private String findValue(final String doc, final String start, final String end) {
final int beginIndex = doc.indexOf(start) + start.length();
final int endIndex = doc.indexOf(end, beginIndex);
return doc.substring(beginIndex, endIndex);
}
}
......@@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.comments.CommentsInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
......@@ -13,6 +12,7 @@ import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor {
......@@ -46,7 +46,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
try {
return getTextFromObject(JsonUtils.getObject(json, "authorText"));
} catch (Exception e) {
return "";
return EMPTY_STRING;
}
}
......@@ -86,7 +86,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
if (contentText.isEmpty()) {
// completely empty comments as described in
// https://github.com/TeamNewPipe/NewPipeExtractor/issues/380#issuecomment-668808584
return "";
return EMPTY_STRING;
}
final String commentText = getTextFromObject(contentText);
// youtube adds U+FEFF in some comments. eg. https://www.youtube.com/watch?v=Nj4F63E59io<feff>
......@@ -125,12 +125,17 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
return json.has("pinnedCommentBadge");
}
public boolean isUploaderVerified() throws ParsingException {
// impossible to get this information from the mobile layout
return false;
}
@Override
public String getUploaderName() throws ParsingException {
try {
return getTextFromObject(JsonUtils.getObject(json, "authorText"));
} catch (Exception e) {
return "";
return EMPTY_STRING;
}
}
......@@ -139,7 +144,7 @@ public class YoutubeCommentsInfoItemExtractor implements CommentsInfoItemExtract
try {
return "https://youtube.com/channel/" + JsonUtils.getString(json, "authorEndpoint.browseEndpoint.browseId");
} catch (Exception e) {
return "";
return EMPTY_STRING;
}
}
......
......@@ -50,6 +50,11 @@ public class YoutubeFeedInfoItemExtractor implements StreamInfoItemExtractor {
return entryElement.select("author > uri").first().text();
}
@Override
public boolean isUploaderVerified() throws ParsingException {
return false;
}
@Nullable
@Override
public String getTextualUploadDate() {
......
......@@ -15,6 +15,7 @@ import org.schabi.newpipe.extractor.localization.TimeAgoParser;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import java.io.IOException;
import java.util.Collections;
......@@ -27,7 +28,6 @@ import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.toJsonArray;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/**
......@@ -57,7 +57,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
throws IOException, ExtractionException {
final String url = getUrl() + "&pbj=1";
final Response response = getResponse(url, getExtractorLocalization());
final JsonArray ajaxJson = toJsonArray(response.responseBody());
final JsonArray ajaxJson = JsonUtils.toJsonArray(response.responseBody());
initialData = ajaxJson.getObject(3).getObject("response");
playlistData = initialData.getObject("contents").getObject("twoColumnWatchNextResults")
.getObject("playlist").getObject("playlist");
......@@ -82,8 +82,8 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
try {
//fallback to thumbnail of current video. Always the case for channel mix
return getThumbnailUrlFromVideoId(
initialData.getObject("currentVideoEndpoint").getObject("watchEndpoint")
.getString("videoId"));
initialData.getObject("currentVideoEndpoint").getObject("watchEndpoint")
.getString("videoId"));
} catch (final Exception ignored) {
}
throw new ParsingException("Could not get playlist thumbnail", e);
......@@ -113,6 +113,11 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
return "";
}
@Override
public boolean isUploaderVerified() throws ParsingException {
return false;
}
@Override
public long getStreamCount() {
// Auto-generated playlist always start with 25 videos and are endless
......@@ -144,7 +149,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
public InfoItemsPage<StreamInfoItem> getPage(final Page page)
throws ExtractionException, IOException {
if (page == null || isNullOrEmpty(page.getUrl())) {
throw new IllegalArgumentException("Page url is empty or null");
throw new IllegalArgumentException("Page url is empty or null");
}
if (!page.getCookies().containsKey(COOKIE_NAME)) {
throw new IllegalArgumentException("Cooke '" + COOKIE_NAME + "' is missing");
......@@ -180,7 +185,7 @@ public class YoutubeMixPlaylistExtractor extends PlaylistExtractor {
for (final Object stream : streams) {
if (stream instanceof JsonObject) {
final JsonObject streamInfo = ((JsonObject) stream)
.getObject("playlistPanelVideoRenderer");
.getObject("playlistPanelVideoRenderer");
if (streamInfo != null) {
collector.commit(new YoutubeStreamInfoItemExtractor(streamInfo, timeAgoParser));
}
......
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import com.grack.nanojson.*;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page;
......@@ -24,25 +19,16 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nonnull;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_PLAYLISTS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_VIDEOS;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.*;
import static org.schabi.newpipe.extractor.utils.Utils.*;
public class YoutubeMusicSearchExtractor extends SearchExtractor {
private JsonObject initialData;
......@@ -107,7 +93,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
.end()
.value("query", getSearchString())
.value("params", params)
.end().done().getBytes("UTF-8");
.end().done().getBytes(UTF_8);
// @formatter:on
final Map<String, List<String>> headers = new HashMap<>();
......@@ -231,7 +217,7 @@ public class YoutubeMusicSearchExtractor extends SearchExtractor {
.value("enableSafetyMode", false)
.end()
.end()
.end().done().getBytes("UTF-8");
.end().done().getBytes(UTF_8);
// @formatter:on
final Map<String, List<String>> headers = new HashMap<>();
......
......@@ -2,10 +2,12 @@ package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonWriter;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
......@@ -26,9 +28,14 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.utils.JsonUtils.toJsonObject;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@SuppressWarnings("WeakerAccess")
......@@ -82,7 +89,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Override
public String getName() throws ParsingException {
final String name = getTextFromObject(playlistInfo.getObject("title"));
if (name != null && !name.isEmpty()) return name;
if (!isNullOrEmpty(name)) return name;
return initialData.getObject("microformat").getObject("microformatDataRenderer").getString("title");
}
......@@ -138,6 +145,11 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
}
}
@Override
public boolean isUploaderVerified() throws ParsingException {
return false;
}
@Override
public long getStreamCount() throws ParsingException {
try {
......@@ -168,7 +180,7 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
@Nonnull
@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() {
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
Page nextPage = null;
......@@ -205,22 +217,37 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
throw new IllegalArgumentException("Page doesn't contain an URL");
}
// @formatter:off
byte[] json = JsonWriter.string()
.object()
.object("context")
.object("client")
.value("clientName", "1")
.value("clientVersion", getClientVersion())
.end()
.end()
.value("continuation", page.getId())
.end()
.done()
.getBytes(UTF_8);
// @formatter:on
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final JsonArray ajaxJson = getJsonResponse(page.getUrl(), getExtractorLocalization());
final Response response = getDownloader().post(page.getUrl(), null, json, getExtractorLocalization());
final JsonObject ajaxJson = toJsonObject(getValidJsonResponseBody(response));
final JsonArray continuation = ajaxJson.getObject(1)
.getObject("response")
.getArray("onResponseReceivedActions")
.getObject(0)
.getObject("appendContinuationItemsAction")
.getArray("continuationItems");
final JsonArray continuation = ajaxJson.getArray("onResponseReceivedActions")
.getObject(0)
.getObject("appendContinuationItemsAction")
.getArray("continuationItems");
collectStreamsFrom(collector, continuation);
return new InfoItemsPage<>(collector, getNextPageFrom(continuation));
}
private Page getNextPageFrom(final JsonArray contents) {
private Page getNextPageFrom(final JsonArray contents) throws IOException, ExtractionException {
if (isNullOrEmpty(contents)) {
return null;
}
......@@ -228,11 +255,13 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
final JsonObject lastElement = contents.getObject(contents.size() - 1);
if (lastElement.has("continuationItemRenderer")) {
final String continuation = lastElement
.getObject("continuationItemRenderer")
.getObject("continuationEndpoint")
.getObject("continuationCommand")
.getString("token");
return new Page("https://www.youtube.com/browse_ajax?continuation=" + continuation);
.getObject("continuationItemRenderer")
.getObject("continuationEndpoint")
.getObject("continuationCommand")
.getString("token");
return new Page(
"https://www.youtube.com/youtubei/v1/browse?key=" + getKey(),
continuation);
} else {
return null;
}
......@@ -311,6 +340,11 @@ public class YoutubePlaylistExtractor extends PlaylistExtractor {
return YoutubePlaylistExtractor.this.getUploaderUrl();
}
@Override
public boolean isUploaderVerified() {
return false;
}
@Nullable
@Override
public String getTextualUploadDate() {
......
package org.schabi.newpipe.extractor.services.youtube.extractors;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;
import com.grack.nanojson.*;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.Page;
......@@ -20,16 +15,15 @@ import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import java.io.IOException;
import java.util.*;
import javax.annotation.Nonnull;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getClientVersion;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/*
......@@ -171,7 +165,7 @@ public class YoutubeSearchExtractor extends SearchExtractor {
.object("user").end()
.end()
.value("continuation", page.getId())
.end().done().getBytes("UTF-8");
.end().done().getBytes(UTF_8);
// @formatter:on
final Map<String, List<String>> headers = new HashMap<>();
......
......@@ -4,7 +4,6 @@ import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
......@@ -29,17 +28,7 @@ import org.schabi.newpipe.extractor.localization.TimeAgoPatternsManager;
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.Frameset;
import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;
......@@ -47,26 +36,13 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.time.LocalDate;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import java.util.*;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/*
......@@ -102,17 +78,21 @@ public class YoutubeStreamExtractor extends StreamExtractor {
/*//////////////////////////////////////////////////////////////////////////*/
@Nullable private static String cachedDeobfuscationCode = null;
@Nullable private String playerJsUrl = null;
@Nullable
private static String cachedDeobfuscationCode = null;
@Nullable
private String playerJsUrl = null;
private JsonArray initialAjaxJson;
private JsonObject initialData;
@Nonnull private final Map<String, String> videoInfoPage = new HashMap<>();
@Nonnull
private final Map<String, String> videoInfoPage = new HashMap<>();
private JsonObject playerResponse;
private JsonObject videoPrimaryInfoRenderer;
private JsonObject videoSecondaryInfoRenderer;
private int ageLimit = -1;
@Nullable private List<SubtitlesStream> subtitles = null;
@Nullable
private List<SubtitlesStream> subtitles = null;
public YoutubeStreamExtractor(StreamingService service, LinkHandler linkHandler) {
super(service, linkHandler);
......@@ -432,6 +412,14 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return uploaderName;
}
@Override
public boolean isUploaderVerified() throws ParsingException {
final JsonArray badges = getVideoSecondaryInfoRenderer().getObject("owner")
.getObject("videoOwnerRenderer").getArray("badges");
return YoutubeParsingHelper.isVerified(badges);
}
@Nonnull
@Override
public String getUploaderAvatarUrl() throws ParsingException {
......@@ -522,7 +510,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
for (Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.AUDIO).entrySet()) {
ItagItem itag = entry.getValue();
AudioStream audioStream = new AudioStream(entry.getKey(), itag.getMediaFormat(), itag.avgBitrate);
AudioStream audioStream = new AudioStream(entry.getKey(), itag);
if (!Stream.containSimilarStream(audioStream, audioStreams)) {
audioStreams.add(audioStream);
}
......@@ -542,7 +530,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
for (Map.Entry<String, ItagItem> entry : getItags(FORMATS, ItagItem.ItagType.VIDEO).entrySet()) {
ItagItem itag = entry.getValue();
VideoStream videoStream = new VideoStream(entry.getKey(), itag.getMediaFormat(), itag.resolutionString);
VideoStream videoStream = new VideoStream(entry.getKey(), false, itag);
if (!Stream.containSimilarStream(videoStream, videoStreams)) {
videoStreams.add(videoStream);
}
......@@ -562,7 +550,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
for (Map.Entry<String, ItagItem> entry : getItags(ADAPTIVE_FORMATS, ItagItem.ItagType.VIDEO_ONLY).entrySet()) {
ItagItem itag = entry.getValue();
VideoStream videoStream = new VideoStream(entry.getKey(), itag.getMediaFormat(), itag.resolutionString, true);
VideoStream videoStream = new VideoStream(entry.getKey(), true, itag);
if (!Stream.containSimilarStream(videoStream, videoOnlyStreams)) {
videoOnlyStreams.add(videoStream);
}
......@@ -787,8 +775,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
}
private String getDeobfuscationFuncName(final String playerCode) throws DeobfuscateException {
Parser.RegexException exception = null;
for (final String regex : REGEXES) {
......@@ -971,6 +957,22 @@ public class YoutubeStreamExtractor extends StreamExtractor {
+ deobfuscateSignature(cipher.get("s"));
}
JsonObject initRange = formatData.getObject("initRange");
JsonObject indexRange = formatData.getObject("indexRange");
String mimeType = formatData.getString("mimeType", EMPTY_STRING);
String codec = mimeType.contains("codecs") ? mimeType.split("\"")[1] : EMPTY_STRING;
itagItem.setBitrate(formatData.getInt("bitrate"));
itagItem.setWidth(formatData.getInt("width"));
itagItem.setHeight(formatData.getInt("height"));
itagItem.setInitStart(Integer.parseInt(initRange.getString("start", "-1")));
itagItem.setInitEnd(Integer.parseInt(initRange.getString("end", "-1")));
itagItem.setIndexStart(Integer.parseInt(indexRange.getString("start", "-1")));
itagItem.setIndexEnd(Integer.parseInt(indexRange.getString("end", "-1")));
itagItem.fps = formatData.getInt("fps");
itagItem.setQuality(formatData.getString("quality"));
itagItem.setCodec(codec);
urlAndItags.put(streamUrl, itagItem);
}
} catch (UnsupportedEncodingException ignored) {
......@@ -1138,6 +1140,6 @@ public class YoutubeStreamExtractor extends StreamExtractor {
public List<MetaInfo> getMetaInfo() throws ParsingException {
return YoutubeParsingHelper.getMetaInfo(
initialData.getObject("contents").getObject("twoColumnWatchNextResults")
.getObject("results").getObject("results").getArray("contents"));
.getObject("results").getObject("results").getArray("contents"));
}
}
......@@ -17,10 +17,8 @@ import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
/*
......@@ -160,6 +158,11 @@ public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
return url;
}
@Override
public boolean isUploaderVerified() throws ParsingException {
return YoutubeParsingHelper.isVerified(videoInfo.getArray("ownerBadges"));
}
@Nullable
@Override
public String getTextualUploadDate() throws ParsingException {
......
......@@ -15,6 +15,8 @@ import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
/*
* Created by Christian Schabesberger on 28.09.16.
*
......@@ -37,8 +39,6 @@ import java.util.List;
public class YoutubeSuggestionExtractor extends SuggestionExtractor {
public static final String CHARSET_UTF_8 = "UTF-8";
public YoutubeSuggestionExtractor(StreamingService service) {
super(service);
}
......@@ -52,8 +52,8 @@ public class YoutubeSuggestionExtractor extends SuggestionExtractor {
+ "?client=" + "youtube" //"firefox" for JSON, 'toolbar' for xml
+ "&jsonp=" + "JP"
+ "&ds=" + "yt"
+ "&gl=" + URLEncoder.encode(getExtractorContentCountry().getCountryCode(), CHARSET_UTF_8)
+ "&q=" + URLEncoder.encode(query, CHARSET_UTF_8);
+ "&gl=" + URLEncoder.encode(getExtractorContentCountry().getCountryCode(), UTF_8)
+ "&q=" + URLEncoder.encode(query, UTF_8);
String response = dl.get(url, getExtractorLocalization()).responseBody();
// trim JSONP part "JP(...)"
......
......@@ -7,8 +7,9 @@ import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.List;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
public static final String CHARSET_UTF_8 = "UTF-8";
public static final String ALL = "all";
public static final String VIDEOS = "videos";
......@@ -37,21 +38,21 @@ public class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory
default:
break;
case VIDEOS:
return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAQ%253D%253D";
return SEARCH_URL + URLEncoder.encode(searchString, UTF_8) + "&sp=EgIQAQ%253D%253D";
case CHANNELS:
return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAg%253D%253D";
return SEARCH_URL + URLEncoder.encode(searchString, UTF_8) + "&sp=EgIQAg%253D%253D";
case PLAYLISTS:
return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8) + "&sp=EgIQAw%253D%253D";
return SEARCH_URL + URLEncoder.encode(searchString, UTF_8) + "&sp=EgIQAw%253D%253D";
case MUSIC_SONGS:
case MUSIC_VIDEOS:
case MUSIC_ALBUMS:
case MUSIC_PLAYLISTS:
case MUSIC_ARTISTS:
return MUSIC_SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8);
return MUSIC_SEARCH_URL + URLEncoder.encode(searchString, UTF_8);
}
}
return SEARCH_URL + URLEncoder.encode(searchString, CHARSET_UTF_8);
return SEARCH_URL + URLEncoder.encode(searchString, UTF_8);
} catch (UnsupportedEncodingException e) {
throw new ParsingException("Could not encode query", e);
}
......
......@@ -11,6 +11,8 @@ import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.Arrays;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
......@@ -38,6 +40,7 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
private static final Pattern YOUTUBE_VIDEO_ID_REGEX_PATTERN = Pattern.compile("^([a-zA-Z0-9_-]{11})");
private static final YoutubeStreamLinkHandlerFactory instance = new YoutubeStreamLinkHandlerFactory();
private static final List<String> SUBPATHS = Arrays.asList("embed/", "shorts/", "watch/", "v/", "w/");
private YoutubeStreamLinkHandlerFactory() {
}
......@@ -124,7 +127,7 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
switch (host.toUpperCase()) {
case "WWW.YOUTUBE-NOCOOKIE.COM": {
if (path.startsWith("embed/")) {
String id = path.split("/")[1];
String id = path.substring(6); // embed/
return assertIsId(id);
}
......@@ -150,11 +153,8 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
return assertIsId(viewQueryValue);
}
if (path.startsWith("embed/") || path.startsWith("shorts/")) {
String id = path.split("/")[1];
return assertIsId(id);
}
String maybeId = getIdFromSubpathsInPath(path);
if (maybeId != null) return maybeId;
String viewQueryValue = Utils.getQueryValue(url, "v");
return assertIsId(viewQueryValue);
......@@ -169,20 +169,7 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
return assertIsId(path);
}
case "HOOKTUBE.COM": {
if (path.startsWith("v/")) {
String id = path.substring("v/".length());
return assertIsId(id);
}
if (path.startsWith("watch/")) {
String id = path.substring("watch/".length());
return assertIsId(id);
}
// there is no break-statement here on purpose so the next code-block gets also run for hooktube
}
case "HOOKTUBE.COM":
case "INVIDIO.US":
case "DEV.INVIDIO.US":
case "WWW.INVIDIO.US":
......@@ -208,11 +195,8 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
return assertIsId(viewQueryValue);
}
}
if (path.startsWith("embed/")) {
String id = path.substring("embed/".length());
return assertIsId(id);
}
String maybeId = getIdFromSubpathsInPath(path);
if (maybeId != null) return maybeId;
String viewQueryValue = Utils.getQueryValue(url, "v");
if (viewQueryValue != null) {
......@@ -237,4 +221,14 @@ public class YoutubeStreamLinkHandlerFactory extends LinkHandlerFactory {
return false;
}
}
private String getIdFromSubpathsInPath(String path) throws ParsingException {
for (final String subpath : SUBPATHS) {
if (path.startsWith(subpath)) {
String id = path.substring(subpath.length());
return assertIsId(id);
}
}
return null;
}
}
......@@ -21,10 +21,21 @@ package org.schabi.newpipe.extractor.stream;
*/
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
public class AudioStream extends Stream {
public int average_bitrate = -1;
// Fields for Dash
private int itag;
private int bitrate;
private int initStart;
private int initEnd;
private int indexStart;
private int indexEnd;
private String quality;
private String codec;
/**
* Create a new audio stream
* @param url the url
......@@ -36,6 +47,23 @@ public class AudioStream extends Stream {
this.average_bitrate = averageBitrate;
}
/**
* Create a new audio stream
* @param url the url
* @param itag the ItagItem of the Stream
*/
public AudioStream(String url, ItagItem itag) {
this(url, itag.getMediaFormat(), itag.avgBitrate);
this.itag = itag.id;
this.quality = itag.getQuality();
this.bitrate = itag.getBitrate();
this.initStart = itag.getInitStart();
this.initEnd = itag.getInitEnd();
this.indexStart = itag.getIndexStart();
this.indexEnd = itag.getIndexEnd();
this.codec = itag.getCodec();
}
@Override
public boolean equalStats(Stream cmp) {
return super.equalStats(cmp) && cmp instanceof AudioStream &&
......@@ -49,4 +77,36 @@ public class AudioStream extends Stream {
public int getAverageBitrate() {
return average_bitrate;
}
public int getItag() {
return itag;
}
public int getBitrate() {
return bitrate;
}
public int getInitStart() {
return initStart;
}
public int getInitEnd() {
return initEnd;
}
public int getIndexStart() {
return indexStart;
}
public int getIndexEnd() {
return indexEnd;
}
public String getQuality() {
return quality;
}
public String getCodec() {
return codec;
}
}
package org.schabi.newpipe.extractor.stream;
import java.io.Serializable;
import java.util.Objects;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public class Description implements Serializable {
public static final int HTML = 1;
public static final int MARKDOWN = 2;
public static final int PLAIN_TEXT = 3;
public static final Description emptyDescription = new Description("", PLAIN_TEXT);
public static final Description emptyDescription = new Description(EMPTY_STRING, PLAIN_TEXT);
private String content;
private int type;
......@@ -28,4 +31,17 @@ public class Description implements Serializable {
public int getType() {
return type;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Description that = (Description) o;
return type == that.type && Objects.equals(content, that.content);
}
@Override
public int hashCode() {
return Objects.hash(content, type);
}
}
......@@ -31,14 +31,13 @@ import org.schabi.newpipe.extractor.linkhandler.LinkHandler;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.utils.Parser;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
/**
* Scrapes information from a video/audio streaming service (eg, YouTube).
*/
......@@ -171,6 +170,15 @@ public abstract class StreamExtractor extends Extractor {
@Nonnull
public abstract String getUploaderName() throws ParsingException;
/**
* Whether the uploader has been verified by the service's provider.
* If there is no verification implemented, return <code>false</code>.
*
* @return whether the uploader has been verified by the service's provider
* @throws ParsingException
*/
public abstract boolean isUploaderVerified() throws ParsingException;
/**
* The url to the image file/profile picture/avatar of the creator/uploader of the stream.
* If the url is not available you can return an empty String.
......@@ -479,8 +487,8 @@ public abstract class StreamExtractor extends Extractor {
public abstract String getSupportInfo() throws ParsingException;
/**
* The list of stream segments by timestamps for the stream.
* If the segment list is not available you can simply return an empty list.
* The list of stream segments by timestamps for the stream.
* If the segment list is not available you can simply return an empty list.
*
* @return The list of segments of the stream or an empty list.
* @throws ParsingException
......@@ -495,7 +503,8 @@ public abstract class StreamExtractor extends Extractor {
* or further information on the topic (e.g. hints that the video might contain conspiracy theories
* or contains information about a current health situation like the Covid-19 pandemic).
* </p>
* The meta information often contains links to external sources like Wikipedia or the WHO.
* The meta information often contains links to external sources like Wikipedia or the WHO.
*
* @return The meta info of the stream or an empty List if not provided.
* @throws ParsingException
*/
......
......@@ -33,11 +33,13 @@ public class StreamInfoItem extends InfoItem {
private String uploaderName;
private String textualUploadDate;
@Nullable private DateWrapper uploadDate;
@Nullable
private DateWrapper uploadDate;
private long viewCount = -1;
private long duration = -1;
private String uploaderUrl = null;
private boolean uploaderVerified = false;
public StreamInfoItem(int serviceId, String url, String name, StreamType streamType) {
super(InfoType.STREAM, serviceId, url, name);
......@@ -98,6 +100,14 @@ public class StreamInfoItem extends InfoItem {
this.uploadDate = uploadDate;
}
public boolean isUploaderVerified() {
return uploaderVerified;
}
public void setUploaderVerified(boolean uploaderVerified) {
this.uploaderVerified = uploaderVerified;
}
@Override
public String toString() {
return "StreamInfoItem{" +
......@@ -112,6 +122,7 @@ public class StreamInfoItem extends InfoItem {
", url='" + getUrl() + '\'' +
", name='" + getName() + '\'' +
", thumbnailUrl='" + getThumbnailUrl() + '\'' +
", uploaderVerified='" + isUploaderVerified() + '\'' +
'}';
}
}
\ No newline at end of file
......@@ -71,6 +71,15 @@ public interface StreamInfoItemExtractor extends InfoItemExtractor {
String getUploaderUrl() throws ParsingException;
/**
* Whether the uploader has been verified by the service's provider.
* If there is no verification implemented, return <code>false</code>.
*
* @return whether the uploader has been verified by the service's provider
* @throws ParsingException
*/
boolean isUploaderVerified() throws ParsingException;
/**
* The original textual date provided by the service. Should be used as a fallback if
* {@link #getUploadDate()} isn't provided by the service, or it fails for some reason.
......
......@@ -90,6 +90,12 @@ public class StreamInfoItemsCollector extends InfoItemsCollector<StreamInfoItem,
} catch (Exception e) {
addError(e);
}
try {
resultItem.setUploaderVerified(extractor.isUploaderVerified());
} catch (Exception e) {
addError(e);
}
return resultItem;
}
......
......@@ -21,20 +21,46 @@ package org.schabi.newpipe.extractor.stream;
*/
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
public class VideoStream extends Stream {
public final String resolution;
public final boolean isVideoOnly;
// Fields for Dash
private int itag;
private int bitrate;
private int initStart;
private int initEnd;
private int indexStart;
private int indexEnd;
private int width;
private int height;
private int fps;
private String quality;
private String codec;
public VideoStream(String url, MediaFormat format, String resolution) {
this(url, format, resolution, false);
}
public VideoStream(String url, MediaFormat format, String resolution, boolean isVideoOnly) {
super(url, format);
this.resolution = resolution;
this.isVideoOnly = isVideoOnly;
this(url, null, format, resolution, isVideoOnly);
}
public VideoStream(String url, boolean isVideoOnly, ItagItem itag) {
this(url, itag.getMediaFormat(), itag.resolutionString, isVideoOnly);
this.itag = itag.id;
this.bitrate = itag.getBitrate();
this.initStart = itag.getInitStart();
this.initEnd = itag.getInitEnd();
this.indexStart = itag.getIndexStart();
this.indexEnd = itag.getIndexEnd();
this.codec = itag.getCodec();
this.height = itag.getHeight();
this.width = itag.getWidth();
this.quality = itag.getQuality();
this.fps = itag.fps;
}
public VideoStream(String url, String torrentUrl, MediaFormat format, String resolution) {
......@@ -73,4 +99,48 @@ public class VideoStream extends Stream {
public boolean isVideoOnly() {
return isVideoOnly;
}
public int getItag() {
return itag;
}
public int getBitrate() {
return bitrate;
}
public int getInitStart() {
return initStart;
}
public int getInitEnd() {
return initEnd;
}
public int getIndexStart() {
return indexStart;
}
public int getIndexEnd() {
return indexEnd;
}
public int getWidth() {
return width;
}
public int getHeight() {
return height;
}
public int getFps() {
return fps;
}
public String getQuality() {
return quality;
}
public String getCodec() {
return codec;
}
}
......@@ -2,18 +2,21 @@ package org.schabi.newpipe.extractor.utils;
import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
public class JsonUtils {
public static final JsonObject EMPTY_OBJECT = new JsonObject();
public static final JsonArray EMPTY_ARRAY = new JsonArray();
public static final String EMPTY_STRING = "";
private JsonUtils() {
}
......@@ -100,4 +103,19 @@ public class JsonUtils {
return result;
}
public static JsonArray toJsonArray(final String responseBody) throws ParsingException {
try {
return JsonParser.array().from(responseBody);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse JSON", e);
}
}
public static JsonObject toJsonObject(final String responseBody) throws ParsingException {
try {
return JsonParser.object().from(responseBody);
} catch (JsonParserException e) {
throw new ParsingException("Could not parse JSON", e);
}
}
}
......@@ -14,6 +14,8 @@ import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
/*
* Created by Christian Schabesberger on 02.02.16.
*
......@@ -87,7 +89,7 @@ public class Parser {
for (String arg : input.split("&")) {
String[] splitArg = arg.split("=");
if (splitArg.length > 1) {
map.put(splitArg[0], URLDecoder.decode(splitArg[1], "UTF-8"));
map.put(splitArg[0], URLDecoder.decode(splitArg[1], UTF_8));
} else {
map.put(splitArg[0], "");
}
......
......@@ -6,16 +6,14 @@ import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.*;
public class Utils {
public static final String HTTP = "http://";
public static final String HTTPS = "https://";
public static final String UTF_8 = "UTF-8";
public static final String EMPTY_STRING = "";
private Utils() {
//no instance
......@@ -117,7 +115,7 @@ public class Utils {
String query;
try {
query = URLDecoder.decode(params[0], "UTF-8");
query = URLDecoder.decode(params[0], UTF_8);
} catch (UnsupportedEncodingException e) {
System.err.println("Cannot decode string with UTF-8. using the string without decoding");
e.printStackTrace();
......@@ -126,7 +124,7 @@ public class Utils {
if (query.equals(parameterName)) {
try {
return URLDecoder.decode(params[1], "UTF-8");
return URLDecoder.decode(params[1], UTF_8);
} catch (UnsupportedEncodingException e) {
System.err.println("Cannot decode string with UTF-8. using the string without decoding");
e.printStackTrace();
......@@ -200,6 +198,7 @@ public class Utils {
/**
* If the provided url is a Google search redirect, then the actual url is extracted from the
* {@code url=} query value and returned, otherwise the original url is returned.
*
* @param url the url which can possibly be a Google search redirect
* @return an url with no Google search redirects
*/
......@@ -208,7 +207,7 @@ public class Utils {
try {
final URL decoded = Utils.stringToURL(url);
if (decoded.getHost().contains("google") && decoded.getPath().equals("/url")) {
return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url), "UTF-8");
return URLDecoder.decode(Parser.matchGroup1("&url=([^&]+)(?:&|$)", url), UTF_8);
}
} catch (final Exception ignored) {
}
......@@ -231,12 +230,12 @@ public class Utils {
return map == null || map.isEmpty();
}
public static boolean isWhitespace(final int c){
public static boolean isWhitespace(final int c) {
return c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\r';
}
public static boolean isBlank(final String string) {
if (string == null || string.isEmpty()) {
if (isNullOrEmpty(string)) {
return true;
}
......@@ -263,10 +262,10 @@ public class Utils {
}
public static String join(final String delimiter, final String mapJoin,
final Map<? extends CharSequence, ? extends CharSequence> elements) {
final Map<? extends CharSequence, ? extends CharSequence> elements) {
final List<String> list = new LinkedList<>();
for (final Map.Entry<? extends CharSequence, ? extends CharSequence> entry : elements
.entrySet()) {
.entrySet()) {
list.add(entry.getKey() + mapJoin + entry.getValue());
}
return join(delimiter, list);
......
......@@ -27,15 +27,17 @@ class MockDownloader extends Downloader {
public MockDownloader(@Nonnull String path) throws IOException {
this.path = path;
this.mocks = new HashMap<>();
File folder = new File(path);
for (File file : folder.listFiles()) {
if (file.getName().startsWith(RecordingDownloader.FILE_NAME_PREFIX)) {
final FileReader reader = new FileReader(file);
final TestRequestResponse response = new GsonBuilder()
.create()
.fromJson(reader, TestRequestResponse.class);
reader.close();
mocks.put(response.getRequest(), response.getResponse());
final File[] files = new File(path).listFiles();
if (files != null) {
for (File file : files) {
if (file.getName().startsWith(RecordingDownloader.FILE_NAME_PREFIX)) {
final FileReader reader = new FileReader(file);
final TestRequestResponse response = new GsonBuilder()
.create()
.fromJson(reader, TestRequestResponse.class);
reader.close();
mocks.put(response.getRequest(), response.getResponse());
}
}
}
}
......
......@@ -27,6 +27,10 @@ import javax.annotation.Nonnull;
* The files <b>must</b> be created on the local dev environment
* and recreated when the requests made by a test class change.
* </p>
* <p>
* Run the test class as a whole and not each test separately.
* Make sure the requests made by a class are unique.
* </p>
*/
class RecordingDownloader extends Downloader {
......
......@@ -7,4 +7,5 @@ public interface BaseChannelExtractorTest extends BaseListExtractorTest {
void testBannerUrl() throws Exception;
void testFeedUrl() throws Exception;
void testSubscriberCount() throws Exception;
void testVerified() throws Exception;
}
......@@ -7,4 +7,5 @@ public interface BasePlaylistExtractorTest extends BaseListExtractorTest {
void testUploaderName() throws Exception;
void testUploaderAvatarUrl() throws Exception;
void testStreamCount() throws Exception;
void testUploaderVerified() throws Exception;
}
......@@ -45,6 +45,7 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<St
public abstract StreamType expectedStreamType();
public abstract String expectedUploaderName();
public abstract String expectedUploaderUrl();
public boolean expectedUploaderVerified() { return false; }
public String expectedSubChannelName() { return ""; } // default: there is no subchannel
public String expectedSubChannelUrl() { return ""; } // default: there is no subchannel
public abstract List<String> expectedDescriptionContains(); // e.g. for full links
......@@ -99,6 +100,11 @@ public abstract class DefaultStreamExtractorTest extends DefaultExtractorTest<St
assertIsSecureUrl(extractor().getUploaderAvatarUrl());
}
@Test
public void testUploaderVerified() throws Exception {
assertEquals(expectedUploaderVerified(), extractor().isUploaderVerified());
}
@Test
@Override
public void testSubChannelName() throws Exception {
......
......@@ -104,6 +104,11 @@ public class PeertubeAccountExtractorTest {
public void testSubscriberCount() throws ParsingException {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 500);
}
@Override
public void testVerified() throws Exception {
assertFalse(extractor.isVerified());
}
}
public static class FreeSoftwareFoundation implements BaseChannelExtractorTest {
......@@ -200,5 +205,10 @@ public class PeertubeAccountExtractorTest {
public void testSubscriberCount() throws ParsingException {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 100);
}
@Override
public void testVerified() throws Exception {
assertFalse(extractor.isVerified());
}
}
}
package org.schabi.newpipe.extractor.services.peertube;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
......@@ -119,6 +118,11 @@ public class PeertubeChannelExtractorTest {
public void testSubscriberCount() throws ParsingException {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 230);
}
@Override
public void testVerified() throws Exception {
assertFalse(extractor.isVerified());
}
}
public static class ChatSceptique implements BaseChannelExtractorTest {
......@@ -231,5 +235,10 @@ public class PeertubeChannelExtractorTest {
public void testSubscriberCount() throws ParsingException {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 700);
}
@Override
public void testVerified() throws Exception {
assertFalse(extractor.isVerified());
}
}
}
......@@ -101,6 +101,11 @@ public class SoundcloudChannelExtractorTest {
public void testSubscriberCount() {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 1e6);
}
@Override
public void testVerified() throws Exception {
assertTrue(extractor.isVerified());
}
}
public static class DubMatix implements BaseChannelExtractorTest {
......@@ -195,5 +200,10 @@ public class SoundcloudChannelExtractorTest {
public void testSubscriberCount() {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 2e6);
}
@Override
public void testVerified() throws Exception {
assertTrue(extractor.isVerified());
}
}
}
......@@ -111,6 +111,11 @@ public class SoundcloudPlaylistExtractorTest {
public void testStreamCount() {
assertTrue("Stream count does not fit: " + extractor.getStreamCount(), extractor.getStreamCount() >= 10);
}
@Override
public void testUploaderVerified() throws Exception {
assertTrue(extractor.isUploaderVerified());
}
}
public static class RandomHouseMusic implements BasePlaylistExtractorTest {
......@@ -203,6 +208,11 @@ public class SoundcloudPlaylistExtractorTest {
public void testStreamCount() {
assertTrue("Stream count does not fit: " + extractor.getStreamCount(), extractor.getStreamCount() >= 10);
}
@Override
public void testUploaderVerified() throws Exception {
assertFalse(extractor.isUploaderVerified());
}
}
public static class EDMxxx implements BasePlaylistExtractorTest {
......@@ -310,6 +320,11 @@ public class SoundcloudPlaylistExtractorTest {
public void testStreamCount() {
assertTrue("Stream count does not fit: " + extractor.getStreamCount(), extractor.getStreamCount() >= 370);
}
@Override
public void testUploaderVerified() throws Exception {
assertFalse(extractor.isUploaderVerified());
}
}
public static class SmallPlaylist implements BasePlaylistExtractorTest {
......@@ -409,5 +424,10 @@ public class SoundcloudPlaylistExtractorTest {
public void testStreamCount() {
assertEquals(2, extractor.getStreamCount());
}
@Override
public void testUploaderVerified() throws Exception {
assertFalse(extractor.isUploaderVerified());
}
}
}
......@@ -8,17 +8,23 @@ import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.List;
import static java.util.Collections.singletonList;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems;
import static org.schabi.newpipe.extractor.services.soundcloud.linkHandler.SoundcloudSearchQueryHandlerFactory.*;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
public class SoundcloudSearchExtractorTest {
......@@ -33,6 +39,7 @@ public class SoundcloudSearchExtractorTest {
extractor.fetchPage();
}
// @formatter:off
@Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return SoundCloud; }
@Override public String expectedName() { return QUERY; }
......@@ -41,6 +48,7 @@ public class SoundcloudSearchExtractorTest {
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
// @formatter:on
}
public static class Tracks extends DefaultSearchExtractorTest {
......@@ -54,6 +62,7 @@ public class SoundcloudSearchExtractorTest {
extractor.fetchPage();
}
// @formatter:off
@Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return SoundCloud; }
@Override public String expectedName() { return QUERY; }
......@@ -62,8 +71,8 @@ public class SoundcloudSearchExtractorTest {
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/tracks?q=" + urlEncode(QUERY); }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
// @formatter:on
}
public static class Users extends DefaultSearchExtractorTest {
......@@ -77,6 +86,7 @@ public class SoundcloudSearchExtractorTest {
extractor.fetchPage();
}
// @formatter:off
@Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return SoundCloud; }
@Override public String expectedName() { return QUERY; }
......@@ -85,8 +95,8 @@ public class SoundcloudSearchExtractorTest {
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/users?q=" + urlEncode(QUERY); }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.CHANNEL; }
// @formatter:on
}
public static class Playlists extends DefaultSearchExtractorTest {
......@@ -100,6 +110,7 @@ public class SoundcloudSearchExtractorTest {
extractor.fetchPage();
}
// @formatter:off
@Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return SoundCloud; }
@Override public String expectedName() { return QUERY; }
......@@ -108,8 +119,8 @@ public class SoundcloudSearchExtractorTest {
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/playlists?q=" + urlEncode(QUERY); }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.PLAYLIST; }
// @formatter:on
}
public static class PagingTest {
......@@ -128,9 +139,45 @@ public class SoundcloudSearchExtractorTest {
private static String urlEncode(String value) {
try {
return URLEncoder.encode(value, CHARSET_UTF_8);
return URLEncoder.encode(value, UTF_8);
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
}
public static class UserVerified extends DefaultSearchExtractorTest {
private static SearchExtractor extractor;
private static final String QUERY = "David Guetta";
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = SoundCloud.getSearchExtractor(QUERY, singletonList(USERS), "");
extractor.fetchPage();
}
@Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return SoundCloud; }
@Override public String expectedName() { return QUERY; }
@Override public String expectedId() { return QUERY; }
@Override public String expectedUrlContains() { return "soundcloud.com/search/users?q=" + urlEncode(QUERY); }
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/users?q=" + urlEncode(QUERY); }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.CHANNEL; }
@Test
public void testIsVerified() throws IOException, ExtractionException {
final List<InfoItem> items = extractor.getInitialPage().getItems();
boolean verified = false;
for (InfoItem item : items) {
if (item.getUrl().equals("https://soundcloud.com/davidguetta")) {
verified = ((ChannelInfoItem) item).isVerified();
break;
}
}
assertTrue(verified);
}
}
}
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
......@@ -12,8 +12,11 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
import java.io.IOException;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.*;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertEmpty;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.*;
......@@ -23,10 +26,13 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.*;
*/
public class YoutubeChannelExtractorTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/channel/";
public static class NotAvailable {
@BeforeClass
public static void setUp() {
NewPipe.init(DownloaderTestImpl.getInstance());
public static void setUp() throws IOException {
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "notAvailable"));
}
@Test(expected = ContentNotAvailableException.class)
......@@ -46,8 +52,9 @@ public class YoutubeChannelExtractorTest {
public static class NotSupported {
@BeforeClass
public static void setUp() {
NewPipe.init(DownloaderTestImpl.getInstance());
public static void setUp() throws IOException {
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "notSupported"));
}
@Test(expected = ContentNotSupportedException.class)
......@@ -63,7 +70,8 @@ public class YoutubeChannelExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "gronkh"));
extractor = (YoutubeChannelExtractor) YouTube
.getChannelExtractor("http://www.youtube.com/user/Gronkh");
extractor.fetchPage();
......@@ -118,7 +126,7 @@ public class YoutubeChannelExtractorTest {
@Test
public void testDescription() throws Exception {
assertTrue(extractor.getDescription().contains("Zart im Schmelz und süffig im Abgang. Ungebremster Spieltrieb"));
assertThat(extractor.getDescription(), containsString("Ungebremster Spieltrieb seit 1896."));
}
@Test
......@@ -145,6 +153,12 @@ public class YoutubeChannelExtractorTest {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 0);
assertTrue("Subscriber count too small", extractor.getSubscriberCount() >= 4e6);
}
@Override
public void testVerified() throws Exception {
assertTrue(extractor.isVerified());
}
}
// Youtube RED/Premium ad blocking test
......@@ -153,7 +167,8 @@ public class YoutubeChannelExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "VSauce"));
extractor = (YoutubeChannelExtractor) YouTube
.getChannelExtractor("https://www.youtube.com/user/Vsauce");
extractor.fetchPage();
......@@ -237,6 +252,11 @@ public class YoutubeChannelExtractorTest {
assertTrue("Subscriber count too small", extractor.getSubscriberCount() >= 10e6);
}
@Override
public void testVerified() throws Exception {
assertTrue(extractor.isVerified());
}
}
public static class Kurzgesagt implements BaseChannelExtractorTest {
......@@ -244,22 +264,13 @@ public class YoutubeChannelExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "kurzgesagt"));
extractor = (YoutubeChannelExtractor) YouTube
.getChannelExtractor("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q");
extractor.fetchPage();
}
/*//////////////////////////////////////////////////////////////////////////
// Additional Testing
//////////////////////////////////////////////////////////////////////////*/
@Test
public void testGetPageInNewExtractor() throws Exception {
final ChannelExtractor newExtractor = YouTube.getChannelExtractor(extractor.getUrl());
defaultTestGetPageInNewExtractor(extractor, newExtractor);
}
/*//////////////////////////////////////////////////////////////////////////
// Extractor
//////////////////////////////////////////////////////////////////////////*/
......@@ -339,14 +350,40 @@ public class YoutubeChannelExtractorTest {
public void testSubscriberCount() throws Exception {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 5e6);
}
@Override
public void testVerified() throws Exception {
assertTrue(extractor.isVerified());
}
}
public static class CaptainDisillusion implements BaseChannelExtractorTest {
public static class KurzgesagtAdditional {
private static YoutubeChannelExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
// Test is not deterministic, mocks can't be used
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (YoutubeChannelExtractor) YouTube
.getChannelExtractor("https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q");
extractor.fetchPage();
}
@Test
public void testGetPageInNewExtractor() throws Exception {
final ChannelExtractor newExtractor = YouTube.getChannelExtractor(extractor.getUrl());
defaultTestGetPageInNewExtractor(extractor, newExtractor);
}
}
public static class CaptainDisillusion implements BaseChannelExtractorTest {
private static YoutubeChannelExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "captainDisillusion"));
extractor = (YoutubeChannelExtractor) YouTube
.getChannelExtractor("https://www.youtube.com/user/CaptainDisillusion/videos");
extractor.fetchPage();
......@@ -428,6 +465,11 @@ public class YoutubeChannelExtractorTest {
public void testSubscriberCount() throws Exception {
assertTrue("Wrong subscriber count", extractor.getSubscriberCount() >= 5e5);
}
@Override
public void testVerified() throws Exception {
assertTrue(extractor.isVerified());
}
}
public static class RandomChannel implements BaseChannelExtractorTest {
......@@ -435,7 +477,8 @@ public class YoutubeChannelExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "random"));
extractor = (YoutubeChannelExtractor) YouTube
.getChannelExtractor("https://www.youtube.com/channel/UCUaQMQS9lY5lit3vurpXQ6w");
extractor.fetchPage();
......@@ -524,6 +567,11 @@ public class YoutubeChannelExtractorTest {
long subscribers = extractor.getSubscriberCount();
assertTrue("Wrong subscriber count: " + subscribers, subscribers >= 50);
}
@Override
public void testVerified() throws Exception {
assertFalse(extractor.isVerified());
}
}
}
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
......@@ -23,14 +22,15 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRela
/**
* A class that tests multiple channels and ranges of "time ago".
*/
@Ignore("Should be ran manually from time to time, as it's too time consuming.")
public class YoutubeChannelLocalizationTest {
private static final boolean DEBUG = true;
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/channel/";
private static final boolean DEBUG = false;
private final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm");
@Test
public void testAllSupportedLocalizations() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "localization"));
testLocalizationsFor("https://www.youtube.com/user/NBCNews");
testLocalizationsFor("https://www.youtube.com/channel/UCcmpeVbSSQlZRvHfdC-CRwg/videos");
......
package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.Page;
......@@ -17,10 +16,15 @@ import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.util.List;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
public class YoutubeCommentsExtractorTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/comments/";
/**
* Test a "normal" YouTube
*/
......@@ -31,7 +35,8 @@ public class YoutubeCommentsExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "thomas"));
extractor = (YoutubeCommentsExtractor) YouTube
.getCommentsExtractor(url);
extractor.fetchPage();
......@@ -118,14 +123,14 @@ public class YoutubeCommentsExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "empty"));
extractor = (YoutubeCommentsExtractor) YouTube
.getCommentsExtractor(url);
extractor.fetchPage();
}
@Test
@Ignore("TODO fix")
public void testGetCommentsAllData() throws IOException, ExtractionException {
final InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
......@@ -157,7 +162,8 @@ public class YoutubeCommentsExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "hearted"));
extractor = (YoutubeCommentsExtractor) YouTube
.getCommentsExtractor(url);
extractor.fetchPage();
......@@ -198,7 +204,8 @@ public class YoutubeCommentsExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "pinned"));
extractor = (YoutubeCommentsExtractor) YouTube
.getCommentsExtractor(url);
extractor.fetchPage();
......
......@@ -2,24 +2,29 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreItems;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
public class YoutubeFeedExtractorTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/feed/";
public static class Kurzgesagt implements BaseListExtractorTest {
private static YoutubeFeedExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH));
extractor = (YoutubeFeedExtractor) YouTube
.getFeedExtractor("https://www.youtube.com/user/Kurzgesagt");
extractor.fetchPage();
......
......@@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.BaseListExtractorTest;
......@@ -14,12 +14,16 @@ import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreIte
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
public class YoutubeKioskExtractorTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/kiosk/";
public static class Trending implements BaseListExtractorTest {
private static YoutubeTrendingExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "trending"));
extractor = (YoutubeTrendingExtractor) YouTube.getKioskList().getDefaultKioskExtractor();
extractor.fetchPage();
}
......
package org.schabi.newpipe.extractor.services.youtube;
import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.hamcrest.MatcherAssert;
import org.junit.BeforeClass;
import org.junit.Test;
......@@ -26,6 +20,12 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubeMixPlaylistExtractor
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import java.io.IOException;
import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.startsWith;
import static org.hamcrest.MatcherAssert.assertThat;
......@@ -43,7 +43,7 @@ public class YoutubeMixPlaylistExtractorTest {
private static final String VIDEO_ID = "_AzeUSL9lZc";
private static final String VIDEO_TITLE =
"Most Beautiful And Emotional Piano: Anime Music Shigatsu wa Kimi no Uso OST IMO";
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/mix/";
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/mix/";
private static final Map<String, String> dummyCookie
= Collections.singletonMap(YoutubeMixPlaylistExtractor.COOKIE_NAME, "whatever");
......@@ -134,9 +134,9 @@ public class YoutubeMixPlaylistExtractorTest {
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "mixWithIndex"));
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD"
+ VIDEO_ID + INDEX);
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD"
+ VIDEO_ID + INDEX);
extractor.fetchPage();
}
......@@ -165,8 +165,8 @@ public class YoutubeMixPlaylistExtractorTest {
@Test
public void getPage() throws Exception {
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(
new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD"
+ VIDEO_ID + INDEX + PBJ, dummyCookie));
new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_NUMBER_13 + "&list=RD"
+ VIDEO_ID + INDEX + PBJ, dummyCookie));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
......@@ -204,9 +204,9 @@ public class YoutubeMixPlaylistExtractorTest {
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "myMix"));
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RDMM"
+ VIDEO_ID);
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + VIDEO_ID + "&list=RDMM"
+ VIDEO_ID);
extractor.fetchPage();
}
......@@ -238,11 +238,12 @@ public class YoutubeMixPlaylistExtractorTest {
@Test
public void getPage() throws Exception {
final InfoItemsPage<StreamInfoItem> streams =
extractor.getPage(new Page("https://www.youtube.com/watch?v=" + VIDEO_ID
+ "&list=RDMM" + VIDEO_ID + PBJ, dummyCookie));
extractor.getPage(new Page("https://www.youtube.com/watch?v=" + VIDEO_ID
+ "&list=RDMM" + VIDEO_ID + PBJ, dummyCookie));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
@Test
public void getContinuations() throws Exception {
InfoItemsPage<StreamInfoItem> streams = extractor.getInitialPage();
......@@ -290,8 +291,8 @@ public class YoutubeMixPlaylistExtractorTest {
@Test(expected = ExtractionException.class)
public void invalidVideoId() throws Exception {
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + "abcde" + "&list=RD" + "abcde");
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + "abcde" + "&list=RD" + "abcde");
extractor.fetchPage();
extractor.getName();
}
......@@ -309,9 +310,9 @@ public class YoutubeMixPlaylistExtractorTest {
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "channelMix"));
extractor = (YoutubeMixPlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL
+ "&list=RDCM" + CHANNEL_ID);
.getPlaylistExtractor(
"https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL
+ "&list=RDCM" + CHANNEL_ID);
extractor.fetchPage();
}
......@@ -339,8 +340,8 @@ public class YoutubeMixPlaylistExtractorTest {
@Test
public void getPage() throws Exception {
final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(
new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL
+ "&list=RDCM" + CHANNEL_ID + PBJ, dummyCookie));
new Page("https://www.youtube.com/watch?v=" + VIDEO_ID_OF_CHANNEL
+ "&list=RDCM" + CHANNEL_ID + PBJ, dummyCookie));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
}
......
......@@ -2,7 +2,7 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
......@@ -13,9 +13,13 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class YoutubeParsingHelperTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/";
@BeforeClass
public static void setUp() {
NewPipe.init(DownloaderTestImpl.getInstance());
public static void setUp() throws IOException {
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "youtubeParsingHelper"));
}
@Test
......
......@@ -6,10 +6,9 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
import org.junit.runners.Suite.SuiteClasses;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistExtractor;
......@@ -22,26 +21,34 @@ import org.schabi.newpipe.extractor.services.youtube.YoutubePlaylistExtractorTes
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import java.io.IOException;
import static junit.framework.TestCase.assertFalse;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.schabi.newpipe.extractor.ExtractorAsserts.assertIsSecureUrl;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.DefaultTests.*;
import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoMoreItems;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestGetPageInNewExtractor;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestListOfItems;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestMoreItems;
import static org.schabi.newpipe.extractor.services.DefaultTests.defaultTestRelatedItems;
/**
* Test for {@link YoutubePlaylistExtractor}
*/
@RunWith(Suite.class)
@SuiteClasses({NotAvailable.class, TimelessPopHits.class, HugePlaylist.class,
LearningPlaylist.class, ContinuationsTests.class})
LearningPlaylist.class, ContinuationsTests.class})
public class YoutubePlaylistExtractorTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/playlist/";
public static class NotAvailable {
@BeforeClass
public static void setUp() {
NewPipe.init(DownloaderTestImpl.getInstance());
public static void setUp() throws IOException {
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "notAvailable"));
}
@Test(expected = ContentNotAvailableException.class)
......@@ -65,7 +72,8 @@ public class YoutubePlaylistExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "TimelessPopHits"));
extractor = (YoutubePlaylistExtractor) YouTube
.getPlaylistExtractor("http://www.youtube.com/watch?v=lp-EO5I60KA&list=PLMC9KNkIncKtPzgY-5rmhvj7fax8fdxoj");
extractor.fetchPage();
......@@ -155,6 +163,11 @@ public class YoutubePlaylistExtractorTest {
public void testStreamCount() throws Exception {
assertTrue("Error in the streams count", extractor.getStreamCount() > 100);
}
@Override
public void testUploaderVerified() throws Exception {
assertFalse(extractor.isUploaderVerified());
}
}
public static class HugePlaylist implements BasePlaylistExtractorTest {
......@@ -162,7 +175,8 @@ public class YoutubePlaylistExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "huge"));
extractor = (YoutubePlaylistExtractor) YouTube
.getPlaylistExtractor("https://www.youtube.com/watch?v=8SbUC-UaAxE&list=PLWwAypAcFRgKAIIFqBr9oy-ZYZnixa_Fj");
extractor.fetchPage();
......@@ -267,6 +281,11 @@ public class YoutubePlaylistExtractorTest {
public void testStreamCount() throws Exception {
assertTrue("Error in the streams count", extractor.getStreamCount() > 100);
}
@Override
public void testUploaderVerified() throws Exception {
assertTrue(extractor.isUploaderVerified());
}
}
public static class LearningPlaylist implements BasePlaylistExtractorTest {
......@@ -274,7 +293,8 @@ public class YoutubePlaylistExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "learning"));
extractor = (YoutubePlaylistExtractor) YouTube
.getPlaylistExtractor("https://www.youtube.com/playlist?list=PL8dPuuaLjXtOAKed_MxxWBNaPno5h3Zs8");
extractor.fetchPage();
......@@ -365,20 +385,26 @@ public class YoutubePlaylistExtractorTest {
public void testStreamCount() throws Exception {
assertTrue("Error in the streams count", extractor.getStreamCount() > 40);
}
@Override
public void testUploaderVerified() throws Exception {
assertTrue(extractor.isUploaderVerified());
}
}
public static class ContinuationsTests {
@BeforeClass
public static void setUp() {
NewPipe.init(DownloaderTestImpl.getInstance());
public static void setUp() throws IOException {
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "continuations"));
}
@Test
public void testNoContinuations() throws Exception {
final YoutubePlaylistExtractor extractor = (YoutubePlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/playlist?list=PLXJg25X-OulsVsnvZ7RVtSDW-id9_RzAO");
.getPlaylistExtractor(
"https://www.youtube.com/playlist?list=PLXJg25X-OulsVsnvZ7RVtSDW-id9_RzAO");
extractor.fetchPage();
assertNoMoreItems(extractor);
......@@ -387,12 +413,12 @@ public class YoutubePlaylistExtractorTest {
@Test
public void testOnlySingleContinuation() throws Exception {
final YoutubePlaylistExtractor extractor = (YoutubePlaylistExtractor) YouTube
.getPlaylistExtractor(
"https://www.youtube.com/playlist?list=PLjgwFL8urN2DFRuRkFTkmtHjyoNWHHdZX");
.getPlaylistExtractor(
"https://www.youtube.com/playlist?list=PLoumn5BIsUDeGF1vy5Nylf_RJKn5aL_nr");
extractor.fetchPage();
final ListExtractor.InfoItemsPage<StreamInfoItem> page = defaultTestMoreItems(
extractor);
extractor);
assertFalse("More items available when it shouldn't", page.hasNextPage());
}
}
......
......@@ -92,6 +92,9 @@ public class YoutubeStreamLinkHandlerFactoryTest {
assertEquals("-cdveCh1kQk", linkHandler.fromUrl("HTTPS://youtu.be/-cdveCh1kQk)").getId());
assertEquals("IOS2fqxwYbA", linkHandler.fromUrl("https://www.youtube.com/shorts/IOS2fqxwYbAhi").getId());
assertEquals("IOS2fqxwYbA", linkHandler.fromUrl("http://www.youtube.com/shorts/IOS2fqxwYbA").getId());
assertEquals("IOS2fqxwYbA", linkHandler.fromUrl("http://www.youtube.com/v/IOS2fqxwYbA").getId());
assertEquals("IOS2fqxwYbA", linkHandler.fromUrl("http://www.youtube.com/w/IOS2fqxwYbA").getId());
assertEquals("IOS2fqxwYbA", linkHandler.fromUrl("http://www.youtube.com/watch/IOS2fqxwYbA").getId());
}
@Test
......@@ -113,6 +116,9 @@ public class YoutubeStreamLinkHandlerFactoryTest {
assertTrue(linkHandler.acceptUrl("vnd.youtube.launch:jZViOEv90dI"));
assertTrue(linkHandler.acceptUrl("https://music.youtube.com/watch?v=O0EDx9WAelc"));
assertTrue(linkHandler.acceptUrl("https://www.youtube.com/shorts/IOS2fqxwYbA"));
assertTrue(linkHandler.acceptUrl("https://www.youtube.com/v/IOS2fqxwYbA"));
assertTrue(linkHandler.acceptUrl("https://www.youtube.com/w/IOS2fqxwYbA"));
assertTrue(linkHandler.acceptUrl("https://www.youtube.com/watch/IOS2fqxwYbA"));
}
@Test
......@@ -134,26 +140,33 @@ public class YoutubeStreamLinkHandlerFactoryTest {
assertEquals("ocH3oSnZG3c", linkHandler.fromUrl("https://hooktube.com/watch?v=ocH3oSnZG3c&list=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2").getId());
assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/watch/3msbfr6pBNE").getId());
assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/v/3msbfr6pBNE").getId());
assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/w/3msbfr6pBNE").getId());
assertEquals("3msbfr6pBNE", linkHandler.fromUrl("hooktube.com/embed/3msbfr6pBNE").getId());
}
@Test
public void testAcceptInvidioUrl() throws ParsingException {
public void testAcceptInvidiousUrl() throws ParsingException {
assertTrue(linkHandler.acceptUrl("https://invidio.us/watch?v=TglNG-yjabU"));
assertTrue(linkHandler.acceptUrl("http://www.invidio.us/watch?v=TglNG-yjabU"));
assertTrue(linkHandler.acceptUrl("http://invidio.us/watch?v=TglNG-yjabU"));
assertTrue(linkHandler.acceptUrl("invidio.us/watch?v=3msbfr6pBNE"));
assertTrue(linkHandler.acceptUrl("https://invidio.us/watch?v=ocH3oSnZG3c&test=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2"));
assertTrue(linkHandler.acceptUrl("invidio.us/embed/3msbfr6pBNE"));
assertTrue(linkHandler.acceptUrl("invidio.us/watch/3msbfr6pBNE"));
assertTrue(linkHandler.acceptUrl("invidio.us/v/3msbfr6pBNE"));
assertTrue(linkHandler.acceptUrl("invidio.us/w/3msbfr6pBNE"));
}
@Test
public void testGetInvidioIdfromUrl() throws ParsingException {
public void testGetInvidiousIdfromUrl() throws ParsingException {
assertEquals("TglNG-yjabU", linkHandler.fromUrl("https://invidio.us/watch?v=TglNG-yjabU").getId());
assertEquals("TglNG-yjabU", linkHandler.fromUrl("http://www.invidio.us/watch?v=TglNG-yjabU").getId());
assertEquals("TglNG-yjabU", linkHandler.fromUrl("http://invidio.us/watch?v=TglNG-yjabU").getId());
assertEquals("3msbfr6pBNE", linkHandler.fromUrl("invidio.us/watch?v=3msbfr6pBNE").getId());
assertEquals("ocH3oSnZG3c", linkHandler.fromUrl("https://invidio.us/watch?v=ocH3oSnZG3c&test=PLS2VU1j4vzuZwooPjV26XM9UEBY2CPNn2").getId());
assertEquals("3msbfr6pBNE", linkHandler.fromUrl("invidio.us/embed/3msbfr6pBNE").getId());
assertEquals("3msbfr6pBNE", linkHandler.fromUrl("invidio.us/v/3msbfr6pBNE").getId());
assertEquals("3msbfr6pBNE", linkHandler.fromUrl("invidio.us/w/3msbfr6pBNE").getId());
assertEquals("3msbfr6pBNE", linkHandler.fromUrl("invidio.us/watch/3msbfr6pBNE").getId());
}
}
\ No newline at end of file
......@@ -12,25 +12,25 @@ import org.schabi.newpipe.extractor.subscription.SubscriptionItem;
import java.io.ByteArrayInputStream;
import java.io.FileInputStream;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.junit.Assert.*;
import static org.schabi.newpipe.FileUtils.resolveTestResource;
import static org.schabi.newpipe.extractor.utils.Utils.UTF_8;
/**
* Test for {@link YoutubeSubscriptionExtractor}
*/
public class YoutubeSubscriptionExtractorTest {
private static YoutubeSubscriptionExtractor subscriptionExtractor;
private static LinkHandlerFactory urlHandler;
@BeforeClass
public static void setupClass() {
//Doesn't make network requests
NewPipe.init(DownloaderTestImpl.getInstance());
subscriptionExtractor = new YoutubeSubscriptionExtractor(ServiceList.YouTube);
urlHandler = ServiceList.YouTube.getChannelLHFactory();
......@@ -53,7 +53,7 @@ public class YoutubeSubscriptionExtractorTest {
@Test
public void testEmptySourceException() throws Exception {
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
new ByteArrayInputStream("[]".getBytes(StandardCharsets.UTF_8)));
new ByteArrayInputStream("[]".getBytes(UTF_8)));
assertTrue(items.isEmpty());
}
......@@ -61,7 +61,7 @@ public class YoutubeSubscriptionExtractorTest {
public void testSubscriptionWithEmptyTitleInSource() throws Exception {
final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"}}}]";
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8)));
new ByteArrayInputStream(source.getBytes(UTF_8)));
assertEquals(1, items.size());
assertEquals(ServiceList.YouTube.getServiceId(), items.get(0).getServiceId());
......@@ -74,7 +74,7 @@ public class YoutubeSubscriptionExtractorTest {
final String source = "[{\"snippet\":{\"resourceId\":{\"channelId\":\"gibberish\"},\"title\":\"name1\"}}," +
"{\"snippet\":{\"resourceId\":{\"channelId\":\"UCEOXxzW2vU0P-0THehuIIeg\"},\"title\":\"name2\"}}]";
final List<SubscriptionItem> items = subscriptionExtractor.fromInputStream(
new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8)));
new ByteArrayInputStream(source.getBytes(UTF_8)));
assertEquals(1, items.size());
assertEquals(ServiceList.YouTube.getServiceId(), items.get(0).getServiceId());
......@@ -98,7 +98,7 @@ public class YoutubeSubscriptionExtractorTest {
for (String invalidContent : invalidList) {
try {
byte[] bytes = invalidContent.getBytes(StandardCharsets.UTF_8);
byte[] bytes = invalidContent.getBytes(UTF_8);
subscriptionExtractor.fromInputStream(new ByteArrayInputStream(bytes));
fail("Extracting from \"" + invalidContent + "\" didn't throw an exception");
} catch (final Exception e) {
......
......@@ -22,7 +22,7 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.localization.Localization;
......@@ -37,11 +37,15 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube;
* Test for {@link SuggestionExtractor}
*/
public class YoutubeSuggestionExtractorTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/suggestions/";
private static SuggestionExtractor suggestionExtractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance(), new Localization("de", "DE"));
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + ""), new Localization("de", "DE"));
suggestionExtractor = YouTube.getSuggestionExtractor();
}
......
......@@ -22,7 +22,7 @@ package org.schabi.newpipe.extractor.services.youtube;
import org.junit.BeforeClass;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.kiosk.KioskInfo;
......@@ -36,14 +36,17 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube;
* Test for {@link KioskInfo}
*/
public class YoutubeTrendingKioskInfoTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "kiosk";
static KioskInfo kioskInfo;
@BeforeClass
public static void setUp()
throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
StreamingService service = YouTube;
LinkHandlerFactory LinkHandlerFactory = service.getKioskList().getListLinkHandlerFactoryByType("Trending");
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH));
LinkHandlerFactory LinkHandlerFactory = ((StreamingService) YouTube).getKioskList().getListLinkHandlerFactoryByType("Trending");
kioskInfo = KioskInfo.getInfo(YouTube, LinkHandlerFactory.fromId("Trending").getUrl());
}
......
......@@ -10,12 +10,14 @@ import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory;
import javax.annotation.Nullable;
import java.net.URLEncoder;
import javax.annotation.Nullable;
import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
// Doesn't work with mocks. Makes request with different `dataToSend` i think
public class YoutubeMusicSearchExtractorTest {
public static class MusicSongs extends DefaultSearchExtractorTest {
private static SearchExtractor extractor;
......
package org.schabi.newpipe.extractor.services.youtube.search;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.*;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.InfoItem;
import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.YoutubeService;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.extractor.channel.ChannelInfoItem;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import javax.annotation.Nullable;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
......@@ -31,13 +38,17 @@ import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeS
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.VIDEOS;
public class YoutubeSearchExtractorTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/search/";
public static class All extends DefaultSearchExtractorTest {
private static SearchExtractor extractor;
private static final String QUERY = "test";
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "all"));
extractor = YouTube.getSearchExtractor(QUERY);
extractor.fetchPage();
}
......@@ -58,7 +69,8 @@ public class YoutubeSearchExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "channel"));
extractor = YouTube.getSearchExtractor(QUERY, singletonList(CHANNELS), "");
extractor.fetchPage();
}
......@@ -81,7 +93,8 @@ public class YoutubeSearchExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "playlist"));
extractor = YouTube.getSearchExtractor(QUERY, singletonList(PLAYLISTS), "");
extractor.fetchPage();
}
......@@ -104,7 +117,8 @@ public class YoutubeSearchExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "videos"));
extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), "");
extractor.fetchPage();
}
......@@ -128,7 +142,8 @@ public class YoutubeSearchExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "suggestions"));
extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), "");
extractor.fetchPage();
}
......@@ -151,7 +166,8 @@ public class YoutubeSearchExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "corrected"));
extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), "");
extractor.fetchPage();
}
......@@ -174,7 +190,8 @@ public class YoutubeSearchExtractorTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "random"));
extractor = YouTube.getSearchExtractor(QUERY);
extractor.fetchPage();
}
......@@ -208,7 +225,8 @@ public class YoutubeSearchExtractorTest {
public static class PagingTest {
@Test
public void duplicatedItemsCheck() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "paging"));
final SearchExtractor extractor = YouTube.getSearchExtractor("cirque du soleil", singletonList(VIDEOS), "");
extractor.fetchPage();
......@@ -219,14 +237,14 @@ public class YoutubeSearchExtractorTest {
}
}
@Ignore("TODO fix")
public static class MetaInfoTest extends DefaultSearchExtractorTest {
private static SearchExtractor extractor;
private static final String QUERY = "Covid";
@Test
public void clarificationTest() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "metaInfo"));
extractor = YouTube.getSearchExtractor(QUERY, singletonList(VIDEOS), "");
extractor.fetchPage();
}
......@@ -255,4 +273,42 @@ public class YoutubeSearchExtractorTest {
@Override public String expectedOriginalUrlContains() throws Exception { return "youtube.com/results?search_query=" + QUERY; }
}
public static class ChannelVerified extends DefaultSearchExtractorTest {
private static SearchExtractor extractor;
private static final String QUERY = "bbc";
@BeforeClass
public static void setUp() throws Exception {
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "verified"));
extractor = YouTube.getSearchExtractor(QUERY, singletonList(CHANNELS), "");
extractor.fetchPage();
}
@Override public SearchExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return QUERY; }
@Override public String expectedId() { return QUERY; }
@Override public String expectedUrlContains() { return "youtube.com/results?search_query=" + QUERY; }
@Override public String expectedOriginalUrlContains() { return "youtube.com/results?search_query=" + QUERY; }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.CHANNEL; }
@Test
public void testAtLeastOneVerified() throws IOException, ExtractionException {
final List<InfoItem> items = extractor.getInitialPage().getItems();
boolean verified = false;
for (InfoItem item : items) {
if (((ChannelInfoItem) item).isVerified()) {
verified = true;
break;
}
}
assertTrue(verified);
}
}
}
package org.schabi.newpipe.extractor.services.youtube.search;
import org.junit.Ignore;
import org.junit.Test;
import static java.util.Arrays.asList;
import static org.junit.Assert.assertEquals;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.*;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.CHANNELS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_SONGS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.PLAYLISTS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.VIDEOS;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
public class YoutubeSearchQHTest {
@Test
@Ignore("TODO fix")
public void testRegularValues() throws Exception {
assertEquals("https://www.youtube.com/results?search_query=asdf", YouTube.getSearchQHFactory().fromQuery("asdf").getUrl());
assertEquals("https://www.youtube.com/results?search_query=hans", YouTube.getSearchQHFactory().fromQuery("hans").getUrl());
......@@ -19,37 +21,37 @@ public class YoutubeSearchQHTest {
assertEquals("https://www.youtube.com/results?search_query=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm").getUrl());
assertEquals("https://www.youtube.com/results?search_query=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B").getUrl());
assertEquals("https://music.youtube.com/search?q=asdf", YouTube.getSearchQHFactory().fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getUrl());
assertEquals("https://music.youtube.com/search?q=hans", YouTube.getSearchQHFactory().fromQuery("hans", asList(new String[]{MUSIC_SONGS}), "").getUrl());
assertEquals("https://music.youtube.com/search?q=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf", asList(new String[]{MUSIC_SONGS}), "").getUrl());
assertEquals("https://music.youtube.com/search?q=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm", asList(new String[]{MUSIC_SONGS}), "").getUrl());
assertEquals("https://music.youtube.com/search?q=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B", asList(new String[]{MUSIC_SONGS}), "").getUrl());
assertEquals("https://music.youtube.com/search?q=asdf", YouTube.getSearchQHFactory().fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
assertEquals("https://music.youtube.com/search?q=hans", YouTube.getSearchQHFactory().fromQuery("hans", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
assertEquals("https://music.youtube.com/search?q=Poifj%26jaijf", YouTube.getSearchQHFactory().fromQuery("Poifj&jaijf", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
assertEquals("https://music.youtube.com/search?q=G%C3%BCl%C3%BCm", YouTube.getSearchQHFactory().fromQuery("Gülüm", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
assertEquals("https://music.youtube.com/search?q=%3Fj%24%29H%C2%A7B", YouTube.getSearchQHFactory().fromQuery("?j$)H§B", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
}
@Test
public void testGetContentFilter() throws Exception {
assertEquals(VIDEOS, YouTube.getSearchQHFactory()
.fromQuery("", asList(new String[]{VIDEOS}), "").getContentFilters().get(0));
.fromQuery(EMPTY_STRING, asList(new String[]{VIDEOS}), EMPTY_STRING).getContentFilters().get(0));
assertEquals(CHANNELS, YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{CHANNELS}), "").getContentFilters().get(0));
.fromQuery("asdf", asList(new String[]{CHANNELS}), EMPTY_STRING).getContentFilters().get(0));
assertEquals(MUSIC_SONGS, YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getContentFilters().get(0));
.fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getContentFilters().get(0));
}
@Test
public void testWithContentfilter() throws Exception {
assertEquals("https://www.youtube.com/results?search_query=asdf&sp=EgIQAQ%253D%253D", YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{VIDEOS}), "").getUrl());
.fromQuery("asdf", asList(new String[]{VIDEOS}), EMPTY_STRING).getUrl());
assertEquals("https://www.youtube.com/results?search_query=asdf&sp=EgIQAg%253D%253D", YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{CHANNELS}), "").getUrl());
.fromQuery("asdf", asList(new String[]{CHANNELS}), EMPTY_STRING).getUrl());
assertEquals("https://www.youtube.com/results?search_query=asdf&sp=EgIQAw%253D%253D", YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{PLAYLISTS}), "").getUrl());
.fromQuery("asdf", asList(new String[]{PLAYLISTS}), EMPTY_STRING).getUrl());
assertEquals("https://www.youtube.com/results?search_query=asdf", YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{"fjiijie"}), "").getUrl());
.fromQuery("asdf", asList(new String[]{"fjiijie"}), EMPTY_STRING).getUrl());
assertEquals("https://music.youtube.com/search?q=asdf", YouTube.getSearchQHFactory()
.fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), "").getUrl());
.fromQuery("asdf", asList(new String[]{MUSIC_SONGS}), EMPTY_STRING).getUrl());
}
@Test
......
package org.schabi.newpipe.extractor.services.youtube.stream;
import org.junit.BeforeClass;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
......@@ -16,6 +17,7 @@ import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
public class YoutubeStreamExtractorAgeRestrictedTest extends DefaultStreamExtractorTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
private static final String ID = "MmBeUZqv1QA";
private static final int TIMESTAMP = 196;
private static final String URL = YoutubeStreamExtractorDefaultTest.BASE_URL + ID + "&t=" + TIMESTAMP;
......@@ -23,7 +25,8 @@ public class YoutubeStreamExtractorAgeRestrictedTest extends DefaultStreamExtrac
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "ageRestricted"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}
......@@ -38,6 +41,7 @@ public class YoutubeStreamExtractorAgeRestrictedTest extends DefaultStreamExtrac
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
@Override public String expectedUploaderName() { return "EpicFiveTV"; }
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCuPUHlLP5POZphOIrjrNxiw"; }
@Override public boolean expectedUploaderVerified() { return true; }
@Override public List<String> expectedDescriptionContains() { return Arrays.asList("http://instagram.com/Ruben_Sole", "AVN"); }
@Override public long expectedLength() { return 1790; }
@Override public long expectedTimestamp() { return TIMESTAMP; }
......
......@@ -2,11 +2,11 @@ package org.schabi.newpipe.extractor.services.youtube.stream;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
......@@ -21,14 +21,17 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube;
/**
* Test for {@link YoutubeStreamLinkHandlerFactory}
*/
@Ignore("Video is not available anymore")
public class YoutubeStreamExtractorControversialTest extends DefaultStreamExtractorTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
private static final String ID = "T4XJQO3qol8";
private static final String URL = YoutubeStreamExtractorDefaultTest.BASE_URL + ID;
private static StreamExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "controversial"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}
......
......@@ -3,29 +3,31 @@ package org.schabi.newpipe.extractor.services.youtube.stream;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import javax.annotation.Nullable;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import javax.annotation.Nullable;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
/*
* Created by Christian Schabesberger on 30.12.15.
......@@ -47,12 +49,14 @@ import static org.schabi.newpipe.extractor.ServiceList.YouTube;
* along with NewPipe. If not, see <http://www.gnu.org/licenses/>.
*/
public class YoutubeStreamExtractorDefaultTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
static final String BASE_URL = "https://www.youtube.com/watch?v=";
public static class NotAvailable {
@BeforeClass
public static void setUp() {
NewPipe.init(DownloaderTestImpl.getInstance());
public static void setUp() throws IOException {
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "notAvailable"));
}
@Test(expected = ContentNotAvailableException.class)
......@@ -78,11 +82,13 @@ public class YoutubeStreamExtractorDefaultTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "pewdiwpie"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}
// @formatter:off
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "Marzia & Felix - Wedding 19.08.2019"; }
......@@ -97,6 +103,7 @@ public class YoutubeStreamExtractorDefaultTest {
return Arrays.asList("https://www.youtube.com/channel/UC7l23W7gFi4Uho6WSzckZRA",
"https://www.handcraftpictures.com/");
}
@Override public boolean expectedUploaderVerified() { return true; }
@Override public long expectedLength() { return 381; }
@Override public long expectedTimestamp() { return TIMESTAMP; }
@Override public long expectedViewCountAtLeast() { return 26682500; }
......@@ -105,9 +112,9 @@ public class YoutubeStreamExtractorDefaultTest {
@Override public long expectedLikeCountAtLeast() { return 5212900; }
@Override public long expectedDislikeCountAtLeast() { return 30600; }
@Override public int expectedStreamSegmentsCount() { return 0; }
// @formatter:on
}
@Ignore("TODO fix")
public static class DescriptionTestUnboxing extends DefaultStreamExtractorTest {
private static final String ID = "cV5TjZCJkuA";
private static final String URL = BASE_URL + ID;
......@@ -115,11 +122,13 @@ public class YoutubeStreamExtractorDefaultTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "unboxing"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}
// @formatter:off
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "This Smartphone Changes Everything..."; }
......@@ -142,6 +151,15 @@ public class YoutubeStreamExtractorDefaultTest {
@Nullable @Override public String expectedTextualUploadDate() { return "2018-06-19"; }
@Override public long expectedLikeCountAtLeast() { return 340100; }
@Override public long expectedDislikeCountAtLeast() { return 18700; }
@Override public boolean expectedUploaderVerified() { return true; }
// @formatter:on
@Override
@Test
@Ignore("TODO fix")
public void testDescription() throws Exception {
super.testDescription();
}
}
@Ignore("TODO fix")
......@@ -153,11 +171,13 @@ public class YoutubeStreamExtractorDefaultTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "ratingsDisabled"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}
// @formatter:off
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "AlphaOmegaSin Fanboy Logic: Likes/Dislikes Disabled = Point Invalid Lol wtf?"; }
......@@ -176,6 +196,7 @@ public class YoutubeStreamExtractorDefaultTest {
@Nullable @Override public String expectedTextualUploadDate() { return "2019-01-02"; }
@Override public long expectedLikeCountAtLeast() { return -1; }
@Override public long expectedDislikeCountAtLeast() { return -1; }
// @formatter:on
}
public static class StreamSegmentsTestOstCollection extends DefaultStreamExtractorTest {
......@@ -186,11 +207,13 @@ public class YoutubeStreamExtractorDefaultTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "streamSegmentsOstCollection"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}
// @formatter:off
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "1 Hour - Most Epic Anime Mix - Battle Anime OST"; }
......@@ -221,6 +244,7 @@ public class YoutubeStreamExtractorDefaultTest {
assertEquals(BASE_URL + ID + "?t=589", segment.getUrl());
assertNotNull(segment.getPreviewUrl());
}
// @formatter:on
}
public static class StreamSegmentsTestMaiLab extends DefaultStreamExtractorTest {
......@@ -231,11 +255,13 @@ public class YoutubeStreamExtractorDefaultTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "streamSegmentsMaiLab"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}
// @formatter:off
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "Vitamin D wissenschaftlich gepr\u00fcft"; }
......@@ -246,9 +272,8 @@ public class YoutubeStreamExtractorDefaultTest {
@Override public StreamType expectedStreamType() { return StreamType.VIDEO_STREAM; }
@Override public String expectedUploaderName() { return "maiLab"; }
@Override public String expectedUploaderUrl() { return "https://www.youtube.com/channel/UCyHDQ5C6z1NDmJ4g6SerW8g"; }
@Override public List<String> expectedDescriptionContains() {
return Arrays.asList("Vitamin", "2:44", "Was ist Vitamin D?");
}
@Override public List<String> expectedDescriptionContains() {return Arrays.asList("Vitamin", "2:44", "Was ist Vitamin D?");}
@Override public boolean expectedUploaderVerified() { return true; }
@Override public long expectedLength() { return 1010; }
@Override public long expectedViewCountAtLeast() { return 815500; }
@Nullable @Override public String expectedUploadDate() { return "2020-11-18 00:00:00.000"; }
......@@ -256,8 +281,9 @@ public class YoutubeStreamExtractorDefaultTest {
@Override public long expectedLikeCountAtLeast() { return 48500; }
@Override public long expectedDislikeCountAtLeast() { return 20000; }
@Override public boolean expectedHasSubtitles() { return true; }
@Override public int expectedStreamSegmentsCount() { return 7; }
// @formatter:on
@Test
public void testStreamSegment() throws Exception {
final StreamSegment segment = extractor.getStreamSegments().get(1);
......@@ -266,6 +292,13 @@ public class YoutubeStreamExtractorDefaultTest {
assertEquals(BASE_URL + ID + "?t=164", segment.getUrl());
assertNotNull(segment.getPreviewUrl());
}
@Override
@Test
@Ignore("encoding problem")
public void testName() throws Exception {
super.testName();
}
}
public static class PublicBroadcasterTest extends DefaultStreamExtractorTest {
......@@ -276,11 +309,13 @@ public class YoutubeStreamExtractorDefaultTest {
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "publicBroadcast"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}
// @formatter:off
@Override public StreamExtractor extractor() { return extractor; }
@Override public StreamingService expectedService() { return YouTube; }
@Override public String expectedName() { return "Was verbirgt sich am tiefsten Punkt des Ozeans?"; }
......@@ -301,12 +336,28 @@ public class YoutubeStreamExtractorDefaultTest {
@Override public long expectedDislikeCountAtLeast() { return 500; }
@Override public List<MetaInfo> expectedMetaInfo() throws MalformedURLException {
return Collections.singletonList(new MetaInfo(
"",
EMPTY_STRING,
new Description("Funk is a German public broadcast service.", Description.PLAIN_TEXT),
Collections.singletonList(new URL("https://de.wikipedia.org/wiki/Funk_(Medienangebot)?wprov=yicw1")),
Collections.singletonList("Wikipedia (German)")
));
}
@Override public boolean expectedUploaderVerified() { return true; }
// @formatter:on
@Override
@Ignore("TODO fix")
@Test
public void testUploaderName() throws Exception {
super.testUploaderName();
}
@Override
@Ignore("TODO fix")
@Test
public void testMetaInfo() throws Exception {
super.testMetaInfo();
}
}
}
package org.schabi.newpipe.extractor.services.youtube.stream;
import org.junit.BeforeClass;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
......@@ -16,6 +17,7 @@ import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
public class YoutubeStreamExtractorLivestreamTest extends DefaultStreamExtractorTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
private static final String ID = "5qap5aO4i9A";
private static final int TIMESTAMP = 1737;
private static final String URL = YoutubeStreamExtractorDefaultTest.BASE_URL + ID + "&t=" + TIMESTAMP;
......@@ -23,7 +25,8 @@ public class YoutubeStreamExtractorLivestreamTest extends DefaultStreamExtractor
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "live"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}
......@@ -42,6 +45,7 @@ public class YoutubeStreamExtractorLivestreamTest extends DefaultStreamExtractor
return Arrays.asList("https://bit.ly/chilledcow-playlists",
"https://bit.ly/chilledcow-submissions");
}
@Override public boolean expectedUploaderVerified() { return true; }
@Override public long expectedLength() { return 0; }
@Override public long expectedTimestamp() { return TIMESTAMP; }
@Override public long expectedViewCountAtLeast() { return 0; }
......
package org.schabi.newpipe.extractor.services.youtube.stream;
import org.junit.BeforeClass;
import org.schabi.newpipe.downloader.DownloaderTestImpl;
import org.schabi.newpipe.downloader.DownloaderFactory;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;
......@@ -16,13 +17,15 @@ import javax.annotation.Nullable;
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
public class YoutubeStreamExtractorUnlistedTest extends DefaultStreamExtractorTest {
private static final String RESOURCE_PATH = DownloaderFactory.RESOURCE_PATH + "services/youtube/extractor/stream/";
static final String ID = "udsB8KnIJTg";
static final String URL = YoutubeStreamExtractorDefaultTest.BASE_URL + ID;
private static StreamExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
YoutubeParsingHelper.resetClientVersionAndKey();
NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "unlisted"));
extractor = YouTube.getStreamExtractor(URL);
extractor.fetchPage();
}
......
This diff is collapsed.