diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java
index aa7987156d..118c6dc678 100644
--- a/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java
+++ b/extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java
@@ -3,6 +3,7 @@
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.localization.Localization;
+import org.schabi.newpipe.extractor.utils.Utils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@@ -25,10 +26,10 @@ public abstract class Downloader {
* localization. It should only be used when the resource that will be fetched won't be affected
* by the localization.
*
- * @param url the URL that is pointing to the wanted resource
+ * @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @return the result of the GET request
*/
- public Response get(final String url) throws IOException, ReCaptchaException {
+ public final Response get(final String url) throws IOException, ReCaptchaException {
return get(url, null, NewPipe.getPreferredLocalization());
}
@@ -37,11 +38,11 @@ public Response get(final String url) throws IOException, ReCaptchaException {
*
* It will set the {@code Accept-Language} header to the language of the localization parameter.
*
- * @param url the URL that is pointing to the wanted resource
+ * @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param localization the source of the value of the {@code Accept-Language} header
* @return the result of the GET request
*/
- public Response get(final String url, final Localization localization)
+ public final Response get(final String url, final Localization localization)
throws IOException, ReCaptchaException {
return get(url, null, localization);
}
@@ -49,12 +50,12 @@ public Response get(final String url, final Localization localization)
/**
* Do a GET request with the specified headers.
*
- * @param url the URL that is pointing to the wanted resource
+ * @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers should be overridden by these.
* @return the result of the GET request
*/
- public Response get(final String url, @Nullable final Map> headers)
+ public final Response get(final String url, @Nullable final Map> headers)
throws IOException, ReCaptchaException {
return get(url, headers, NewPipe.getPreferredLocalization());
}
@@ -64,44 +65,42 @@ public Response get(final String url, @Nullable final Map>
*
* It will set the {@code Accept-Language} header to the language of the localization parameter.
*
- * @param url the URL that is pointing to the wanted resource
+ * @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers should be overridden by these.
* @param localization the source of the value of the {@code Accept-Language} header
* @return the result of the GET request
*/
- public Response get(final String url,
+ public final Response get(final String url,
@Nullable final Map> headers,
final Localization localization)
throws IOException, ReCaptchaException {
- return execute(Request.newBuilder()
+ return executeIfHttps(Request.newBuilder()
.get(url)
- .headers(headers)
- .localization(localization)
.build());
}
/**
* Do a HEAD request.
*
- * @param url the URL that is pointing to the wanted resource
+ * @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @return the result of the HEAD request
*/
- public Response head(final String url) throws IOException, ReCaptchaException {
+ public final Response head(final String url) throws IOException, ReCaptchaException {
return head(url, null);
}
/**
* Do a HEAD request with the specified headers.
*
- * @param url the URL that is pointing to the wanted resource
+ * @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers should be overridden by these.
* @return the result of the HEAD request
*/
- public Response head(final String url, @Nullable final Map> headers)
+ public final Response head(final String url, @Nullable final Map> headers)
throws IOException, ReCaptchaException {
- return execute(Request.newBuilder()
+ return executeIfHttps(Request.newBuilder()
.head(url)
.headers(headers)
.build());
@@ -110,13 +109,13 @@ public Response head(final String url, @Nullable final Map>
/**
* Do a POST request with the specified headers, sending the data array.
*
- * @param url the URL that is pointing to the wanted resource
+ * @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers should be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
* @return the result of the POST request
*/
- public Response post(final String url,
+ public final Response post(final String url,
@Nullable final Map> headers,
@Nullable final byte[] dataToSend)
throws IOException, ReCaptchaException {
@@ -128,19 +127,19 @@ public Response post(final String url,
*
* It will set the {@code Accept-Language} header to the language of the localization parameter.
*
- * @param url the URL that is pointing to the wanted resource
+ * @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers should be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
* @param localization the source of the value of the {@code Accept-Language} header
* @return the result of the POST request
*/
- public Response post(final String url,
+ public final Response post(final String url,
@Nullable final Map> headers,
@Nullable final byte[] dataToSend,
final Localization localization)
throws IOException, ReCaptchaException {
- return execute(Request.newBuilder()
+ return executeIfHttps(Request.newBuilder()
.post(url, dataToSend)
.headers(headers)
.localization(localization)
@@ -151,7 +150,7 @@ public Response post(final String url,
* Convenient method to send a POST request using the specified value of the
* {@code Content-Type} header with a given {@link Localization}.
*
- * @param url the URL that is pointing to the wanted resource
+ * @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers should be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
@@ -161,7 +160,7 @@ public Response post(final String url,
* @return the result of the POST request
* @see #post(String, Map, byte[], Localization)
*/
- public Response postWithContentType(final String url,
+ public final Response postWithContentType(final String url,
@Nullable final Map> headers,
@Nullable final byte[] dataToSend,
final Localization localization,
@@ -179,7 +178,7 @@ public Response postWithContentType(final String url,
* Convenient method to send a POST request using the specified value of the
* {@code Content-Type} header.
*
- * @param url the URL that is pointing to the wanted resource
+ * @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers should be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
@@ -188,7 +187,7 @@ public Response postWithContentType(final String url,
* @return the result of the POST request
* @see #post(String, Map, byte[], Localization)
*/
- public Response postWithContentType(final String url,
+ public final Response postWithContentType(final String url,
@Nullable final Map> headers,
@Nullable final byte[] dataToSend,
final String contentType)
@@ -201,7 +200,7 @@ public Response postWithContentType(final String url,
* Convenient method to send a POST request the JSON mime type as the value of the
* {@code Content-Type} header with a given {@link Localization}.
*
- * @param url the URL that is pointing to the wanted resource
+ * @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers should be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
@@ -209,7 +208,7 @@ public Response postWithContentType(final String url,
* @return the result of the POST request
* @see #post(String, Map, byte[], Localization)
*/
- public Response postWithContentTypeJson(final String url,
+ public final Response postWithContentTypeJson(final String url,
@Nullable final Map> headers,
@Nullable final byte[] dataToSend,
final Localization localization)
@@ -221,14 +220,14 @@ public Response postWithContentTypeJson(final String url,
* Convenient method to send a POST request the JSON mime type as the value of the
* {@code Content-Type} header.
*
- * @param url the URL that is pointing to the wanted resource
+ * @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers should be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
* @return the result of the POST request
* @see #post(String, Map, byte[], Localization)
*/
- public Response postWithContentTypeJson(final String url,
+ public final Response postWithContentTypeJson(final String url,
@Nullable final Map> headers,
@Nullable final byte[] dataToSend)
throws IOException, ReCaptchaException {
@@ -236,11 +235,24 @@ public Response postWithContentTypeJson(final String url,
NewPipe.getPreferredLocalization());
}
+ public final Response executeIfHttps(final @Nonnull Request request)
+ throws IOException, ReCaptchaException {
+
+ if (!request.url().equals(Utils.replaceHttpWithHttps(request.url()))) {
+ throw new IOException(
+ "All queries must be made using HTTPS. Extractors must guarantee "
+ + "that HTTPS links are provided."
+ );
+ } else {
+ return execute(request);
+ }
+ }
+
/**
* Do a request using the specified {@link Request} object.
*
* @return the result of the request
*/
- public abstract Response execute(@Nonnull Request request)
+ protected abstract Response execute(@Nonnull Request request)
throws IOException, ReCaptchaException;
}
diff --git a/extractor/src/test/java/org/schabi/newpipe/downloader/DownloaderTestImpl.java b/extractor/src/test/java/org/schabi/newpipe/downloader/DownloaderTestImpl.java
index 3461269914..0697c30ce2 100644
--- a/extractor/src/test/java/org/schabi/newpipe/downloader/DownloaderTestImpl.java
+++ b/extractor/src/test/java/org/schabi/newpipe/downloader/DownloaderTestImpl.java
@@ -47,7 +47,7 @@ public static DownloaderTestImpl getInstance() {
}
@Override
- public Response execute(@Nonnull final Request request)
+ protected Response execute(@Nonnull final Request request)
throws IOException, ReCaptchaException {
final String httpMethod = request.httpMethod();
final String url = request.url();
diff --git a/extractor/src/test/java/org/schabi/newpipe/downloader/MockDownloader.java b/extractor/src/test/java/org/schabi/newpipe/downloader/MockDownloader.java
index bcf72b213e..c77a9ca69e 100644
--- a/extractor/src/test/java/org/schabi/newpipe/downloader/MockDownloader.java
+++ b/extractor/src/test/java/org/schabi/newpipe/downloader/MockDownloader.java
@@ -46,7 +46,7 @@ public MockDownloader(@Nonnull final String path) throws IOException {
}
@Override
- public Response execute(@Nonnull final Request request) {
+ protected Response execute(@Nonnull final Request request) {
final Response result = mocks.get(request);
if (result == null) {
throw new NullPointerException("No mock response for request with url '" + request
diff --git a/extractor/src/test/java/org/schabi/newpipe/downloader/RecordingDownloader.java b/extractor/src/test/java/org/schabi/newpipe/downloader/RecordingDownloader.java
index 7d649732bb..9fcb4142b4 100644
--- a/extractor/src/test/java/org/schabi/newpipe/downloader/RecordingDownloader.java
+++ b/extractor/src/test/java/org/schabi/newpipe/downloader/RecordingDownloader.java
@@ -70,7 +70,7 @@ public RecordingDownloader(final String stringPath) throws IOException {
public Response execute(@Nonnull final Request request) throws IOException,
ReCaptchaException {
final Downloader downloader = DownloaderTestImpl.getInstance();
- Response response = downloader.execute(request);
+ Response response = downloader.executeIfHttps(request);
String cleanedResponseBody = response.responseBody().replaceAll(IP_V4_PATTERN, "127.0.0.1");
response = new Response(
response.responseCode(),