Skip to content

Commit

Permalink
Throw upon trying to fetch unencrypted URL
Browse files Browse the repository at this point in the history
  • Loading branch information
@fynngodau
fynngodau committed May 24, 2024
1 parent fafd471 commit ec019d3
Show file tree
Hide file tree
Showing 4 changed files with 45 additions and 33 deletions.
72 changes: 42 additions & 30 deletions extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.utils.Utils;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
Expand All @@ -25,10 +26,10 @@ public abstract class Downloader {
* localization. It should only be used when the resource that will be fetched won't be affected
* by the localization.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @return the result of the GET request
*/
public Response get(final String url) throws IOException, ReCaptchaException {
public final Response get(final String url) throws IOException, ReCaptchaException {
return get(url, null, NewPipe.getPreferredLocalization());
}

Expand All @@ -37,24 +38,24 @@ public Response get(final String url) throws IOException, ReCaptchaException {
* <br>
* It will set the {@code Accept-Language} header to the language of the localization parameter.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param localization the source of the value of the {@code Accept-Language} header
* @return the result of the GET request
*/
public Response get(final String url, final Localization localization)
public final Response get(final String url, final Localization localization)
throws IOException, ReCaptchaException {
return get(url, null, localization);
}

/**
* Do a GET request with the specified headers.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @return the result of the GET request
*/
public Response get(final String url, @Nullable final Map<String, List<String>> headers)
public final Response get(final String url, @Nullable final Map<String, List<String>> headers)
throws IOException, ReCaptchaException {
return get(url, headers, NewPipe.getPreferredLocalization());
}
Expand All @@ -64,44 +65,42 @@ public Response get(final String url, @Nullable final Map<String, List<String>>
* <br>
* It will set the {@code Accept-Language} header to the language of the localization parameter.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param localization the source of the value of the {@code Accept-Language} header
* @return the result of the GET request
*/
public Response get(final String url,
public final Response get(final String url,
@Nullable final Map<String, List<String>> headers,
final Localization localization)
throws IOException, ReCaptchaException {
return execute(Request.newBuilder()
return executeIfHttps(Request.newBuilder()
.get(url)
.headers(headers)
.localization(localization)
.build());
}

/**
* Do a HEAD request.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @return the result of the HEAD request
*/
public Response head(final String url) throws IOException, ReCaptchaException {
public final Response head(final String url) throws IOException, ReCaptchaException {
return head(url, null);
}

/**
* Do a HEAD request with the specified headers.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @return the result of the HEAD request
*/
public Response head(final String url, @Nullable final Map<String, List<String>> headers)
public final Response head(final String url, @Nullable final Map<String, List<String>> headers)
throws IOException, ReCaptchaException {
return execute(Request.newBuilder()
return executeIfHttps(Request.newBuilder()
.head(url)
.headers(headers)
.build());
Expand All @@ -110,13 +109,13 @@ public Response head(final String url, @Nullable final Map<String, List<String>>
/**
* Do a POST request with the specified headers, sending the data array.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
* @return the result of the POST request
*/
public Response post(final String url,
public final Response post(final String url,
@Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend)
throws IOException, ReCaptchaException {
Expand All @@ -128,19 +127,19 @@ public Response post(final String url,
* <br>
* It will set the {@code Accept-Language} header to the language of the localization parameter.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
* @param localization the source of the value of the {@code Accept-Language} header
* @return the result of the POST request
*/
public Response post(final String url,
public final Response post(final String url,
@Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend,
final Localization localization)
throws IOException, ReCaptchaException {
return execute(Request.newBuilder()
return executeIfHttps(Request.newBuilder()
.post(url, dataToSend)
.headers(headers)
.localization(localization)
Expand All @@ -151,7 +150,7 @@ public Response post(final String url,
* Convenient method to send a POST request using the specified value of the
* {@code Content-Type} header with a given {@link Localization}.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
Expand All @@ -161,7 +160,7 @@ public Response post(final String url,
* @return the result of the POST request
* @see #post(String, Map, byte[], Localization)
*/
public Response postWithContentType(final String url,
public final Response postWithContentType(final String url,
@Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend,
final Localization localization,
Expand All @@ -179,7 +178,7 @@ public Response postWithContentType(final String url,
* Convenient method to send a POST request using the specified value of the
* {@code Content-Type} header.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
Expand All @@ -188,7 +187,7 @@ public Response postWithContentType(final String url,
* @return the result of the POST request
* @see #post(String, Map, byte[], Localization)
*/
public Response postWithContentType(final String url,
public final Response postWithContentType(final String url,
@Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend,
final String contentType)
Expand All @@ -201,15 +200,15 @@ public Response postWithContentType(final String url,
* Convenient method to send a POST request the JSON mime type as the value of the
* {@code Content-Type} header with a given {@link Localization}.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
* @param localization the source of the value of the {@code Accept-Language} header
* @return the result of the POST request
* @see #post(String, Map, byte[], Localization)
*/
public Response postWithContentTypeJson(final String url,
public final Response postWithContentTypeJson(final String url,
@Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend,
final Localization localization)
Expand All @@ -221,26 +220,39 @@ public Response postWithContentTypeJson(final String url,
* Convenient method to send a POST request the JSON mime type as the value of the
* {@code Content-Type} header.
*
* @param url the URL that is pointing to the wanted resource
* @param url the URL that is pointing to the wanted resource (must start with HTTPS)
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
* @return the result of the POST request
* @see #post(String, Map, byte[], Localization)
*/
public Response postWithContentTypeJson(final String url,
public final Response postWithContentTypeJson(final String url,
@Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend)
throws IOException, ReCaptchaException {
return postWithContentTypeJson(url, headers, dataToSend,
NewPipe.getPreferredLocalization());
}

public final Response executeIfHttps(final @Nonnull Request request)
throws IOException, ReCaptchaException {

if (!request.url().equals(Utils.replaceHttpWithHttps(request.url()))) {
throw new IOException(
"All queries must be made using HTTPS. Extractors must guarantee "
+ "that HTTPS links are provided."
);
} else {
return execute(request);
}
}

/**
* Do a request using the specified {@link Request} object.
*
* @return the result of the request
*/
public abstract Response execute(@Nonnull Request request)
protected abstract Response execute(@Nonnull Request request)
throws IOException, ReCaptchaException;
}
2 changes: 1 addition & 1 deletion extractor/src/test/java/org/schabi/newpipe/downloader/DownloaderTestImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ public static DownloaderTestImpl getInstance() {
}

@Override
public Response execute(@Nonnull final Request request)
protected Response execute(@Nonnull final Request request)
throws IOException, ReCaptchaException {
final String httpMethod = request.httpMethod();
final String url = request.url();
Expand Down
2 changes: 1 addition & 1 deletion extractor/src/test/java/org/schabi/newpipe/downloader/MockDownloader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public MockDownloader(@Nonnull final String path) throws IOException {
}

@Override
public Response execute(@Nonnull final Request request) {
protected Response execute(@Nonnull final Request request) {
final Response result = mocks.get(request);
if (result == null) {
throw new NullPointerException("No mock response for request with url '" + request
Expand Down
2 changes: 1 addition & 1 deletion extractor/src/test/java/org/schabi/newpipe/downloader/RecordingDownloader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ public RecordingDownloader(final String stringPath) throws IOException {
public Response execute(@Nonnull final Request request) throws IOException,
ReCaptchaException {
final Downloader downloader = DownloaderTestImpl.getInstance();
Response response = downloader.execute(request);
Response response = downloader.executeIfHttps(request);
String cleanedResponseBody = response.responseBody().replaceAll(IP_V4_PATTERN, "127.0.0.1");
response = new Response(
response.responseCode(),
Expand Down

0 comments on commit ec019d3

Please sign in to comment.