1
0
mirror of https://github.com/RipMeApp/ripme.git synced 2025-08-17 19:26:34 +02:00

cleanup utils/Http.java

This commit is contained in:
soloturn
2021-12-20 02:16:16 +01:00
parent 3bd9d1c412
commit 98cc12d892

View File

@@ -1,28 +1,26 @@
package com.rarchives.ripme.utils;
import com.rarchives.ripme.ripper.AbstractRipper;
import org.apache.commons.lang.ArrayUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.json.JSONObject;
import org.jsoup.Connection;
import org.jsoup.Connection.Method;
import org.jsoup.Connection.Response;
import org.jsoup.HttpStatusException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.json.JSONObject;
import org.jsoup.Connection;
import org.jsoup.Connection.Method;
import org.jsoup.Connection.Response;
import org.jsoup.Jsoup;
import org.jsoup.HttpStatusException;
import org.jsoup.nodes.Document;
import com.rarchives.ripme.ripper.AbstractRipper;
/**
* Wrapper around the Jsoup connection methods.
*
* <p>
* Benefit is retry logic.
*/
public class Http {
@@ -31,7 +29,7 @@ public class Http {
private static final Logger logger = LogManager.getLogger(Http.class);
private int retries;
private String url;
private final String url;
private Connection connection;
// Constructors
@@ -39,6 +37,7 @@ public class Http {
this.url = url;
defaultSettings();
}
private Http(URL url) {
this.url = url.toExternalForm();
defaultSettings();
@@ -47,6 +46,7 @@ public class Http {
public static Http url(String url) {
return new Http(url);
}
public static Http url(URL url) {
return new Http(url);
}
@@ -110,42 +110,52 @@ public class Http {
connection.timeout(timeout);
return this;
}
public Http ignoreContentType() {
connection.ignoreContentType(true);
return this;
}
public Http referrer(String ref) {
public Http referrer(String ref) {
connection.referrer(ref);
return this;
}
public Http referrer(URL ref) {
return referrer(ref.toExternalForm());
}
public Http userAgent(String ua) {
public Http userAgent(String ua) {
connection.userAgent(ua);
return this;
}
public Http retries(int tries) {
this.retries = tries;
return this;
}
public Http header(String name, String value) {
connection.header(name, value);
connection.header(name, value);
return this;
}
public Http cookies(Map<String,String> cookies) {
public Http cookies(Map<String, String> cookies) {
connection.cookies(cookies);
return this;
}
public Http data(Map<String,String> data) {
public Http data(Map<String, String> data) {
connection.data(data);
return this;
}
public Http data(String name, String value) {
Map<String,String> data = new HashMap<>();
Map<String, String> data = new HashMap<>();
data.put(name, value);
return data(data);
}
public Http method(Method method) {
connection.method(method);
return this;
@@ -155,6 +165,7 @@ public class Http {
public Connection connection() {
return connection;
}
public Document get() throws IOException {
connection.method(Method.GET);
return response().parse();
@@ -172,7 +183,7 @@ public class Http {
}
public Response response() throws IOException {
Response response = null;
Response response;
IOException lastException = null;
int retries = this.retries;
while (--retries >= 0) {
@@ -182,15 +193,15 @@ public class Http {
} catch (IOException e) {
// Warn users about possibly fixable permission error
if (e instanceof org.jsoup.HttpStatusException) {
HttpStatusException ex = (HttpStatusException)e;
HttpStatusException ex = (HttpStatusException) e;
// These status codes might indicate missing cookies
// 401 Unauthorized
// 403 Forbidden
int status = ex.getStatusCode();
int status = ex.getStatusCode();
if (status == 401 || status == 403) {
throw new IOException("Failed to load " + url + ": Status Code " + Integer.toString(status) + ". You might be able to circumvent this error by setting cookies for this domain" , e);
throw new IOException("Failed to load " + url + ": Status Code " + status + ". You might be able to circumvent this error by setting cookies for this domain", e);
}
}