mirror/ripme
1
0
Fork 0
mirror of https://github.com/RipMeApp/ripme.git synced 2025-08-18 11:41:21 +02:00
Code Issues Releases Wiki Activity

cleanup utils/Http.java

Browse Source
This commit is contained in:
soloturn
2021-12-20 02:16:16 +01:00
parent 3bd9d1c412
commit 98cc12d892
1 changed files with 40 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
src/main/java/com/rarchives/ripme/utils/Http.java
View File

@@ -1,28 +1,26 @@
package com.rarchives.ripme.utils; package com.rarchives.ripme.utils;
import com.rarchives.ripme.ripper.AbstractRipper;
import org.apache.commons.lang.ArrayUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.json.JSONObject;
import org.jsoup.Connection;
import org.jsoup.Connection.Method;
import org.jsoup.Connection.Response;
import org.jsoup.HttpStatusException;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.json.JSONObject;
import org.jsoup.Connection;
import org.jsoup.Connection.Method;
import org.jsoup.Connection.Response;
import org.jsoup.Jsoup;
import org.jsoup.HttpStatusException;
import org.jsoup.nodes.Document;
import com.rarchives.ripme.ripper.AbstractRipper;
/** /**
* Wrapper around the Jsoup connection methods. * Wrapper around the Jsoup connection methods.
* * <p>
* Benefit is retry logic. * Benefit is retry logic.
*/ */
public class Http { public class Http {
@@ -31,7 +29,7 @@ public class Http {
private static final Logger logger = LogManager.getLogger(Http.class); private static final Logger logger = LogManager.getLogger(Http.class);
private int retries; private int retries;
private String url; private final String url;
private Connection connection; private Connection connection;
// Constructors // Constructors
@@ -39,6 +37,7 @@ public class Http {
this.url = url; this.url = url;
defaultSettings(); defaultSettings();
} }
private Http(URL url) { private Http(URL url) {
this.url = url.toExternalForm(); this.url = url.toExternalForm();
defaultSettings(); defaultSettings();
@@ -47,6 +46,7 @@ public class Http {
public static Http url(String url) { public static Http url(String url) {
return new Http(url); return new Http(url);
} }
public static Http url(URL url) { public static Http url(URL url) {
return new Http(url); return new Http(url);
} }
@@ -110,42 +110,52 @@ public class Http {
connection.timeout(timeout); connection.timeout(timeout);
return this; return this;
} }
public Http ignoreContentType() { public Http ignoreContentType() {
connection.ignoreContentType(true); connection.ignoreContentType(true);
return this; return this;
} }
public Http referrer(String ref) {
public Http referrer(String ref) {
connection.referrer(ref); connection.referrer(ref);
return this; return this;
} }
public Http referrer(URL ref) { public Http referrer(URL ref) {
return referrer(ref.toExternalForm()); return referrer(ref.toExternalForm());
} }
public Http userAgent(String ua) {
public Http userAgent(String ua) {
connection.userAgent(ua); connection.userAgent(ua);
return this; return this;
} }
public Http retries(int tries) { public Http retries(int tries) {
this.retries = tries; this.retries = tries;
return this; return this;
} }
public Http header(String name, String value) { public Http header(String name, String value) {
connection.header(name, value); connection.header(name, value);
return this; return this;
} }
public Http cookies(Map<String,String> cookies) {
public Http cookies(Map<String, String> cookies) {
connection.cookies(cookies); connection.cookies(cookies);
return this; return this;
} }
public Http data(Map<String,String> data) {
public Http data(Map<String, String> data) {
connection.data(data); connection.data(data);
return this; return this;
} }
public Http data(String name, String value) { public Http data(String name, String value) {
Map<String,String> data = new HashMap<>(); Map<String, String> data = new HashMap<>();
data.put(name, value); data.put(name, value);
return data(data); return data(data);
} }
public Http method(Method method) { public Http method(Method method) {
connection.method(method); connection.method(method);
return this; return this;
@@ -155,6 +165,7 @@ public class Http {
public Connection connection() { public Connection connection() {
return connection; return connection;
} }
public Document get() throws IOException { public Document get() throws IOException {
connection.method(Method.GET); connection.method(Method.GET);
return response().parse(); return response().parse();
@@ -172,7 +183,7 @@ public class Http {
} }
public Response response() throws IOException { public Response response() throws IOException {
Response response = null; Response response;
IOException lastException = null; IOException lastException = null;
int retries = this.retries; int retries = this.retries;
while (--retries >= 0) { while (--retries >= 0) {
@@ -182,15 +193,15 @@ public class Http {
} catch (IOException e) { } catch (IOException e) {
// Warn users about possibly fixable permission error // Warn users about possibly fixable permission error
if (e instanceof org.jsoup.HttpStatusException) { if (e instanceof org.jsoup.HttpStatusException) {
HttpStatusException ex = (HttpStatusException)e; HttpStatusException ex = (HttpStatusException) e;
// These status codes might indicate missing cookies // These status codes might indicate missing cookies
// 401 Unauthorized // 401 Unauthorized
// 403 Forbidden // 403 Forbidden
int status = ex.getStatusCode(); int status = ex.getStatusCode();
if (status == 401 || status == 403) { if (status == 401 || status == 403) {
throw new IOException("Failed to load " + url + ": Status Code " + Integer.toString(status) + ". You might be able to circumvent this error by setting cookies for this domain" , e); throw new IOException("Failed to load " + url + ": Status Code " + status + ". You might be able to circumvent this error by setting cookies for this domain", e);
} }
} }