From 98cc12d892988a3b327bfb3014390c6b47454f6f Mon Sep 17 00:00:00 2001 From: soloturn Date: Mon, 20 Dec 2021 02:16:16 +0100 Subject: [PATCH] cleanup utils/Http.java --- .../java/com/rarchives/ripme/utils/Http.java | 69 +++++++++++-------- 1 file changed, 40 insertions(+), 29 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/utils/Http.java b/src/main/java/com/rarchives/ripme/utils/Http.java index 47ee0f29..fe020041 100644 --- a/src/main/java/com/rarchives/ripme/utils/Http.java +++ b/src/main/java/com/rarchives/ripme/utils/Http.java @@ -1,28 +1,26 @@ package com.rarchives.ripme.utils; +import com.rarchives.ripme.ripper.AbstractRipper; +import org.apache.commons.lang.ArrayUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONObject; +import org.jsoup.Connection; +import org.jsoup.Connection.Method; +import org.jsoup.Connection.Response; +import org.jsoup.HttpStatusException; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; + import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.util.HashMap; import java.util.Map; -import org.apache.commons.lang.ArrayUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.json.JSONObject; -import org.jsoup.Connection; -import org.jsoup.Connection.Method; -import org.jsoup.Connection.Response; -import org.jsoup.Jsoup; -import org.jsoup.HttpStatusException; -import org.jsoup.nodes.Document; - -import com.rarchives.ripme.ripper.AbstractRipper; - /** * Wrapper around the Jsoup connection methods. - * + *

* Benefit is retry logic. */ public class Http { @@ -31,7 +29,7 @@ public class Http { private static final Logger logger = LogManager.getLogger(Http.class); private int retries; - private String url; + private final String url; private Connection connection; // Constructors @@ -39,6 +37,7 @@ public class Http { this.url = url; defaultSettings(); } + private Http(URL url) { this.url = url.toExternalForm(); defaultSettings(); @@ -47,6 +46,7 @@ public class Http { public static Http url(String url) { return new Http(url); } + public static Http url(URL url) { return new Http(url); } @@ -68,7 +68,7 @@ public class Http { private Map cookiesForURL(String u) { Map cookiesParsed = new HashMap<>(); - String cookieDomain = ""; + String cookieDomain = ""; try { URL parsed = new URL(u); String cookieStr = ""; @@ -84,7 +84,7 @@ public class Http { logger.info("Trying to load cookies from config for " + domain); cookieStr = Utils.getConfigString("cookies." + domain, ""); if (!cookieStr.equals("")) { - cookieDomain = domain; + cookieDomain = domain; // we found something, start parsing break; } @@ -110,42 +110,52 @@ public class Http { connection.timeout(timeout); return this; } + public Http ignoreContentType() { connection.ignoreContentType(true); return this; } - public Http referrer(String ref) { + + public Http referrer(String ref) { connection.referrer(ref); return this; } + public Http referrer(URL ref) { return referrer(ref.toExternalForm()); } - public Http userAgent(String ua) { + + public Http userAgent(String ua) { connection.userAgent(ua); return this; } + public Http retries(int tries) { this.retries = tries; return this; } + public Http header(String name, String value) { - connection.header(name, value); + connection.header(name, value); return this; } - public Http cookies(Map cookies) { + + public Http cookies(Map cookies) { connection.cookies(cookies); return this; } - public Http data(Map data) { + + public Http data(Map data) { connection.data(data); return this; } + public Http data(String name, String value) { - Map data = new HashMap<>(); + Map data = new HashMap<>(); data.put(name, value); return data(data); } + public Http method(Method method) { connection.method(method); return this; @@ -155,6 +165,7 @@ public class Http { public Connection connection() { return connection; } + public Document get() throws IOException { connection.method(Method.GET); return response().parse(); @@ -172,7 +183,7 @@ public class Http { } public Response response() throws IOException { - Response response = null; + Response response; IOException lastException = null; int retries = this.retries; while (--retries >= 0) { @@ -182,15 +193,15 @@ public class Http { } catch (IOException e) { // Warn users about possibly fixable permission error if (e instanceof org.jsoup.HttpStatusException) { - HttpStatusException ex = (HttpStatusException)e; - + HttpStatusException ex = (HttpStatusException) e; + // These status codes might indicate missing cookies // 401 Unauthorized // 403 Forbidden - int status = ex.getStatusCode(); + int status = ex.getStatusCode(); if (status == 401 || status == 403) { - throw new IOException("Failed to load " + url + ": Status Code " + Integer.toString(status) + ". You might be able to circumvent this error by setting cookies for this domain" , e); + throw new IOException("Failed to load " + url + ": Status Code " + status + ". You might be able to circumvent this error by setting cookies for this domain", e); } }