From 34ee3a6f60d3d274cc19dedeba4ceaa3495b4bc8 Mon Sep 17 00:00:00 2001 From: Random Committer Date: Fri, 20 Apr 2018 07:51:40 -0700 Subject: [PATCH 1/4] Removed Socks Proxy setup from App.java, added new class Proxy under com.rarchives.ripme.utils which allows to set both HTTP Proxy and SOCKS Proxy --- src/main/java/com/rarchives/ripme/App.java | 40 +++----- .../java/com/rarchives/ripme/utils/Proxy.java | 99 +++++++++++++++++++ 2 files changed, 112 insertions(+), 27 deletions(-) create mode 100644 src/main/java/com/rarchives/ripme/utils/Proxy.java diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index 508401d2..f9357e62 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -7,9 +7,7 @@ import java.io.BufferedReader; import java.io.FileReader; import java.io.FileNotFoundException; -import java.net.Authenticator; import java.net.MalformedURLException; -import java.net.PasswordAuthentication; import java.net.URL; import java.util.Arrays; import java.util.List; @@ -29,6 +27,7 @@ import com.rarchives.ripme.ui.History; import com.rarchives.ripme.ui.HistoryEntry; import com.rarchives.ripme.ui.MainWindow; import com.rarchives.ripme.ui.UpdateUtils; +import com.rarchives.ripme.utils.Proxy; import com.rarchives.ripme.utils.RipUtils; import com.rarchives.ripme.utils.Utils; @@ -69,7 +68,7 @@ public class App { /** * Creates an abstract ripper and instructs it to rip. * @param url URL to be ripped - * @throws Exception + * @throws Exception */ private static void rip(URL url) throws Exception { AbstractRipper ripper = AbstractRipper.getRipper(url); @@ -96,29 +95,15 @@ public class App { if (cl.hasOption('w')) { Utils.setConfigBoolean("file.overwrite", true); } - + if (cl.hasOption('s')) { String sservfull = cl.getOptionValue('s').trim(); - if (sservfull.lastIndexOf("@") != -1) { - int sservli = sservfull.lastIndexOf("@"); - String userpw = sservfull.substring(0, sservli); - String[] usersplit = userpw.split(":"); - String user = usersplit[0]; - String password = usersplit[1]; - Authenticator.setDefault(new Authenticator(){ - protected PasswordAuthentication getPasswordAuthentication(){ - PasswordAuthentication p = new PasswordAuthentication(user, password.toCharArray()); - return p; - } - }); - - sservfull = sservfull.substring(sservli + 1); - } - String[] servsplit = sservfull.split(":"); - if (servsplit.length == 2) { - System.setProperty("socksProxyPort", servsplit[1]); - } - System.setProperty("socksProxyHost", servsplit[0]); + Proxy.setSocks(sservfull); + } + + if (cl.hasOption('p')) { + String proxyserverfull = cl.getOptionValue('p').trim(); + Proxy.setHTTPProxy(proxyserverfull); } if (cl.hasOption('t')) { @@ -221,7 +206,7 @@ public class App { String url = cl.getOptionValue('u').trim(); ripURL(url, cl.hasOption("n")); } - + } /** @@ -269,7 +254,8 @@ public class App { opts.addOption("n", "no-prop-file", false, "Do not create properties file."); opts.addOption("f", "urls-file", true, "Rip URLs from a file."); opts.addOption("v", "version", false, "Show current version"); - opts.addOption("s", "socks-server", true, "Use socks server ([user:password]@host[:port]"); + opts.addOption("s", "socks-server", true, "Use socks server ([user:password]@host[:port])"); + opts.addOption("p", "proxy-server", true, "Use HTTP Proxy server ([user:password]@host[:port])"); return opts; } @@ -288,7 +274,7 @@ public class App { return null; } } - + /** * Loads history from history file into memory. */ diff --git a/src/main/java/com/rarchives/ripme/utils/Proxy.java b/src/main/java/com/rarchives/ripme/utils/Proxy.java new file mode 100644 index 00000000..2f657b88 --- /dev/null +++ b/src/main/java/com/rarchives/ripme/utils/Proxy.java @@ -0,0 +1,99 @@ +package com.rarchives.ripme.utils; + +import java.net.Authenticator; +import java.net.PasswordAuthentication; +import java.util.Map; +import java.util.HashMap; + +/** + * Proxy/Socks setter + */ +public class Proxy { + private Proxy() { + } + + /** + * Parse the proxy server settings from string, using the format + * [user:password]@host[:port]. + * + * @param fullproxy the string to parse + * @return HashMap containing proxy server, port, user and password + */ + private static Map parseServer(String fullproxy) { + Map proxy = new HashMap(); + + if (fullproxy.lastIndexOf("@") != -1) { + int sservli = fullproxy.lastIndexOf("@"); + String userpw = fullproxy.substring(0, sservli); + String[] usersplit = userpw.split(":"); + proxy.put("user", usersplit[0]); + proxy.put("password", usersplit[1]); + fullproxy = fullproxy.substring(sservli + 1); + } + String[] servsplit = fullproxy.split(":"); + if (servsplit.length == 2) { + proxy.put("port", servsplit[1]); + } + proxy.put("server", servsplit[0]); + return proxy; + } + + /** + * Set a HTTP Proxy. + * WARNING: Authenticated HTTP Proxy won't work from jdk1.8.111 unless + * passing the flag -Djdk.http.auth.tunneling.disabledSchemes="" to java + * see https://stackoverflow.com/q/41505219 + * + * @param fullproxy the proxy, using format [user:password]@host[:port] + */ + public static void setHTTPProxy(String fullproxy) { + Map proxyServer = parseServer(fullproxy); + + if (proxyServer.get("user") != null && proxyServer.get("password") != null) { + Authenticator.setDefault(new Authenticator(){ + protected PasswordAuthentication getPasswordAuthentication(){ + PasswordAuthentication p = new PasswordAuthentication(proxyServer.get("user"), proxyServer.get("password").toCharArray()); + return p; + } + }); + System.setProperty("http.proxyUser", proxyServer.get("user")); + System.setProperty("http.proxyPassword", proxyServer.get("password")); + System.setProperty("https.proxyUser", proxyServer.get("user")); + System.setProperty("https.proxyPassword", proxyServer.get("password")); + } + + if (proxyServer.get("port") != null) { + System.setProperty("http.proxyPort", proxyServer.get("port")); + System.setProperty("https.proxyPort", proxyServer.get("port")); + } + + System.setProperty("http.proxyHost", proxyServer.get("server")); + System.setProperty("https.proxyHost", proxyServer.get("server")); + } + + /** + * Set a Socks Proxy Server (globally). + * + * @param fullsocks the socks server, using format [user:password]@host[:port] + */ + public static void setSocks(String fullsocks) { + + Map socksServer = parseServer(fullsocks); + if (socksServer.get("user") != null && socksServer.get("password") != null) { + Authenticator.setDefault(new Authenticator(){ + protected PasswordAuthentication getPasswordAuthentication(){ + PasswordAuthentication p = new PasswordAuthentication(socksServer.get("user"), socksServer.get("password").toCharArray()); + return p; + } + }); + System.setProperty("java.net.socks.username", socksServer.get("user")); + System.setProperty("java.net.socks.password", socksServer.get("password")); + } + if (socksServer.get("port") != null) { + System.setProperty("socksProxyPort", socksServer.get("port")); + } + + System.setProperty("socksProxyHost", socksServer.get("server")); + } + +} \ No newline at end of file From f493b1ed80384349511ba31f7099e4576b0a92d3 Mon Sep 17 00:00:00 2001 From: Random Committer Date: Fri, 20 Apr 2018 08:03:45 -0700 Subject: [PATCH 2/4] Added test for class Proxy --- .../com/rarchives/ripme/tst/proxyTest.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/proxyTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/proxyTest.java b/src/test/java/com/rarchives/ripme/tst/proxyTest.java new file mode 100644 index 00000000..32b7958c --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/proxyTest.java @@ -0,0 +1,42 @@ +package com.rarchives.ripme.tst; + +import java.io.IOException; +import java.net.URL; +import com.rarchives.ripme.utils.Proxy; +import com.rarchives.ripme.utils.Utils; +import junit.framework.TestCase; +import com.rarchives.ripme.utils.Http; + + +public class proxyTest extends TestCase { + // This test will only run on machines where the user has added a entry for socks.server + public void testSocksProxy() throws IOException { + URL url = new URL("https://icanhazip.com"); + String proxyConfig = Utils.getConfigString("socks.server", ""); + if (!proxyConfig.equals("")) { + String ip1 = Http.url(url).ignoreContentType().get().text(); + Proxy.setSocks(Utils.getConfigString("socks.server", "")); + String ip2 = Http.url(url).ignoreContentType().get().text(); + assertFalse(ip1.equals(ip2)); + } else { + System.out.println("Skipping testSocksProxy"); + assert(true); + } + } + + // This test will only run on machines where the user has added a entry for proxy.server + public void testHTTPProxy() throws IOException { + URL url = new URL("https://icanhazip.com"); + String proxyConfig = Utils.getConfigString("proxy.server", ""); + if (!proxyConfig.equals("")) { + String ip1 = Http.url(url).ignoreContentType().get().text(); + Proxy.setHTTPProxy(Utils.getConfigString("proxy.server", "")); + String ip2 = Http.url(url).ignoreContentType().get().text(); + assertFalse(ip1.equals(ip2)); + } else { + System.out.println("Skipping testHTTPProxy"); + assert(true); + } + } + +} From 95d1e0a12046ff9ca1673a8df5ad5508630e2270 Mon Sep 17 00:00:00 2001 From: Random Committer Date: Fri, 20 Apr 2018 09:15:39 -0700 Subject: [PATCH 3/4] Changed socks and proxy config values to proxy.socks and proxy.http; Checking config on start for these values and set proxy/socks accordingly. --- src/main/java/com/rarchives/ripme/App.java | 6 ++++++ src/main/java/com/rarchives/ripme/utils/Proxy.java | 2 +- src/test/java/com/rarchives/ripme/tst/proxyTest.java | 12 ++++++------ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/App.java b/src/main/java/com/rarchives/ripme/App.java index f9357e62..6f650d56 100644 --- a/src/main/java/com/rarchives/ripme/App.java +++ b/src/main/java/com/rarchives/ripme/App.java @@ -48,6 +48,12 @@ public class App { System.exit(0); } + if (Utils.getConfigString("proxy.http", null) != null) { + Proxy.setHTTPProxy(Utils.getConfigString("proxy.http", null)); + } else if (Utils.getConfigString("proxy.socks", null) != null) { + Proxy.setSocks(Utils.getConfigString("proxy.socks", null)); + } + if (GraphicsEnvironment.isHeadless() || args.length > 0) { handleArguments(args); } else { diff --git a/src/main/java/com/rarchives/ripme/utils/Proxy.java b/src/main/java/com/rarchives/ripme/utils/Proxy.java index 2f657b88..be3c3b7e 100644 --- a/src/main/java/com/rarchives/ripme/utils/Proxy.java +++ b/src/main/java/com/rarchives/ripme/utils/Proxy.java @@ -96,4 +96,4 @@ public class Proxy { System.setProperty("socksProxyHost", socksServer.get("server")); } -} \ No newline at end of file +} diff --git a/src/test/java/com/rarchives/ripme/tst/proxyTest.java b/src/test/java/com/rarchives/ripme/tst/proxyTest.java index 32b7958c..1d702c6b 100644 --- a/src/test/java/com/rarchives/ripme/tst/proxyTest.java +++ b/src/test/java/com/rarchives/ripme/tst/proxyTest.java @@ -9,13 +9,13 @@ import com.rarchives.ripme.utils.Http; public class proxyTest extends TestCase { - // This test will only run on machines where the user has added a entry for socks.server + // This test will only run on machines where the user has added a entry for proxy.socks public void testSocksProxy() throws IOException { URL url = new URL("https://icanhazip.com"); - String proxyConfig = Utils.getConfigString("socks.server", ""); + String proxyConfig = Utils.getConfigString("proxy.socks", ""); if (!proxyConfig.equals("")) { String ip1 = Http.url(url).ignoreContentType().get().text(); - Proxy.setSocks(Utils.getConfigString("socks.server", "")); + Proxy.setSocks(Utils.getConfigString("proxy.socks", "")); String ip2 = Http.url(url).ignoreContentType().get().text(); assertFalse(ip1.equals(ip2)); } else { @@ -24,13 +24,13 @@ public class proxyTest extends TestCase { } } - // This test will only run on machines where the user has added a entry for proxy.server + // This test will only run on machines where the user has added a entry for proxy.http public void testHTTPProxy() throws IOException { URL url = new URL("https://icanhazip.com"); - String proxyConfig = Utils.getConfigString("proxy.server", ""); + String proxyConfig = Utils.getConfigString("proxy.http", ""); if (!proxyConfig.equals("")) { String ip1 = Http.url(url).ignoreContentType().get().text(); - Proxy.setHTTPProxy(Utils.getConfigString("proxy.server", "")); + Proxy.setHTTPProxy(Utils.getConfigString("proxy.http", "")); String ip2 = Http.url(url).ignoreContentType().get().text(); assertFalse(ip1.equals(ip2)); } else { From 34699bf8c66296f8eabb03e9658646ceac19c88b Mon Sep 17 00:00:00 2001 From: Random Committer Date: Fri, 20 Apr 2018 09:53:49 -0700 Subject: [PATCH 4/4] proxyTest.java: unset proxy/socks before testing it --- src/test/java/com/rarchives/ripme/tst/proxyTest.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/proxyTest.java b/src/test/java/com/rarchives/ripme/tst/proxyTest.java index 1d702c6b..36ea2f55 100644 --- a/src/test/java/com/rarchives/ripme/tst/proxyTest.java +++ b/src/test/java/com/rarchives/ripme/tst/proxyTest.java @@ -9,8 +9,14 @@ import com.rarchives.ripme.utils.Http; public class proxyTest extends TestCase { + + // This test will only run on machines where the user has added a entry for proxy.socks public void testSocksProxy() throws IOException { + // Unset proxy before testing + System.setProperty("http.proxyHost", ""); + System.setProperty("https.proxyHost", ""); + System.setProperty("socksProxyHost", ""); URL url = new URL("https://icanhazip.com"); String proxyConfig = Utils.getConfigString("proxy.socks", ""); if (!proxyConfig.equals("")) { @@ -26,6 +32,10 @@ public class proxyTest extends TestCase { // This test will only run on machines where the user has added a entry for proxy.http public void testHTTPProxy() throws IOException { + // Unset proxy before testing + System.setProperty("http.proxyHost", ""); + System.setProperty("https.proxyHost", ""); + System.setProperty("socksProxyHost", ""); URL url = new URL("https://icanhazip.com"); String proxyConfig = Utils.getConfigString("proxy.http", ""); if (!proxyConfig.equals("")) {