From c77593570b16a11af26ccd3ce119bbfb26960a95 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 9 Jun 2018 15:05:21 -0400 Subject: [PATCH 1/4] Started work on getting DA login working again --- .../ripme/ripper/rippers/DeviantartRipper.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java index 99942adb..e23e7478 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java @@ -437,15 +437,19 @@ public class DeviantartRipper extends AbstractHTMLRipper { if (username == null || password == null) { throw new IOException("could not find username or password in config"); } - Response resp = Http.url("http://www.deviantart.com/") + Response resp = Http.url("http://www.deviantart.com/users/login") .response(); - for (Element input : resp.parse().select("form#form-login input[type=hidden]")) { - postData.put(input.attr("name"), input.attr("value")); - } +// for (Element input : resp.parse().select("form#form-login input[type=hidden]")) { +// postData.put(input.attr("name"), input.attr("value")); +// } + Document r = resp.parse(); + LOGGER.info("R: " + r.html()); postData.put("username", username); postData.put("password", password); - postData.put("remember_me", "1"); - + postData.put("validate_token", "829439d1f0b53e20f2f5"); + postData.put("validate_key", "1528530690"); +// postData.put("remember_me", "1"); + LOGGER.info(postData); // Send login request resp = Http.url("https://www.deviantart.com/users/login") .userAgent(USER_AGENT) @@ -453,6 +457,7 @@ public class DeviantartRipper extends AbstractHTMLRipper { .cookies(resp.cookies()) .method(Method.POST) .response(); +// LOGGER.info("RESP: " + resp.parse().html()); // Assert we are logged in if (resp.hasHeader("Location") && resp.header("Location").contains("password")) { From 2c5a9fdac37bb4ca9f4f8d82a445444cfc78207b Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 10 Jun 2018 12:12:29 -0400 Subject: [PATCH 2/4] Added getCookiesFromString func to turn a cookie string into a hashMap --- .../java/com/rarchives/ripme/utils/RipUtils.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java index 01d20e7c..84b17ca1 100644 --- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java +++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java @@ -3,9 +3,7 @@ package com.rarchives.ripme.utils; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -279,4 +277,16 @@ public class RipUtils { } return url; } + /** + * Reads a cookie string (Key1=value1;key2=value2) from the config file and turns it into a hashmap + * @return Map of cookies containing session data. + */ + public static Map getCookiesFromString(String line) { + Map cookies = new HashMap<>(); + for (String pair : line.split(";")) { + String[] kv = pair.split("="); + cookies.put(kv[0], kv[1]); + } + return cookies; + } } From 312e7e5c3316fd6e55e56a4705326ab33bf531bb Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 10 Jun 2018 12:12:51 -0400 Subject: [PATCH 3/4] DeviantartRipper now logs in using cookies --- .../ripper/rippers/DeviantartRipper.java | 81 +++++-------------- 1 file changed, 20 insertions(+), 61 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java index e23e7478..cb1c9614 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java @@ -3,6 +3,7 @@ package com.rarchives.ripme.ripper.rippers; import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.utils.Base64; import com.rarchives.ripme.utils.Http; +import com.rarchives.ripme.utils.RipUtils; import com.rarchives.ripme.utils.Utils; import java.io.IOException; import java.net.HttpURLConnection; @@ -37,6 +38,10 @@ public class DeviantartRipper extends AbstractHTMLRipper { super(url); } + String loginCookies = "auth=__0f9158aaec09f417b235%3B%221ff79836392a515d154216d919eae573%22;" + + "auth_secure=__41d14dd0da101f411bb0%3B%2281cf2cf9477776162a1172543aae85ce%22;" + + "userinfo=__bf84ac233bfa8ae642e8%3B%7B%22username%22%3A%22grabpy%22%2C%22uniqueid%22%3A%22a0a876aa37dbd4b30e1c80406ee9c280%22%2C%22vd%22%3A%22BbHUXZ%2CBbHUXZ%2CA%2CU%2CA%2C%2CB%2CA%2CB%2CBbHUXZ%2CBbHUdj%2CL%2CL%2CA%2CBbHUdj%2C13%2CA%2CB%2CA%2C%2CA%2CA%2CB%2CA%2CA%2C%2CA%22%2C%22attr%22%3A56%7D"; + @Override public String getHost() { return "deviantart"; @@ -117,24 +122,20 @@ public class DeviantartRipper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { - //Test to see if there is a login: - String username = Utils.getConfigString("deviantart.username", new String(Base64.decode("Z3JhYnB5"))); - String password = Utils.getConfigString("deviantart.password", new String(Base64.decode("ZmFrZXJz"))); - - if (username == null || password == null) { - LOGGER.debug("No DeviantArt login provided."); - cookies.put("agegate_state","1"); // Bypasses the age gate - } else { - // Attempt Login - try { - cookies = loginToDeviantart(); - } catch (IOException e) { - LOGGER.warn("Failed to login: ", e); + // Base64 da login + // username: Z3JhYnB5 + // password: ZmFrZXJz + + + cookies = getDACookies(); + if (cookies.isEmpty()) { + LOGGER.warn("Failed to get login cookies"); cookies.put("agegate_state","1"); // Bypasses the age gate } - } - - + + LOGGER.info(Http.url(this.url) + .cookies(cookies) + .get()); return Http.url(this.url) .cookies(cookies) .get(); @@ -426,52 +427,10 @@ public class DeviantartRipper extends AbstractHTMLRipper { } /** - * Logs into deviant art. Required to rip full-size NSFW content. + * Returns DA cookies. * @return Map of cookies containing session data. */ - private Map loginToDeviantart() throws IOException { - // Populate postData fields - Map postData = new HashMap<>(); - String username = Utils.getConfigString("deviantart.username", new String(Base64.decode("Z3JhYnB5"))); - String password = Utils.getConfigString("deviantart.password", new String(Base64.decode("ZmFrZXJz"))); - if (username == null || password == null) { - throw new IOException("could not find username or password in config"); - } - Response resp = Http.url("http://www.deviantart.com/users/login") - .response(); -// for (Element input : resp.parse().select("form#form-login input[type=hidden]")) { -// postData.put(input.attr("name"), input.attr("value")); -// } - Document r = resp.parse(); - LOGGER.info("R: " + r.html()); - postData.put("username", username); - postData.put("password", password); - postData.put("validate_token", "829439d1f0b53e20f2f5"); - postData.put("validate_key", "1528530690"); -// postData.put("remember_me", "1"); - LOGGER.info(postData); - // Send login request - resp = Http.url("https://www.deviantart.com/users/login") - .userAgent(USER_AGENT) - .data(postData) - .cookies(resp.cookies()) - .method(Method.POST) - .response(); -// LOGGER.info("RESP: " + resp.parse().html()); - - // Assert we are logged in - if (resp.hasHeader("Location") && resp.header("Location").contains("password")) { - // Wrong password - throw new IOException("Wrong password"); - } - if (resp.url().toExternalForm().contains("bad_form")) { - throw new IOException("Login form was incorrectly submitted"); - } - if (resp.cookie("auth_secure") == null || - resp.cookie("auth") == null) { - throw new IOException("No auth_secure or auth cookies received"); - } - // We are logged in, save the cookies - return resp.cookies(); + private Map getDACookies() { + return RipUtils.getCookiesFromString(Utils.getConfigString("deviantart.cookies", loginCookies)); } } \ No newline at end of file From 79aaabf58023d20191bf7aafb0787347212fdb1a Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 15 Jun 2018 15:13:13 -0400 Subject: [PATCH 4/4] Removed unneeded logging line --- .../com/rarchives/ripme/ripper/rippers/DeviantartRipper.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java index cb1c9614..2afae2dc 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java @@ -132,10 +132,7 @@ public class DeviantartRipper extends AbstractHTMLRipper { LOGGER.warn("Failed to get login cookies"); cookies.put("agegate_state","1"); // Bypasses the age gate } - - LOGGER.info(Http.url(this.url) - .cookies(cookies) - .get()); + return Http.url(this.url) .cookies(cookies) .get();