mirror of
https://github.com/RipMeApp/ripme.git
synced 2025-08-26 15:24:51 +02:00
Merge pull request #1216 from MrPlaygon/master
Deviantart Ripper changes
This commit is contained in:
cyian-1756
GitHub
@@ -1,54 +1,118 @@
|
||||
package com.rarchives.ripme.ripper.rippers;
|
||||
|
||||
import com.rarchives.ripme.ripper.AbstractJSONRipper;
|
||||
import com.rarchives.ripme.utils.Base64;
|
||||
import com.rarchives.ripme.ripper.AbstractHTMLRipper;
|
||||
import com.rarchives.ripme.ripper.DownloadThreadPool;
|
||||
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
|
||||
import com.rarchives.ripme.utils.Http;
|
||||
import com.rarchives.ripme.utils.RipUtils;
|
||||
import com.rarchives.ripme.utils.Utils;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.io.ObjectInputStream;
|
||||
import java.io.ObjectOutputStream;
|
||||
import java.io.Serializable;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Base64;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.Iterator;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
import org.jsoup.Connection;
|
||||
import org.jsoup.Connection.Method;
|
||||
import org.jsoup.Connection.Response;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.nodes.Element;
|
||||
import org.jsoup.select.Elements;
|
||||
|
||||
/**
|
||||
*
|
||||
* @author MrPlaygon
|
||||
*
|
||||
* NOT using Deviantart API like the old JSON ripper because it is SLOW
|
||||
* and somehow annoying to use. Things to consider: Using the API might
|
||||
* be less work/maintenance later because APIs do not change as
|
||||
* frequently as HTML source code does...?
|
||||
*
|
||||
*
|
||||
*
|
||||
* Tested for:
|
||||
*
|
||||
* SFW:
|
||||
*
|
||||
* https://www.deviantart.com/apofiss/gallery/41388863/sceneries
|
||||
* https://www.deviantart.com/kageuri/gallery/
|
||||
* https://www.deviantart.com/kageuri/gallery/?catpath=/
|
||||
* https://www.deviantart.com/apofiss/favourites/39881418/gifts-and
|
||||
* https://www.deviantart.com/kageuri/favourites/
|
||||
* https://www.deviantart.com/kageuri/favourites/?catpath=/
|
||||
*
|
||||
* NSFW:
|
||||
*
|
||||
* https://www.deviantart.com/revpeng/gallery/67734353/Siren-Lee-Agent-of-S-I-R-E-N-S
|
||||
*
|
||||
*
|
||||
* Deactivated account:
|
||||
*
|
||||
* https://www.deviantart.com/gingerbreadpony
|
||||
*
|
||||
* Banned Account:
|
||||
*
|
||||
* https://www.deviantart.com/ghostofflossenburg
|
||||
*
|
||||
*
|
||||
*
|
||||
*
|
||||
* Login Data (PLEASE DONT ACTUALLY USE!!!):
|
||||
*
|
||||
* email: 5g5_8l4dii5lbbpc@byom.de
|
||||
*
|
||||
* username: 5g58l4dii5lbbpc
|
||||
*
|
||||
* password: 5g5_8l4dii5lbbpc
|
||||
*
|
||||
*
|
||||
*
|
||||
*/
|
||||
public class DeviantartRipper extends AbstractHTMLRipper {
|
||||
|
||||
public class DeviantartRipper extends AbstractJSONRipper {
|
||||
String requestID;
|
||||
String galleryID;
|
||||
String username;
|
||||
String baseApiUrl = "https://www.deviantart.com/dapi/v1/gallery/";
|
||||
String csrf;
|
||||
Map<String, String> pageCookies = new HashMap<>();
|
||||
private final String username = "5g58l4dii5lbbpc";
|
||||
private final String password = "5g5_8l4dii5lbbpc";
|
||||
private int offset = 0;
|
||||
private boolean usingCatPath = false;
|
||||
private int downloadCount = 0;
|
||||
private Map<String, String> cookies = new HashMap<String, String>();
|
||||
private DownloadThreadPool deviantartThreadPool = new DownloadThreadPool("deviantart");
|
||||
private ArrayList<String> names = new ArrayList<String>();
|
||||
|
||||
private static final int PAGE_SLEEP_TIME = 3000,
|
||||
IMAGE_SLEEP_TIME = 2000;
|
||||
List<String> allowedCookies = Arrays.asList("agegate_state", "userinfo", "auth", "auth_secure");
|
||||
|
||||
private Map<String,String> cookies = new HashMap<>();
|
||||
private Set<String> triedURLs = new HashSet<>();
|
||||
private Connection conn = null;
|
||||
|
||||
// Constants
|
||||
private final String referer = "https://www.deviantart.com/";
|
||||
private final String userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0";
|
||||
private final String utilsKey = "DeviantartLogin.cookies";
|
||||
|
||||
@Override
|
||||
public DownloadThreadPool getThreadPool() {
|
||||
return deviantartThreadPool;
|
||||
}
|
||||
|
||||
public DeviantartRipper(URL url) throws IOException {
|
||||
super(url);
|
||||
}
|
||||
|
||||
String loginCookies = "auth=__0f9158aaec09f417b235%3B%221ff79836392a515d154216d919eae573%22;" +
|
||||
"auth_secure=__41d14dd0da101f411bb0%3B%2281cf2cf9477776162a1172543aae85ce%22;" +
|
||||
"userinfo=__bf84ac233bfa8ae642e8%3B%7B%22username%22%3A%22grabpy%22%2C%22uniqueid%22%3A%22a0a876aa37dbd4b30e1c80406ee9c280%22%2C%22vd%22%3A%22BbHUXZ%2CBbHUXZ%2CA%2CU%2CA%2C%2CB%2CA%2CB%2CBbHUXZ%2CBbHUdj%2CL%2CL%2CA%2CBbHUdj%2C13%2CA%2CB%2CA%2C%2CA%2CA%2CB%2CA%2CA%2C%2CA%22%2C%22attr%22%3A56%7D";
|
||||
@Override
|
||||
protected String getDomain() {
|
||||
return "deviantart.com";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getHost() {
|
||||
@@ -56,354 +120,526 @@ public class DeviantartRipper extends AbstractJSONRipper {
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDomain() {
|
||||
return "deviantart.com";
|
||||
protected Document getFirstPage() throws IOException {
|
||||
if (isDeactivated()) {
|
||||
throw new IOException("Account Deactivated");
|
||||
}
|
||||
login();
|
||||
|
||||
@Override
|
||||
public URL sanitizeURL(URL url) throws MalformedURLException {
|
||||
String u = url.toExternalForm();
|
||||
if (u.contains("/gallery/")) {
|
||||
return url;
|
||||
} else if (u.contains("/favourites")) {
|
||||
return url;
|
||||
} else if (u.contains("/favorites")) {
|
||||
return url;
|
||||
}
|
||||
// Saving connection to reuse later for following pages.
|
||||
this.conn = Http.url(urlWithParams(this.offset)).cookies(getDACookie()).referrer(this.referer)
|
||||
.userAgent(this.userAgent).connection();
|
||||
|
||||
if (!u.endsWith("/gallery/") && !u.endsWith("/gallery")) {
|
||||
if (!u.endsWith("/")) {
|
||||
u += "/gallery/";
|
||||
} else {
|
||||
u += "gallery/";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Pattern p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)/favou?rites/([0-9]+)/*?$");
|
||||
Matcher m = p.matcher(url.toExternalForm());
|
||||
if (!m.matches()) {
|
||||
String subdir = "/";
|
||||
if (u.contains("catpath=scraps")) {
|
||||
subdir = "scraps";
|
||||
}
|
||||
u = u.replaceAll("\\?.*", "?catpath=" + subdir);
|
||||
}
|
||||
return new URL(u);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getGID(URL url) throws MalformedURLException {
|
||||
Pattern p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)(/gallery)?/?(\\?.*)?$");
|
||||
Matcher m = p.matcher(url.toExternalForm());
|
||||
if (m.matches()) {
|
||||
// Root gallery
|
||||
if (url.toExternalForm().contains("catpath=scraps")) {
|
||||
return m.group(1) + "_scraps";
|
||||
}
|
||||
else {
|
||||
return m.group(1);
|
||||
}
|
||||
}
|
||||
p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)/gallery/([0-9]+).*$");
|
||||
m = p.matcher(url.toExternalForm());
|
||||
if (m.matches()) {
|
||||
// Subgallery
|
||||
return m.group(1) + "_" + m.group(2);
|
||||
}
|
||||
p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)/favou?rites/([0-9]+)/.*?$");
|
||||
m = p.matcher(url.toExternalForm());
|
||||
if (m.matches()) {
|
||||
return m.group(1) + "_faves_" + m.group(2);
|
||||
}
|
||||
p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)/favou?rites/?$");
|
||||
m = p.matcher(url.toExternalForm());
|
||||
if (m.matches()) {
|
||||
// Subgallery
|
||||
return m.group(1) + "_faves";
|
||||
}
|
||||
throw new MalformedURLException("Expected URL format: http://www.deviantart.com/username[/gallery/#####], got: " + url);
|
||||
}
|
||||
|
||||
private String getUsernameFromURL(String u) {
|
||||
Pattern p = Pattern.compile("^https?://www\\.deviantart\\.com/([a-zA-Z0-9\\-]+)/gallery/?(\\S+)?");
|
||||
Matcher m = p.matcher(url.toExternalForm());
|
||||
if (m.matches()) {
|
||||
return m.group(1);
|
||||
}
|
||||
return null;
|
||||
|
||||
}
|
||||
|
||||
private String getFullsizedNSFWImage(String pageURL) {
|
||||
try {
|
||||
Document doc = Http.url(pageURL).cookies(cookies).get();
|
||||
String imageToReturn = "";
|
||||
String[] d = doc.select("img").attr("srcset").split(",");
|
||||
|
||||
String s = d[d.length -1].split(" ")[0];
|
||||
LOGGER.info("2:" + s);
|
||||
|
||||
if (s == null || s.equals("")) {
|
||||
LOGGER.error("Could not find full sized image at " + pageURL);
|
||||
}
|
||||
return s;
|
||||
} catch (IOException e) {
|
||||
LOGGER.error("Could not find full sized image at " + pageURL);
|
||||
return null;
|
||||
}
|
||||
return this.conn.get();
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets first page.
|
||||
* Will determine if login is supplied,
|
||||
* if there is a login, then login and add that login cookies.
|
||||
* Otherwise, just bypass the age gate with an anonymous flag.
|
||||
* @return
|
||||
* @throws IOException
|
||||
* Checks if the URL refers to a deactivated account using the HTTP status Codes
|
||||
*
|
||||
* @return true when the account is good
|
||||
* @throws IOException when the account is deactivated
|
||||
*/
|
||||
private boolean isDeactivated() throws IOException {
|
||||
Response res = Http.url(this.url).connection().followRedirects(true).referrer(this.referer)
|
||||
.userAgent(this.userAgent).execute();
|
||||
return res.statusCode() != 200 ? true : false;
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Stores logged in Cookies. Needed for art pieces only visible to logged in
|
||||
* users.
|
||||
*
|
||||
*
|
||||
* @throws IOException when failed to load webpage or failed to read/write
|
||||
* cookies in file (used when running multiple instances of
|
||||
* RipMe)
|
||||
*/
|
||||
private void login() throws IOException {
|
||||
|
||||
String customUsername = Utils.getConfigString("DeviantartCustomLoginUsername", this.username);
|
||||
String customPassword = Utils.getConfigString("DeviantartCustomLoginPassword", this.password);
|
||||
try {
|
||||
String dACookies = Utils.getConfigString(utilsKey, null);
|
||||
updateCookie(dACookies != null ? deserialize(dACookies) : null);
|
||||
} catch (ClassNotFoundException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
if (getDACookie() == null || !checkLogin()) {
|
||||
LOGGER.info("Do Login now");
|
||||
// Do login now
|
||||
|
||||
// Load login page
|
||||
Response res = Http.url("https://www.deviantart.com/users/login").connection().method(Method.GET)
|
||||
.referrer(referer).userAgent(userAgent).execute();
|
||||
|
||||
updateCookie(res.cookies());
|
||||
|
||||
// Find tokens
|
||||
Document doc = res.parse();
|
||||
Element form = doc.getElementById("login");
|
||||
String token = form.select("input[name=\"validate_token\"]").first().attr("value");
|
||||
String key = form.select("input[name=\"validate_key\"]").first().attr("value");
|
||||
LOGGER.info("Token: " + token + " & Key: " + key);
|
||||
|
||||
// Build Login Data
|
||||
HashMap<String, String> loginData = new HashMap<String, String>();
|
||||
loginData.put("challenge", "");
|
||||
loginData.put("username", customUsername);
|
||||
loginData.put("password", customPassword);
|
||||
loginData.put("remember_me", "1");
|
||||
loginData.put("validate_token", token);
|
||||
loginData.put("validate_key", key);
|
||||
Map<String, String> cookies = res.cookies();
|
||||
|
||||
// Log in using data. Handle redirect
|
||||
res = Http.url("https://www.deviantart.com/users/login").connection().referrer(referer).userAgent(userAgent)
|
||||
.method(Method.POST).data(loginData).cookies(cookies).followRedirects(false).execute();
|
||||
updateCookie(res.cookies());
|
||||
|
||||
res = Http.url(res.header("location")).connection().referrer(referer).userAgent(userAgent)
|
||||
.method(Method.GET).cookies(cookies).followRedirects(false).execute();
|
||||
|
||||
// Store cookies
|
||||
updateCookie(res.cookies());
|
||||
|
||||
// Write Cookie to file for other RipMe Instances or later use
|
||||
Utils.setConfigString(utilsKey, serialize(new HashMap<String, String>(getDACookie())));
|
||||
Utils.saveConfig(); // save now because of other instances that might work simultaneously
|
||||
|
||||
} else {
|
||||
LOGGER.info("No new Login needed");
|
||||
}
|
||||
|
||||
LOGGER.info("DA Cookies: " + getDACookie());
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns next page Document using offset.
|
||||
*/
|
||||
@Override
|
||||
public JSONObject getFirstPage() throws IOException {
|
||||
public Document getNextPage(Document doc) throws IOException {
|
||||
this.offset += 24;
|
||||
this.conn.url(urlWithParams(this.offset)).cookies(getDACookie());
|
||||
Response re = this.conn.execute();
|
||||
// Response re = Http.url(urlWithParams(this.offset)).cookies(getDACookie()).referrer(referer).userAgent(userAgent)
|
||||
// .response();
|
||||
updateCookie(re.cookies());
|
||||
Document docu = re.parse();
|
||||
Elements messages = docu.getElementsByClass("message");
|
||||
LOGGER.info("Current Offset: " + this.offset);
|
||||
|
||||
// Base64 da login
|
||||
// username: Z3JhYnB5
|
||||
// password: ZmFrZXJz
|
||||
|
||||
|
||||
cookies = getDACookies();
|
||||
if (cookies.isEmpty()) {
|
||||
LOGGER.warn("Failed to get login cookies");
|
||||
cookies.put("agegate_state","1"); // Bypasses the age gate
|
||||
}
|
||||
cookies.put("agegate_state", "1");
|
||||
|
||||
Response res = Http.url(this.url)
|
||||
.cookies(cookies)
|
||||
.response();
|
||||
Document page = res.parse();
|
||||
|
||||
JSONObject firstPageJSON = getFirstPageJSON(page);
|
||||
requestID = firstPageJSON.getJSONObject("dapx").getString("requestid");
|
||||
galleryID = getGalleryID(page);
|
||||
username = getUsernameFromURL(url.toExternalForm());
|
||||
csrf = firstPageJSON.getString("csrf");
|
||||
pageCookies = res.cookies();
|
||||
|
||||
return requestPage(0, galleryID, username, requestID, csrf, pageCookies);
|
||||
}
|
||||
|
||||
private JSONObject requestPage(int offset, String galleryID, String username, String requestID, String csfr, Map<String, String> c) {
|
||||
LOGGER.debug("offset: " + Integer.toString(offset));
|
||||
LOGGER.debug("galleryID: " + galleryID);
|
||||
LOGGER.debug("username: " + username);
|
||||
LOGGER.debug("requestID: " + requestID);
|
||||
String url = baseApiUrl + galleryID + "?iid=" + requestID;
|
||||
try {
|
||||
Document doc = Http.url(url).cookies(c).data("username", username).data("offset", Integer.toString(offset))
|
||||
.data("limit", "24").data("_csrf", csfr).data("id", requestID)
|
||||
.ignoreContentType().post();
|
||||
return new JSONObject(doc.body().text());
|
||||
} catch (IOException e) {
|
||||
LOGGER.error("Got error trying to get page: " + e.getMessage());
|
||||
e.printStackTrace();
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
private JSONObject getFirstPageJSON(Document doc) {
|
||||
for (Element js : doc.select("script")) {
|
||||
if (js.html().contains("requestid")) {
|
||||
String json = js.html().replaceAll("window.__initial_body_data=", "").replaceAll("\\);", "")
|
||||
.replaceAll(";__wake\\(.+", "");
|
||||
JSONObject j = new JSONObject(json);
|
||||
return j;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public String getGalleryID(Document doc) {
|
||||
// If the url contains catpath we return 0 as the DA api will provide all galery images if you sent the
|
||||
// gallery id to 0
|
||||
if (url.toExternalForm().contains("catpath=")) {
|
||||
return "0";
|
||||
}
|
||||
Pattern p = Pattern.compile("^https?://www\\.deviantart\\.com/[a-zA-Z0-9\\-]+/gallery/([0-9]+)/?\\S+");
|
||||
Matcher m = p.matcher(url.toExternalForm());
|
||||
if (m.matches()) {
|
||||
return m.group(1);
|
||||
}
|
||||
for (Element el : doc.select("input[name=set]")) {
|
||||
try {
|
||||
String galleryID = el.attr("value");
|
||||
return galleryID;
|
||||
} catch (NullPointerException e) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
LOGGER.error("Could not find gallery ID");
|
||||
return null;
|
||||
}
|
||||
|
||||
public String getUsername(Document doc) {
|
||||
return doc.select("meta[property=og:title]").attr("content")
|
||||
.replaceAll("'s DeviantArt gallery", "").replaceAll("'s DeviantArt Gallery", "");
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public List<String> getURLsFromJSON(JSONObject json) {
|
||||
List<String> imageURLs = new ArrayList<>();
|
||||
JSONArray results = json.getJSONObject("content").getJSONArray("results");
|
||||
for (int i = 0; i < results.length(); i++) {
|
||||
Document doc = Jsoup.parseBodyFragment(results.getJSONObject(i).getString("html"));
|
||||
if (doc.html().contains("ismature")) {
|
||||
LOGGER.info("Downloading nsfw image");
|
||||
String nsfwImage = getFullsizedNSFWImage(doc.select("span").attr("href"));
|
||||
if (nsfwImage != null && nsfwImage.startsWith("http")) {
|
||||
imageURLs.add(nsfwImage);
|
||||
}
|
||||
}
|
||||
try {
|
||||
String imageURL = doc.select("span").first().attr("data-super-full-img");
|
||||
if (!imageURL.isEmpty() && imageURL.startsWith("http")) {
|
||||
imageURLs.add(imageURL);
|
||||
}
|
||||
} catch (NullPointerException e) {
|
||||
LOGGER.info(i + " does not contain any images");
|
||||
}
|
||||
|
||||
}
|
||||
return imageURLs;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public JSONObject getNextPage(JSONObject page) throws IOException {
|
||||
boolean hasMore = page.getJSONObject("content").getBoolean("has_more");
|
||||
if (hasMore) {
|
||||
return requestPage(page.getJSONObject("content").getInt("next_offset"), galleryID, username, requestID, csrf, pageCookies);
|
||||
}
|
||||
if (messages.size() > 0) {
|
||||
|
||||
// if message exists -> last page
|
||||
LOGGER.info("Messages amount: " + messages.size() + " - Next Page does not exists");
|
||||
throw new IOException("No more pages");
|
||||
}
|
||||
|
||||
return Http.url(urlWithParams(this.offset)).referrer(referer).userAgent(userAgent).cookies(getDACookie()).get();
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns list of Links to the Image pages. NOT links to fullsize image!!! e.g.
|
||||
* https://www.deviantart.com/kageuri/art/RUBY-568396655
|
||||
*/
|
||||
@Override
|
||||
public boolean keepSortOrder() {
|
||||
// Don't keep sort order (do not add prefixes).
|
||||
// Causes file duplication, as outlined in https://github.com/4pr0n/ripme/issues/113
|
||||
protected List<String> getURLsFromPage(Document page) {
|
||||
|
||||
List<String> result = new ArrayList<String>();
|
||||
|
||||
Element div;
|
||||
if (usingCatPath) {
|
||||
div = page.getElementById("gmi-");
|
||||
|
||||
} else {
|
||||
div = page.getElementsByClass("folderview-art").first().child(0);
|
||||
|
||||
}
|
||||
Elements links = div.select("a.torpedo-thumb-link");
|
||||
|
||||
for (Element el : links) {
|
||||
result.add(el.attr("href"));
|
||||
|
||||
}
|
||||
|
||||
LOGGER.info("Amount of Images on Page: " + result.size());
|
||||
LOGGER.info(page.location());
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts new Thread to find download link + filename + filetype
|
||||
*/
|
||||
@Override
|
||||
protected void downloadURL(URL url, int index) {
|
||||
this.downloadCount += 1;
|
||||
LOGGER.info("Downloading URL Number " + this.downloadCount);
|
||||
LOGGER.info("Deviant Art URL: " + url.toExternalForm());
|
||||
try {
|
||||
Response re = Http.url(urlWithParams(this.offset)).cookies(getDACookie()).referrer(referer)
|
||||
.userAgent(userAgent).response();
|
||||
updateCookie(re.cookies());
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
// Start Thread and add to pool.
|
||||
DeviantartImageThread t = new DeviantartImageThread(url);
|
||||
deviantartThreadPool.addThread(t);
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public String normalizeUrl(String url) {
|
||||
return (urlWithParams(this.offset).toExternalForm());
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns name of album. Album name consists of 3 words: - Artist (owner of
|
||||
* gallery) - Type (gallery or favorites folder) - Name of the folder
|
||||
*
|
||||
* Returns artist_type_name
|
||||
*/
|
||||
@Override
|
||||
public String getGID(URL url) throws MalformedURLException {
|
||||
|
||||
String s = url.toExternalForm();
|
||||
String artist = "unknown";
|
||||
String what = "unknown";
|
||||
String albumname = "unknown";
|
||||
|
||||
if (url.toExternalForm().contains("catpath=/")) {
|
||||
this.usingCatPath = true;
|
||||
}
|
||||
|
||||
Pattern p = Pattern.compile("^https?://www.deviantart\\.com/([a-zA-Z0-9]+).*$");
|
||||
Matcher m = p.matcher(s);
|
||||
|
||||
// Artist
|
||||
if (m.matches()) {
|
||||
artist = m.group(1);
|
||||
} else {
|
||||
throw new MalformedURLException("Expected deviantart.com URL format: "
|
||||
+ "www.deviantart.com/<ARTIST>/gallery/<NUMBERS>/<NAME>\nOR\nwww.deviantart.com/<ARTIST>/favourites/<NUMBERS>/<NAME>\\nOr simply the gallery or favorites of some artist - got "
|
||||
+ url + " instead");
|
||||
}
|
||||
|
||||
// What is it
|
||||
if (s.contains("/gallery/")) {
|
||||
what = "gallery";
|
||||
} else if (s.contains("/favourites/")) {
|
||||
what = "favourites";
|
||||
} else {
|
||||
throw new MalformedURLException("Expected deviantart.com URL format: "
|
||||
+ "www.deviantart.com/<ARTIST>/gallery/<NUMBERS>/<NAME>\nOR\nwww.deviantart.com/<ARTIST>/favourites/<NUMBERS>/<NAME>\nOr simply the gallery or favorites of some artist - got "
|
||||
+ url + " instead");
|
||||
}
|
||||
|
||||
// Album Name
|
||||
Pattern artistP = Pattern
|
||||
.compile("^https?://www.deviantart\\.com/[a-zA-Z0-9]+/[a-zA-Z]+/[0-9]+/([a-zA-Z0-9-]+).*$");
|
||||
Matcher artistM = artistP.matcher(s);
|
||||
if (s.endsWith("?catpath=/")) {
|
||||
albumname = "all";
|
||||
} else if (s.endsWith("/favourites/") || s.endsWith("/gallery/")) {
|
||||
albumname = "featured";
|
||||
} else if (artistM.matches()) {
|
||||
albumname = artistM.group(1);
|
||||
}
|
||||
LOGGER.info("Album Name: " + artist + "_" + what + "_" + albumname);
|
||||
|
||||
return artist + "_" + what + "_" + albumname;
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @return Clean URL as String
|
||||
*/
|
||||
private String cleanURL() {
|
||||
return (this.url.toExternalForm().split("\\?"))[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* Return correct url with params (catpath) and current offset
|
||||
*
|
||||
* @return URL to page with offset
|
||||
*/
|
||||
private URL urlWithParams(int offset) {
|
||||
try {
|
||||
String url = cleanURL();
|
||||
if (this.usingCatPath) {
|
||||
return (new URL(url + "?catpath=/&offset=" + offset));
|
||||
} else {
|
||||
return (new URL(url + "?offset=" + offset));
|
||||
}
|
||||
} catch (MalformedURLException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns Hashmap usable as Cookie for NSFW Artworks Not really needed but
|
||||
* maybe useful later.
|
||||
*
|
||||
* @return Cookie Hashmap
|
||||
*/
|
||||
private Map<String, String> getDACookie() {
|
||||
return this.cookies;
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates cookies
|
||||
*
|
||||
* @param m new Cookies
|
||||
*/
|
||||
private void updateCookie(Map<String, String> m) {
|
||||
|
||||
if (m == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
Iterator<String> iter = m.keySet().iterator();
|
||||
while (iter.hasNext()) {
|
||||
String current = iter.next();
|
||||
if (!this.allowedCookies.contains(current)) {
|
||||
// m.remove(current);
|
||||
iter.remove();
|
||||
}
|
||||
}
|
||||
|
||||
LOGGER.info("Updating Cookies");
|
||||
LOGGER.info("Old Cookies: " + getDACookie() + " ");
|
||||
LOGGER.info("New Cookies: " + m + " ");
|
||||
this.cookies.putAll(m);
|
||||
this.cookies.put("agegate_state", "1");
|
||||
LOGGER.info("Merged Cookies: " + getDACookie() + " ");
|
||||
|
||||
try {
|
||||
Utils.setConfigString(utilsKey, serialize(new HashMap<String, String>(getDACookie())));
|
||||
Utils.saveConfig();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Serializes an Object and returns a String ready to store Used to store
|
||||
* cookies in the config file because the deviantart cookies contain all sort of
|
||||
* special characters like ; , = : and so on.
|
||||
*
|
||||
* @param o Object to serialize
|
||||
* @return The serialized base64 encoded object
|
||||
* @throws IOException
|
||||
*/
|
||||
private String serialize(Serializable o) throws IOException {
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
ObjectOutputStream oos = new ObjectOutputStream(baos);
|
||||
oos.writeObject(o);
|
||||
oos.close();
|
||||
return Base64.getEncoder().encodeToString(baos.toByteArray());
|
||||
}
|
||||
|
||||
/**
|
||||
* Recreates the object from the base64 encoded String. Used for Cookies
|
||||
*
|
||||
* @param s the Base64 encoded string
|
||||
* @return the Cookie Map
|
||||
* @throws IOException
|
||||
* @throws ClassNotFoundException
|
||||
*/
|
||||
private Map<String, String> deserialize(String s) throws IOException, ClassNotFoundException {
|
||||
byte[] data = Base64.getDecoder().decode(s);
|
||||
ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(data));
|
||||
HashMap<String, String> o = (HashMap<String, String>) ois.readObject(); // Unchecked cast here but should never
|
||||
// be something else
|
||||
ois.close();
|
||||
return o;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the current cookies are still valid/usable. Also checks if agegate
|
||||
* is given.
|
||||
*
|
||||
* @return True when all is good.
|
||||
*/
|
||||
private boolean checkLogin() {
|
||||
if (!getDACookie().containsKey("agegate_state")) {
|
||||
LOGGER.info("No agegate key");
|
||||
return false;
|
||||
} else if (!getDACookie().get("agegate_state").equals("1")) {
|
||||
LOGGER.info("Wrong agegate value");
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
LOGGER.info("Login with Cookies: " + getDACookie());
|
||||
Response res = Http.url("https://www.deviantart.com/users/login").connection().followRedirects(true)
|
||||
.cookies(getDACookie()).referrer(this.referer).userAgent(this.userAgent).execute();
|
||||
if (!res.url().toExternalForm().equals("https://www.deviantart.com/users/login")) {
|
||||
LOGGER.info("Cookies are valid");
|
||||
LOGGER.info(res.url());
|
||||
return true;
|
||||
} else {
|
||||
LOGGER.info("Cookies invalid. Wrong URL: " + res.url());
|
||||
LOGGER.info(res.statusCode());
|
||||
LOGGER.info(res.parse());
|
||||
return false;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyzes an image page like
|
||||
* https://www.deviantart.com/kageuri/art/RUBY-568396655 .
|
||||
*
|
||||
* Looks for download button, follows the authentications and redirects and adds
|
||||
* the Image URL to the download queue. If no download button is present it will
|
||||
* use the largest version of the image.
|
||||
*
|
||||
* Should work with all filetypes on Deviantart. Tested with .JPG .PNG and .PDF
|
||||
*
|
||||
* @author MrPlaygon
|
||||
*
|
||||
*/
|
||||
private class DeviantartImageThread extends Thread {
|
||||
private URL url;
|
||||
|
||||
public DeviantartImageThread(URL url) {
|
||||
this.url = url;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void downloadURL(URL url, int index) {
|
||||
addURLToDownload(url, getPrefix(index), "", this.url.toExternalForm(), cookies);
|
||||
sleep(IMAGE_SLEEP_TIME);
|
||||
public void run() {
|
||||
getFullSizeURL();
|
||||
}
|
||||
|
||||
/**
|
||||
* Tries to get full size image from thumbnail URL
|
||||
* @param thumb Thumbnail URL
|
||||
* @param throwException Whether or not to throw exception when full size image isn't found
|
||||
* @return Full-size image URL
|
||||
* @throws Exception If it can't find the full-size URL
|
||||
* Get URL to Artwork and return fullsize URL with file ending.
|
||||
*
|
||||
* @param page Like
|
||||
* https://www.deviantart.com/apofiss/art/warmest-of-the-days-455668450
|
||||
* @return URL like
|
||||
* https://images-wixmp-ed30a86b8c4ca887773594c2.wixmp.com/intermediary/f/07f7a6bb-2d35-4630-93fc-be249af22b3e/d7jak0y-d20e5932-df72-4d13-b002-5e122037b373.jpg
|
||||
*
|
||||
*
|
||||
*/
|
||||
private static String thumbToFull(String thumb, boolean throwException) throws Exception {
|
||||
thumb = thumb.replace("http://th", "http://fc");
|
||||
List<String> fields = new ArrayList<>(Arrays.asList(thumb.split("/")));
|
||||
fields.remove(4);
|
||||
if (!fields.get(4).equals("f") && throwException) {
|
||||
// Not a full-size image
|
||||
throw new Exception("Can't get full size image from " + thumb);
|
||||
}
|
||||
StringBuilder result = new StringBuilder();
|
||||
for (int i = 0; i < fields.size(); i++) {
|
||||
if (i > 0) {
|
||||
result.append("/");
|
||||
}
|
||||
result.append(fields.get(i));
|
||||
}
|
||||
return result.toString();
|
||||
}
|
||||
private void getFullSizeURL() {
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* If largest resolution for image at 'thumb' is found, starts downloading
|
||||
* and returns null.
|
||||
* If it finds a larger resolution on another page, returns the image URL.
|
||||
* @param thumb Thumbnail URL
|
||||
* @param page Page the thumbnail is retrieved from
|
||||
* @return Highest-resolution version of the image based on thumbnail URL and the page.
|
||||
*/
|
||||
private String smallToFull(String thumb, String page) {
|
||||
LOGGER.info("Searching max. Resolution for " + url);
|
||||
sendUpdate(STATUS.LOADING_RESOURCE, "Searching max. resolution for " + url);
|
||||
try {
|
||||
// Fetch the image page
|
||||
Response resp = Http.url(page)
|
||||
.referrer(this.url)
|
||||
.cookies(cookies)
|
||||
.response();
|
||||
cookies.putAll(resp.cookies());
|
||||
Document doc = resp.parse();
|
||||
Elements els = doc.select("img.dev-content-full");
|
||||
String fsimage = null;
|
||||
// Get the largest resolution image on the page
|
||||
if (!els.isEmpty()) {
|
||||
// Large image
|
||||
fsimage = els.get(0).attr("src");
|
||||
LOGGER.info("Found large-scale: " + fsimage);
|
||||
if (fsimage.contains("//orig")) {
|
||||
return fsimage;
|
||||
Response re = Http.url(url).connection().referrer(referer).userAgent(userAgent).cookies(getDACookie())
|
||||
.execute();
|
||||
Document doc = re.parse();
|
||||
|
||||
// Artwork Title
|
||||
String title = doc.select("a.title").first().html();
|
||||
title = title.replaceAll("[^a-zA-Z0-9\\.\\-]", "_").toLowerCase();
|
||||
|
||||
int counter = 1;
|
||||
if (names.contains(title)) {
|
||||
while (names.contains(title + "_" + counter)) {
|
||||
counter++;
|
||||
}
|
||||
title = title + "_" + counter;
|
||||
}
|
||||
// Try to find the download button
|
||||
els = doc.select("a.dev-page-download");
|
||||
if (!els.isEmpty()) {
|
||||
// Full-size image
|
||||
String downloadLink = els.get(0).attr("href");
|
||||
LOGGER.info("Found download button link: " + downloadLink);
|
||||
HttpURLConnection con = (HttpURLConnection) new URL(downloadLink).openConnection();
|
||||
con.setRequestProperty("Referer",this.url.toString());
|
||||
String cookieString = "";
|
||||
for (Map.Entry<String, String> entry : cookies.entrySet()) {
|
||||
cookieString = cookieString + entry.getKey() + "=" + entry.getValue() + "; ";
|
||||
names.add(title);
|
||||
|
||||
// Check for download button
|
||||
Element downloadButton = null;
|
||||
|
||||
downloadButton = doc.select("a.dev-page-download").first();
|
||||
|
||||
// Download Button
|
||||
if (downloadButton != null) {
|
||||
LOGGER.info("Download Button found: " + downloadButton.attr("href"));
|
||||
|
||||
Response download = Http.url(downloadButton.attr("href")).connection().cookies(getDACookie())
|
||||
.method(Method.GET).referrer(referer).userAgent(userAgent).ignoreContentType(true)
|
||||
.followRedirects(true).execute();
|
||||
URL location = download.url();
|
||||
|
||||
System.out.println("----------------> " + url);
|
||||
String[] filetypePart = download.header("Content-Disposition").split("\\.");
|
||||
|
||||
LOGGER.info("Found Image URL");
|
||||
LOGGER.info(url);
|
||||
LOGGER.info(location);
|
||||
|
||||
addURLToDownload(location, "", "", "", getDACookie(),
|
||||
title + "." + filetypePart[filetypePart.length - 1]);
|
||||
return;
|
||||
}
|
||||
cookieString = cookieString.substring(0,cookieString.length() - 1);
|
||||
con.setRequestProperty("Cookie",cookieString);
|
||||
con.setRequestProperty("User-Agent", USER_AGENT);
|
||||
con.setInstanceFollowRedirects(true);
|
||||
con.connect();
|
||||
int code = con.getResponseCode();
|
||||
String location = con.getURL().toString();
|
||||
con.disconnect();
|
||||
if (location.contains("//orig")) {
|
||||
fsimage = location;
|
||||
LOGGER.info("Found image download: " + location);
|
||||
|
||||
// No Download Button
|
||||
Element div = doc.select("div.dev-view-deviation").first();
|
||||
|
||||
Element image = div.getElementsByTag("img").first();
|
||||
|
||||
String source = "";
|
||||
if (image == null) {
|
||||
LOGGER.error("ERROR on " + url);
|
||||
|
||||
LOGGER.error("Cookies: " + getDACookie() + " ");
|
||||
LOGGER.error(div);
|
||||
sendUpdate(STATUS.DOWNLOAD_ERRORED, "ERROR at\n" + url);
|
||||
return;
|
||||
}
|
||||
|
||||
// When it is text art (e.g. story) the only image is the avator (profile
|
||||
// picture)
|
||||
if (image.hasClass("avatar")) {
|
||||
LOGGER.error("No Image found, probably text art");
|
||||
LOGGER.error(url);
|
||||
return;
|
||||
}
|
||||
if (fsimage != null) {
|
||||
return fsimage;
|
||||
|
||||
source = image.attr("src");
|
||||
|
||||
String[] parts = source.split("/v1/");
|
||||
|
||||
// Image page uses scaled down version. Split at /v1/ to receive max size.
|
||||
if (parts.length > 2) {
|
||||
LOGGER.error("Unexpected URL Format");
|
||||
sendUpdate(STATUS.DOWNLOAD_ERRORED, "Unexpected URL Format");
|
||||
return;
|
||||
}
|
||||
throw new IOException("No download page found");
|
||||
} catch (IOException ioe) {
|
||||
|
||||
String[] tmpParts = parts[0].split("\\.");
|
||||
|
||||
LOGGER.info("Found Image URL");
|
||||
LOGGER.info(url);
|
||||
LOGGER.info(parts[0]);
|
||||
while (Http.url(parts[0]).connection().execute().statusCode() == 404) {
|
||||
try {
|
||||
LOGGER.info("Failed to get full size download image at " + page + " : '" + ioe.getMessage() + "'");
|
||||
String lessThanFull = thumbToFull(thumb, false);
|
||||
LOGGER.info("Falling back to less-than-full-size image " + lessThanFull);
|
||||
return lessThanFull;
|
||||
LOGGER.error("404 on " + url);
|
||||
Thread.sleep(1000);
|
||||
} catch (Exception e) {
|
||||
return null;
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
addURLToDownload(new URL(parts[0]), "", "", "", new HashMap<String, String>(),
|
||||
title + "." + tmpParts[tmpParts.length - 1]);
|
||||
return;
|
||||
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns DA cookies.
|
||||
* @return Map of cookies containing session data.
|
||||
*/
|
||||
private Map<String, String> getDACookies() {
|
||||
return RipUtils.getCookiesFromString(Utils.getConfigString("deviantart.cookies", loginCookies));
|
||||
LOGGER.error("No Full Size URL for: " + url);
|
||||
sendUpdate(STATUS.DOWNLOAD_ERRORED, "No image found for " + url);
|
||||
|
||||
return;
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -31,8 +31,9 @@ public class DeviantartRipperTest extends RippersTest {
|
||||
URL url = new URL("https://www.deviantart.com/airgee/gallery/");
|
||||
DeviantartRipper ripper = new DeviantartRipper(url);
|
||||
Document doc = Http.url(url).get();
|
||||
assertEquals("airgee", ripper.getUsername(doc));
|
||||
assertEquals("714589", ripper.getGalleryID(doc));
|
||||
//Had to comment because of refactoring/style change
|
||||
//assertEquals("airgee", ripper.getUsername(doc));
|
||||
//assertEquals("714589", ripper.getGalleryID(doc));
|
||||
}
|
||||
|
||||
public void testSanitizeURL() throws IOException {
|
||||
|
||||
Reference in New Issue
Block a user