mirror/ripme
1
0
Fork 0
mirror of https://github.com/RipMeApp/ripme.git synced 2025-08-26 15:24:51 +02:00
Code Issues Releases Wiki Activity

Using constants for useragent and referer now.

Browse Source
This commit is contained in:
MrPlaygon
2019-02-17 10:39:14 +01:00
parent 48474ad7e7
commit 82e98abf7f
1 changed files with 20 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
src/main/java/com/rarchives/ripme/ripper/rippers/DeviantartRipper.java
View File

@@ -31,8 +31,9 @@ import org.jsoup.select.Elements;
* @author MrPlaygon * @author MrPlaygon
* *
* NOT using Deviantart API like the old JSON ripper because it is SLOW * NOT using Deviantart API like the old JSON ripper because it is SLOW
* and somehow annoying to use. * and somehow annoying to use. Things to consider: Using the API might
* Things to consider: Using the API might be less work/maintenance later because APIs do not change as frequently as HTML source code...? * be less work/maintenance later because APIs do not change as
* frequently as HTML source code...?
* *
* *
* *
@@ -75,6 +76,10 @@ public class DeviantartRipper extends AbstractHTMLRipper {
private DownloadThreadPool deviantartThreadPool = new DownloadThreadPool("deviantart"); private DownloadThreadPool deviantartThreadPool = new DownloadThreadPool("deviantart");
private ArrayList<String> names = new ArrayList<String>(); private ArrayList<String> names = new ArrayList<String>();
// Constants
private final String referer = "https://www.deviantart.com/";
private final String userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0";
@Override @Override
public DownloadThreadPool getThreadPool() { public DownloadThreadPool getThreadPool() {
return deviantartThreadPool; return deviantartThreadPool;
@@ -97,8 +102,7 @@ public class DeviantartRipper extends AbstractHTMLRipper {
@Override @Override
protected Document getFirstPage() throws IOException { protected Document getFirstPage() throws IOException {
login(); login();
return Http.url(urlWithParams(this.offset)).cookies(getDACookie()).referrer("https://www.deviantart.com/") return Http.url(urlWithParams(this.offset)).cookies(getDACookie()).referrer(referer).userAgent(userAgent).get();
.userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0").get();
} }
/** /**
@@ -119,9 +123,7 @@ public class DeviantartRipper extends AbstractHTMLRipper {
// Load login page // Load login page
Response res = Http.url("https://www.deviantart.com/users/login").connection().method(Method.GET) Response res = Http.url("https://www.deviantart.com/users/login").connection().method(Method.GET)
.referrer("https://www.deviantart.com/") .referrer(referer).userAgent(userAgent).execute();
.userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0")
.execute();
// Find tokens // Find tokens
Document doc = res.parse(); Document doc = res.parse();
@@ -142,14 +144,11 @@ public class DeviantartRipper extends AbstractHTMLRipper {
Map<String, String> cookies = res.cookies(); Map<String, String> cookies = res.cookies();
// Log in using data. Handle redirect // Log in using data. Handle redirect
res = Http.url("https://www.deviantart.com/users/login").connection() res = Http.url("https://www.deviantart.com/users/login").connection().referrer(referer).userAgent(userAgent)
.referrer("https://www.deviantart.com/")
.userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0")
.method(Method.POST).data(loginData).cookies(cookies).followRedirects(false).execute(); .method(Method.POST).data(loginData).cookies(cookies).followRedirects(false).execute();
this.cookies = res.cookies(); this.cookies = res.cookies();
res = Http.url(res.header("location")).connection().referrer("https://www.deviantart.com/") res = Http.url(res.header("location")).connection().referrer(referer).userAgent(userAgent)
.userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0")
.method(Method.GET).cookies(cookies).followRedirects(false).execute(); .method(Method.GET).cookies(cookies).followRedirects(false).execute();
// Store cookies // Store cookies
@@ -195,9 +194,8 @@ public class DeviantartRipper extends AbstractHTMLRipper {
@Override @Override
public Document getNextPage(Document doc) throws IOException { public Document getNextPage(Document doc) throws IOException {
this.offset += 24; this.offset += 24;
Response re = Http.url(urlWithParams(this.offset)).cookies(getDACookie()) Response re = Http.url(urlWithParams(this.offset)).cookies(getDACookie()).referrer(referer).userAgent(userAgent)
.referrer("https://www.deviantart.com/") .response();
.userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0").response();
updateCookie(re.cookies()); updateCookie(re.cookies());
Document docu = re.parse(); Document docu = re.parse();
Elements messages = docu.getElementsByClass("message"); Elements messages = docu.getElementsByClass("message");
@@ -212,9 +210,7 @@ public class DeviantartRipper extends AbstractHTMLRipper {
throw new IOException("No more pages"); throw new IOException("No more pages");
} }
return Http.url(urlWithParams(this.offset)).referrer("https://www.deviantart.com/") return Http.url(urlWithParams(this.offset)).referrer(referer).userAgent(userAgent).cookies(getDACookie()).get();
.userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0")
.cookies(getDACookie()).get();
} }
@@ -260,10 +256,8 @@ public class DeviantartRipper extends AbstractHTMLRipper {
System.out.println( System.out.println(
"------------------------------DAURL: " + url.toExternalForm() + "------------------------------"); "------------------------------DAURL: " + url.toExternalForm() + "------------------------------");
try { try {
Response re = Http.url(urlWithParams(this.offset)).cookies(getDACookie()) Response re = Http.url(urlWithParams(this.offset)).cookies(getDACookie()).referrer(referer)
.referrer("https://www.deviantart.com/") .userAgent(userAgent).response();
.userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0")
.response();
updateCookie(re.cookies()); updateCookie(re.cookies());
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
@@ -431,9 +425,8 @@ public class DeviantartRipper extends AbstractHTMLRipper {
+ "------------------------------"); + "------------------------------");
sendUpdate(STATUS.LOADING_RESOURCE, "Searching max. resolution for " + url); sendUpdate(STATUS.LOADING_RESOURCE, "Searching max. resolution for " + url);
try { try {
Response re = Http.url(url).connection().referrer("https://www.deviantart.com/") Response re = Http.url(url).connection().referrer(referer).userAgent(userAgent).cookies(getDACookie())
.userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0") .execute();
.cookies(getDACookie()).execute();
Document doc = re.parse(); Document doc = re.parse();
// Artwork Title // Artwork Title
@@ -460,9 +453,8 @@ public class DeviantartRipper extends AbstractHTMLRipper {
+ downloadButton.attr("href") + "------------------------------"); + downloadButton.attr("href") + "------------------------------");
Response download = Http.url(downloadButton.attr("href")).connection().cookies(getDACookie()) Response download = Http.url(downloadButton.attr("href")).connection().cookies(getDACookie())
.method(Method.GET).referrer("https://www.deviantart.com/") .method(Method.GET).referrer(referer).userAgent(userAgent).ignoreContentType(true)
.userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:65.0) Gecko/20100101 Firefox/65.0") .followRedirects(true).execute();
.ignoreContentType(true).followRedirects(true).execute();
URL location = download.url(); URL location = download.url();
String[] filetypePart = download.header("Content-Disposition").split("\\."); String[] filetypePart = download.header("Content-Disposition").split("\\.");