mirror/ripme
1
0
Fork 0
mirror of https://github.com/RipMeApp/ripme.git synced 2025-08-06 13:56:34 +02:00
Code Issues Releases Wiki Activity

add e621 cloudflare captcha cookies

Browse Source
This commit is contained in:
BlackBirdd
2020-04-08 15:06:39 +02:00
parent 03abd937f3
commit fec50e465e
1 changed files with 25 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java
View File

@@ -3,12 +3,16 @@ package com.rarchives.ripme.ripper.rippers;
import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.ripper.DownloadThreadPool; import com.rarchives.ripme.ripper.DownloadThreadPool;
import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.RipUtils;
import com.rarchives.ripme.utils.Utils; import com.rarchives.ripme.utils.Utils;
import com.rarchives.ripme.ui.RipStatusMessage;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
@@ -16,10 +20,7 @@ import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
import org.jsoup.select.Elements; import org.jsoup.select.Elements;
// updated by BlackBirdd (https://github.com/blackbirddx) after e621 update in March 2020
// old url style => new url style:
// /post/index/1/<tags> => /posts?tags=<tags>
// /pool/show/<id> => /pools/id
public class E621Ripper extends AbstractHTMLRipper { public class E621Ripper extends AbstractHTMLRipper {
private static final Logger logger = Logger.getLogger(E621Ripper.class); private static final Logger logger = Logger.getLogger(E621Ripper.class);
@@ -33,8 +34,21 @@ public class E621Ripper extends AbstractHTMLRipper {
private DownloadThreadPool e621ThreadPool = new DownloadThreadPool("e621"); private DownloadThreadPool e621ThreadPool = new DownloadThreadPool("e621");
private Map<String, String> cookies = new HashMap<String, String>();
private boolean useAPIRipper = false;
public E621Ripper(URL url) throws IOException { public E621Ripper(URL url) throws IOException {
super(url); super(url);
loadConfig();
}
private void loadConfig() {
String cookiesString = Utils.getConfigString("e621.captcha_cookies", "");
if(!cookiesString.equals(" ")) {
cookies = RipUtils.getCookiesFromString(cookiesString);
sendUpdate(RipStatusMessage.STATUS.RIP_ERRORED, "Using CloudFlare captcha cookies, make sure to update them!");
}
useAPIRipper = Utils.getConfigBoolean("e621.use_API_ripper", false);
} }
@Override @Override
@@ -55,9 +69,9 @@ public class E621Ripper extends AbstractHTMLRipper {
@Override @Override
public Document getFirstPage() throws IOException { public Document getFirstPage() throws IOException {
if (url.getPath().startsWith("/pool")) if (url.getPath().startsWith("/pool"))
return Http.url("https://e621.net/pools/" + getTerm(url)).get(); return Http.url("https://e621.net/pools/" + getTerm(url)).cookies(cookies).get();
else else
return Http.url("https://e621.net/posts?tags=" + getTerm(url)).get(); return Http.url("https://e621.net/posts?tags=" + getTerm(url)).cookies(cookies).get();
} }
@Override @Override
@@ -77,7 +91,7 @@ public class E621Ripper extends AbstractHTMLRipper {
@Override @Override
public Document getNextPage(Document page) throws IOException { public Document getNextPage(Document page) throws IOException {
if (!page.select("a#paginator-next").isEmpty()) { if (!page.select("a#paginator-next").isEmpty()) {
return Http.url(page.select("a#paginator-next").attr("abs:href")).get(); return Http.url(page.select("a#paginator-next").attr("abs:href")).cookies(cookies).get();
} else { } else {
throw new IOException("No more pages."); throw new IOException("No more pages.");
} }
@@ -90,6 +104,9 @@ public class E621Ripper extends AbstractHTMLRipper {
} }
private String getTerm(URL url) throws MalformedURLException { private String getTerm(URL url) throws MalformedURLException {
// old url style => new url style:
// /post/index/1/<tags> => /posts?tags=<tags>
// /pool/show/<id> => /pools/id
if (gidPattern == null) if (gidPattern == null)
gidPattern = Pattern.compile( gidPattern = Pattern.compile(
"^https?://(www\\.)?e621\\.net/post/index/[^/]+/([a-zA-Z0-9$_.+!*'():,%\\-]+)(/.*)?(#.*)?$"); "^https?://(www\\.)?e621\\.net/post/index/[^/]+/([a-zA-Z0-9$_.+!*'():,%\\-]+)(/.*)?(#.*)?$");
@@ -173,7 +190,7 @@ public class E621Ripper extends AbstractHTMLRipper {
} }
private String getFullSizedImage(URL imageURL) throws IOException { private String getFullSizedImage(URL imageURL) throws IOException {
Document page = Http.url(imageURL).retries(3).get(); Document page = Http.url(imageURL).cookies(cookies).retries(3).get();
/*Elements video = page.select("video > source"); /*Elements video = page.select("video > source");
Elements flash = page.select("embed"); Elements flash = page.select("embed");
Elements image = page.select("a#highres"); Elements image = page.select("a#highres");