mirror/ripme
1
0
Fork 0
mirror of https://github.com/RipMeApp/ripme.git synced 2025-04-22 12:42:02 +02:00
Code Issues Releases Wiki Activity

Merge pull request #1638 from blackbirddx/master

Browse Source 
Updated e621 ripper
This commit is contained in:
cyian-1756 2020-05-21 04:01:44 +00:00 committed by GitHub
commit bfb8d3eb52
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 112 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java
View File

@ -3,12 +3,18 @@ package com.rarchives.ripme.ripper.rippers;
import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.ripper.DownloadThreadPool;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.RipUtils;
import com.rarchives.ripme.utils.Utils;
import com.rarchives.ripme.ui.RipStatusMessage;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.log4j.Logger;
@ -16,6 +22,8 @@ import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
public class E621Ripper extends AbstractHTMLRipper {
private static final Logger logger = Logger.getLogger(E621Ripper.class);
@ -23,12 +31,44 @@ public class E621Ripper extends AbstractHTMLRipper {
private static Pattern gidPattern2 = null;
private static Pattern gidPatternPool = null;
private static Pattern gidPatternNew = null;
private static Pattern gidPatternPoolNew = null;
private DownloadThreadPool e621ThreadPool = new DownloadThreadPool("e621");
private Map<String, String> cookies = new HashMap<String, String>();
private String userAgent = USER_AGENT;
public E621Ripper(URL url) throws IOException {
super(url);
}
private void loadConfig() {
String cookiesString = Utils.getConfigString("e621.cookies", "");
if(!cookiesString.equals("")) {
cookies = RipUtils.getCookiesFromString(cookiesString);
if(cookies.containsKey("cf_clearance"))
sendUpdate(STATUS.DOWNLOAD_WARN, "Using CloudFlare captcha cookies, make sure to update them and set your browser's useragent in config!");
if(cookies.containsKey("remember"))
sendUpdate(STATUS.DOWNLOAD_WARN, "Logging in using auth cookie.");
}
userAgent = Utils.getConfigString("e621.useragent", USER_AGENT);
}
private void warnAboutBlacklist(Document page) {
if(!page.select("div.hidden-posts-notice").isEmpty())
sendUpdate(STATUS.DOWNLOAD_WARN, "Some posts are blacklisted. Consider logging in. Search for \"e621\" in this wiki page: https://github.com/RipMeApp/ripme/wiki/Config-options");
}
private Document getDocument(String url, int retries) throws IOException {
return Http.url(url).userAgent(userAgent).retries(retries).cookies(cookies).get();
}
private Document getDocument(String url) throws IOException {
return getDocument(url, 1);
}
@Override
public DownloadThreadPool getThreadPool() {
return e621ThreadPool;
@ -46,15 +86,20 @@ public class E621Ripper extends AbstractHTMLRipper {
@Override
public Document getFirstPage() throws IOException {
if (url.getPath().startsWith("/pool/show/"))
return Http.url("https://e621.net/pool/show/" + getTerm(url)).get();
loadConfig();
Document page;
if (url.getPath().startsWith("/pool"))
page = getDocument("https://e621.net/pools/" + getTerm(url));
else
return Http.url("https://e621.net/post/index/1/" + getTerm(url)).get();
page = getDocument("https://e621.net/posts?tags=" + getTerm(url));
warnAboutBlacklist(page);
return page;
}
@Override
public List<String> getURLsFromPage(Document page) {
Elements elements = page.select("div > span.thumb > a");
Elements elements = page.select("article > a");
List<String> res = new ArrayList<>();
for (Element e : elements) {
@ -68,8 +113,9 @@ public class E621Ripper extends AbstractHTMLRipper {
@Override
public Document getNextPage(Document page) throws IOException {
if (!page.select("a.next_page").isEmpty()) {
return Http.url(page.select("a.next_page").attr("abs:href")).get();
warnAboutBlacklist(page);
if (!page.select("a#paginator-next").isEmpty()) {
return getDocument(page.select("a#paginator-next").attr("abs:href"));
} else {
throw new IOException("No more pages.");
}
@ -82,12 +128,19 @@ public class E621Ripper extends AbstractHTMLRipper {
}
private String getTerm(URL url) throws MalformedURLException {
// old url style => new url style:
// /post/index/1/<tags> => /posts?tags=<tags>
// /pool/show/<id> => /pools/id
if (gidPattern == null)
gidPattern = Pattern.compile(
"^https?://(www\\.)?e621\\.net/post/index/[^/]+/([a-zA-Z0-9$_.+!*'():,%\\-]+)(/.*)?(#.*)?$");
if (gidPatternPool == null)
gidPatternPool = Pattern.compile(
"^https?://(www\\.)?e621\\.net/pool/show/([a-zA-Z0-9$_.+!*'(),%:\\-]+)(\\?.*)?(/.*)?(#.*)?$");
if (gidPatternNew == null)
gidPatternNew = Pattern.compile("^https?://(www\\.)?e621\\.net/posts\\?([\\S]*?)tags=([a-zA-Z0-9$_.+!*'(),%:\\-]+)(\\&[\\S]+)?");
if (gidPatternPoolNew == null)
gidPatternPoolNew = Pattern.compile("^https?://(www\\.)?e621\\.net/pools/([\\d]+)(\\?[\\S]*)?");
Matcher m = gidPattern.matcher(url.toExternalForm());
if (m.matches()) {
@ -100,14 +153,26 @@ public class E621Ripper extends AbstractHTMLRipper {
return m.group(2);
}
m = gidPatternNew.matcher(url.toExternalForm());
if (m.matches()) {
LOGGER.info(m.group(3));
return m.group(3);
}
m = gidPatternPoolNew.matcher(url.toExternalForm());
if (m.matches()) {
LOGGER.info(m.group(2));
return m.group(2);
}
throw new MalformedURLException(
"Expected e621.net URL format: e621.net/post/index/1/searchterm - got " + url + " instead");
"Expected e621.net URL format: e621.net/posts?tags=searchterm - got " + url + " instead");
}
@Override
public String getGID(URL url) throws MalformedURLException {
String prefix = "";
if (url.getPath().startsWith("/pool/show/")) {
if (url.getPath().startsWith("/pool")) {
prefix = "pool_";
}
return Utils.filesystemSafe(prefix + getTerm(url));
@ -149,8 +214,8 @@ public class E621Ripper extends AbstractHTMLRipper {
}
private String getFullSizedImage(URL imageURL) throws IOException {
Document page = Http.url(imageURL).retries(3).get();
Elements video = page.select("video > source");
Document page = getDocument(imageURL.toExternalForm(), 3);
/*Elements video = page.select("video > source");
Elements flash = page.select("embed");
Elements image = page.select("a#highres");
if (video.size() > 0) {
@ -161,8 +226,15 @@ public class E621Ripper extends AbstractHTMLRipper {
return image.attr("href");
} else {
throw new IOException();
}
}*/
if (!page.select("div#image-download-link > a").isEmpty()) {
return page.select("div#image-download-link > a").attr("abs:href");
} else {
if(!page.select("#blacklist-box").isEmpty())
sendUpdate(RipStatusMessage.STATUS.RIP_ERRORED, "Cannot download image - blocked by blacklist. Consider logging in. Search for \"e621\" in this wiki page: https://github.com/RipMeApp/ripme/wiki/Config-options");
throw new IOException();
}
}
}

31
src/test/java/com/rarchives/ripme/tst/ripper/rippers/E621RipperTest.java
View File

@ -8,16 +8,43 @@ import org.junit.jupiter.api.Test;
public class E621RipperTest extends RippersTest {
public void testRip() throws IOException {
E621Ripper ripper = new E621Ripper(new URL("https://e621.net/post/index/1/beach"));
E621Ripper ripper = new E621Ripper(new URL("https://e621.net/posts?tags=beach"));
testRipper(ripper);
}
@Test
public void testFlashOrWebm() throws IOException {
E621Ripper ripper = new E621Ripper(new URL("https://e621.net/post/index/1/gif"));
E621Ripper ripper = new E621Ripper(new URL("https://e621.net/posts?page=4&tags=gif+rating%3As+3d"));
testRipper(ripper);
}
@Test
public void testGetNextPage() throws IOException {
E621Ripper nextPageRipper = new E621Ripper(new URL("https://e621.net/posts?tags=cosmicminerals"));
try {
nextPageRipper.getNextPage(nextPageRipper.getFirstPage());
assert (true);
} catch (IOException e) {
throw e;
}
E621Ripper noNextPageRipper = new E621Ripper(new URL("https://e621.net/post/index/1/cosmicminerals"));
try {
noNextPageRipper.getNextPage(noNextPageRipper.getFirstPage());
} catch (IOException e) {
assertEquals(e.getMessage(), "No more pages.");
}
}
@Test
public void testOldRip() throws IOException {
E621Ripper ripper = new E621Ripper(new URL("https://e621.net/post/index/1/beach"));
testRipper(ripper);
}
@Test
public void testOldFlashOrWebm() throws IOException {
E621Ripper ripper = new E621Ripper(new URL("https://e621.net/post/index/1/gif"));
testRipper(ripper);
}
@Test
public void testOldGetNextPage() throws IOException {
E621Ripper nextPageRipper = new E621Ripper(new URL("https://e621.net/post/index/1/cosmicminerals"));
try {
nextPageRipper.getNextPage(nextPageRipper.getFirstPage());