Merge pull request #1638 from blackbirddx/master

Updated e621 ripper

src/main/java/com/rarchives/ripme/ripper/rippers/E621Ripper.java

@@ -3,12 +3,18 @@ package com.rarchives.ripme.ripper.rippers;
 import com.rarchives.ripme.ripper.AbstractHTMLRipper;
 import com.rarchives.ripme.ripper.DownloadThreadPool;
 import com.rarchives.ripme.utils.Http;
+import com.rarchives.ripme.utils.RipUtils;
 import com.rarchives.ripme.utils.Utils;
+import com.rarchives.ripme.ui.RipStatusMessage;
+import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
+
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import org.apache.log4j.Logger;
@@ -16,6 +22,8 @@ import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.jsoup.select.Elements;
 
+
+
 public class E621Ripper extends AbstractHTMLRipper {
     private static final Logger logger = Logger.getLogger(E621Ripper.class);
 
@@ -23,12 +31,44 @@ public class E621Ripper extends AbstractHTMLRipper {
     private static Pattern gidPattern2 = null;
     private static Pattern gidPatternPool = null;
 
+    private static Pattern gidPatternNew = null;
+    private static Pattern gidPatternPoolNew = null;
+
     private DownloadThreadPool e621ThreadPool = new DownloadThreadPool("e621");
 
+    private Map<String, String> cookies = new HashMap<String, String>();
+    private String userAgent = USER_AGENT;
+
     public E621Ripper(URL url) throws IOException {
         super(url);
     }
 
+    private void loadConfig() {
+        String cookiesString = Utils.getConfigString("e621.cookies", "");
+        if(!cookiesString.equals("")) {
+            cookies = RipUtils.getCookiesFromString(cookiesString);
+            if(cookies.containsKey("cf_clearance"))
+                sendUpdate(STATUS.DOWNLOAD_WARN, "Using CloudFlare captcha cookies, make sure to update them and set your browser's useragent in config!");
+            if(cookies.containsKey("remember"))
+                sendUpdate(STATUS.DOWNLOAD_WARN, "Logging in using auth cookie.");
+        }
+        userAgent = Utils.getConfigString("e621.useragent", USER_AGENT);
+        
+    }
+
+    private void warnAboutBlacklist(Document page) {
+        if(!page.select("div.hidden-posts-notice").isEmpty()) 
+            sendUpdate(STATUS.DOWNLOAD_WARN, "Some posts are blacklisted. Consider logging in. Search for \"e621\" in this wiki page: https://github.com/RipMeApp/ripme/wiki/Config-options");
+    }
+
+    private Document getDocument(String url, int retries) throws IOException {
+        return Http.url(url).userAgent(userAgent).retries(retries).cookies(cookies).get();
+    }
+
+    private Document getDocument(String url) throws IOException {
+        return getDocument(url, 1);
+    }
+
     @Override
     public DownloadThreadPool getThreadPool() {
         return e621ThreadPool;
@@ -46,15 +86,20 @@ public class E621Ripper extends AbstractHTMLRipper {
 
     @Override
     public Document getFirstPage() throws IOException {
-        if (url.getPath().startsWith("/pool/show/"))
+        loadConfig();
-            return Http.url("https://e621.net/pool/show/" + getTerm(url)).get();
+        Document page;
+        if (url.getPath().startsWith("/pool"))
+            page = getDocument("https://e621.net/pools/" + getTerm(url));
         else
-            return Http.url("https://e621.net/post/index/1/" + getTerm(url)).get();
+            page = getDocument("https://e621.net/posts?tags=" + getTerm(url));
+        
+        warnAboutBlacklist(page);
+        return page;
     }
 
     @Override
     public List<String> getURLsFromPage(Document page) {
-        Elements elements = page.select("div > span.thumb > a");
+        Elements elements = page.select("article > a");
         List<String> res = new ArrayList<>();
 
         for (Element e : elements) {
@@ -68,8 +113,9 @@ public class E621Ripper extends AbstractHTMLRipper {
 
     @Override
     public Document getNextPage(Document page) throws IOException {
-        if (!page.select("a.next_page").isEmpty()) {
+        warnAboutBlacklist(page);
-            return Http.url(page.select("a.next_page").attr("abs:href")).get();
+        if (!page.select("a#paginator-next").isEmpty()) {
+            return getDocument(page.select("a#paginator-next").attr("abs:href"));
         } else {
             throw new IOException("No more pages.");
         }
@@ -82,12 +128,19 @@ public class E621Ripper extends AbstractHTMLRipper {
     }
 
     private String getTerm(URL url) throws MalformedURLException {
+        // old url style => new url style:
+        // /post/index/1/<tags> => /posts?tags=<tags>
+        // /pool/show/<id> => /pools/id
         if (gidPattern == null)
             gidPattern = Pattern.compile(
                     "^https?://(www\\.)?e621\\.net/post/index/[^/]+/([a-zA-Z0-9$_.+!*'():,%\\-]+)(/.*)?(#.*)?$");
         if (gidPatternPool == null)
             gidPatternPool = Pattern.compile(
                     "^https?://(www\\.)?e621\\.net/pool/show/([a-zA-Z0-9$_.+!*'(),%:\\-]+)(\\?.*)?(/.*)?(#.*)?$");
+        if (gidPatternNew == null)
+            gidPatternNew = Pattern.compile("^https?://(www\\.)?e621\\.net/posts\\?([\\S]*?)tags=([a-zA-Z0-9$_.+!*'(),%:\\-]+)(\\&[\\S]+)?");
+        if (gidPatternPoolNew == null)
+            gidPatternPoolNew = Pattern.compile("^https?://(www\\.)?e621\\.net/pools/([\\d]+)(\\?[\\S]*)?");
 
         Matcher m = gidPattern.matcher(url.toExternalForm());
         if (m.matches()) {
@@ -100,14 +153,26 @@ public class E621Ripper extends AbstractHTMLRipper {
             return m.group(2);
         }
 
+        m = gidPatternNew.matcher(url.toExternalForm());
+        if (m.matches()) {
+            LOGGER.info(m.group(3));
+            return m.group(3);
+        }
+
+        m = gidPatternPoolNew.matcher(url.toExternalForm());
+        if (m.matches()) {
+            LOGGER.info(m.group(2));
+            return m.group(2);
+        }
+
         throw new MalformedURLException(
-                "Expected e621.net URL format: e621.net/post/index/1/searchterm - got " + url + " instead");
+                "Expected e621.net URL format: e621.net/posts?tags=searchterm - got " + url + " instead");
     }
 
     @Override
     public String getGID(URL url) throws MalformedURLException {
         String prefix = "";
-        if (url.getPath().startsWith("/pool/show/")) {
+        if (url.getPath().startsWith("/pool")) {
             prefix = "pool_";
         }
         return Utils.filesystemSafe(prefix + getTerm(url));
@@ -149,8 +214,8 @@ public class E621Ripper extends AbstractHTMLRipper {
         }
 
         private String getFullSizedImage(URL imageURL) throws IOException {
-            Document page = Http.url(imageURL).retries(3).get();
+            Document page = getDocument(imageURL.toExternalForm(), 3);
-            Elements video = page.select("video > source");
+            /*Elements video = page.select("video > source");
             Elements flash = page.select("embed");
             Elements image = page.select("a#highres");
             if (video.size() > 0) {
@@ -161,8 +226,15 @@ public class E621Ripper extends AbstractHTMLRipper {
                 return image.attr("href");
             } else {
                 throw new IOException();
-            }
+            }*/
 
+            if (!page.select("div#image-download-link > a").isEmpty()) {
+                return page.select("div#image-download-link > a").attr("abs:href");
+            } else {
+                if(!page.select("#blacklist-box").isEmpty())
+                    sendUpdate(RipStatusMessage.STATUS.RIP_ERRORED, "Cannot download image - blocked by blacklist. Consider logging in. Search for \"e621\" in this wiki page: https://github.com/RipMeApp/ripme/wiki/Config-options");
+                throw new IOException();
+            }
         }
 
     }

src/test/java/com/rarchives/ripme/tst/ripper/rippers/E621RipperTest.java

@@ -8,16 +8,43 @@ import org.junit.jupiter.api.Test;
 
 public class E621RipperTest extends RippersTest {
     public void testRip() throws IOException {
-        E621Ripper ripper = new E621Ripper(new URL("https://e621.net/post/index/1/beach"));
+        E621Ripper ripper = new E621Ripper(new URL("https://e621.net/posts?tags=beach"));
         testRipper(ripper);
     }
     @Test
     public void testFlashOrWebm() throws IOException {
-        E621Ripper ripper = new E621Ripper(new URL("https://e621.net/post/index/1/gif"));
+        E621Ripper ripper = new E621Ripper(new URL("https://e621.net/posts?page=4&tags=gif+rating%3As+3d"));
         testRipper(ripper);
     }
     @Test
     public void testGetNextPage() throws IOException {
+        E621Ripper nextPageRipper = new E621Ripper(new URL("https://e621.net/posts?tags=cosmicminerals"));
+        try {
+            nextPageRipper.getNextPage(nextPageRipper.getFirstPage());
+            assert (true);
+        } catch (IOException e) {
+            throw e;
+        }
+
+        E621Ripper noNextPageRipper = new E621Ripper(new URL("https://e621.net/post/index/1/cosmicminerals"));
+        try {
+            noNextPageRipper.getNextPage(noNextPageRipper.getFirstPage());
+        } catch (IOException e) {
+            assertEquals(e.getMessage(), "No more pages.");
+        }
+    }
+    @Test
+    public void testOldRip() throws IOException {
+        E621Ripper ripper = new E621Ripper(new URL("https://e621.net/post/index/1/beach"));
+        testRipper(ripper);
+    }
+    @Test
+    public void testOldFlashOrWebm() throws IOException {
+        E621Ripper ripper = new E621Ripper(new URL("https://e621.net/post/index/1/gif"));
+        testRipper(ripper);
+    }
+    @Test
+    public void testOldGetNextPage() throws IOException {
         E621Ripper nextPageRipper = new E621Ripper(new URL("https://e621.net/post/index/1/cosmicminerals"));
         try {
             nextPageRipper.getNextPage(nextPageRipper.getFirstPage());