From 466b70c8ab67d1117b7e2137c8a92fae71b9a57e Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 5 Apr 2019 19:26:41 -0400 Subject: [PATCH 1/7] Added ripper for Erofus.com --- .../ripme/ripper/rippers/ErofusRipper.java | 134 ++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java new file mode 100644 index 00000000..a3fef42d --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java @@ -0,0 +1,134 @@ +package com.rarchives.ripme.ripper.rippers; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; +import com.rarchives.ripme.ui.RipStatusMessage; +import com.rarchives.ripme.utils.Http; +import org.json.JSONObject; +import org.jsoup.Connection; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class ErofusRipper extends AbstractHTMLRipper { + + private Document albumDoc = null; + private Map cookies = new HashMap<>(); + // TODO put up a wiki page on using maps to store titles + // the map for storing the title of each album when downloading sub albums + private Map urlTitles = new HashMap<>(); + + private Boolean rippingSubalbums = false; + + public ErofusRipper(URL url) throws IOException { + super(url); + } + + @Override + public boolean hasASAPRipping() { + return true; + } + + @Override + public String getHost() { + return "erofus"; + } + + @Override + public String getDomain() { + return "erofus.com"; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("^https://www.erofus.com/comics/([a-zA-Z0-9\\-_]+).*$"); + Matcher m = p.matcher(url.toExternalForm()); + if (!m.matches()) { + throw new MalformedURLException("Expected URL format: http://www.8muses.com/index/category/albumname, got: " + url); + } + return m.group(m.groupCount()); + } + + @Override + public Document getFirstPage() throws IOException { + if (albumDoc == null) { + Connection.Response resp = Http.url(url).response(); + cookies.putAll(resp.cookies()); + albumDoc = resp.parse(); + } + return albumDoc; + } + + @Override + public List getURLsFromPage(Document page) { + LOGGER.info(page); + List imageURLs = new ArrayList<>(); + int x = 1; + if (pageContainsImages(page)) { + LOGGER.info("Page contains images"); + imageURLs.addAll(ripAlbum(page)); + } else { + // This contains the thumbnails of all images on the page + Elements pageImages = page.select("a.a-click"); + for (Element pageLink : pageImages) { + if (super.isStopped()) break; + if (pageLink.attr("href").contains("comics")) { + String subUrl = "https://erofus.com" + pageLink.attr("href"); + try { + LOGGER.info("Retrieving " + subUrl); + sendUpdate(RipStatusMessage.STATUS.LOADING_RESOURCE, subUrl); + Document subPage = Http.url(subUrl).get(); + List subalbumImages = getURLsFromPage(subPage); + } catch (IOException e) { + LOGGER.warn("Error while loading subalbum " + subUrl, e); + } + } + if (isThisATest()) break; + } + } + + + return imageURLs; + } + + public List ripAlbum(Document page) { + int x = 1; + List imageURLs = new ArrayList<>(); + Elements thumbs = page.select("a.a-click > div.thumbnail > img"); + for (Element thumb : thumbs) { + String image = "https://www.erofus.com" + thumb.attr("src").replaceAll("thumb", "medium"); + imageURLs.add(image); + try { + addURLToDownload(new URL(image), getPrefix(x)); + } catch (MalformedURLException e) { + LOGGER.info(e.getMessage()); + } + x++; + } + return imageURLs; + } + + private boolean pageContainsImages(Document page) { + Elements pageImages = page.select("a.a-click"); + for (Element pageLink : pageImages) { + if (pageLink.attr("href").contains("/pic/")) { + return true; + } + } + return false; + } + + @Override + public void downloadURL(URL url, int index) { + addURLToDownload(url, getPrefix(index), "", this.url.toExternalForm(), cookies); + } +} \ No newline at end of file From e510973a21d7bb579f060577729af46098e274d0 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 5 Apr 2019 19:41:50 -0400 Subject: [PATCH 2/7] Fixed minor bug with addURLToDownload(url, map) which caused the file name to default to an empty string instead of null --- src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java index 6c79c63f..1220c5f4 100644 --- a/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java @@ -236,7 +236,7 @@ public abstract class AbstractRipper // Bit of a hack but this lets us pass a bool using a map boolean useMIME = options.getOrDefault("getFileExtFromMIME", "false").toLowerCase().equals("true"); return addURLToDownload(url, options.getOrDefault("prefix", ""), options.getOrDefault("subdirectory", ""), options.getOrDefault("referrer", null), - cookies, options.getOrDefault("fileName", ""), options.getOrDefault("extension", null), useMIME); + cookies, options.getOrDefault("fileName", null), options.getOrDefault("extension", null), useMIME); } From 4806519aedd0e3029a6dc13d884eeefbabe9027d Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 5 Apr 2019 19:42:17 -0400 Subject: [PATCH 3/7] Got sub album suppport working --- .../rarchives/ripme/ripper/rippers/ErofusRipper.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java index a3fef42d..db79f123 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java @@ -75,7 +75,7 @@ public class ErofusRipper extends AbstractHTMLRipper { int x = 1; if (pageContainsImages(page)) { LOGGER.info("Page contains images"); - imageURLs.addAll(ripAlbum(page)); + ripAlbum(page); } else { // This contains the thumbnails of all images on the page Elements pageImages = page.select("a.a-click"); @@ -100,21 +100,21 @@ public class ErofusRipper extends AbstractHTMLRipper { return imageURLs; } - public List ripAlbum(Document page) { + public void ripAlbum(Document page) { int x = 1; - List imageURLs = new ArrayList<>(); Elements thumbs = page.select("a.a-click > div.thumbnail > img"); for (Element thumb : thumbs) { String image = "https://www.erofus.com" + thumb.attr("src").replaceAll("thumb", "medium"); - imageURLs.add(image); try { - addURLToDownload(new URL(image), getPrefix(x)); + Map opts = new HashMap(); + opts.put("subdirectory", page.title().replaceAll(" \\| Erofus - Sex and Porn Comics", "").replaceAll(" ", "_")); + opts.put("prefix", getPrefix(x)); + addURLToDownload(new URL(image), opts); } catch (MalformedURLException e) { LOGGER.info(e.getMessage()); } x++; } - return imageURLs; } private boolean pageContainsImages(Document page) { From 8cadeee240793f50dbbcc0e51bd57c3d3dbeb1d2 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 5 Apr 2019 21:09:01 -0400 Subject: [PATCH 4/7] Remove some dead code --- .../ripme/ripper/rippers/ErofusRipper.java | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java index db79f123..dc535dea 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ErofusRipper.java @@ -3,8 +3,6 @@ package com.rarchives.ripme.ripper.rippers; import com.rarchives.ripme.ripper.AbstractHTMLRipper; import com.rarchives.ripme.ui.RipStatusMessage; import com.rarchives.ripme.utils.Http; -import org.json.JSONObject; -import org.jsoup.Connection; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; @@ -21,14 +19,6 @@ import java.util.regex.Pattern; public class ErofusRipper extends AbstractHTMLRipper { - private Document albumDoc = null; - private Map cookies = new HashMap<>(); - // TODO put up a wiki page on using maps to store titles - // the map for storing the title of each album when downloading sub albums - private Map urlTitles = new HashMap<>(); - - private Boolean rippingSubalbums = false; - public ErofusRipper(URL url) throws IOException { super(url); } @@ -60,12 +50,7 @@ public class ErofusRipper extends AbstractHTMLRipper { @Override public Document getFirstPage() throws IOException { - if (albumDoc == null) { - Connection.Response resp = Http.url(url).response(); - cookies.putAll(resp.cookies()); - albumDoc = resp.parse(); - } - return albumDoc; + return Http.url(url).get(); } @Override @@ -129,6 +114,6 @@ public class ErofusRipper extends AbstractHTMLRipper { @Override public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index), "", this.url.toExternalForm(), cookies); + addURLToDownload(url, getPrefix(index)); } } \ No newline at end of file From 681d6b5f726fcddf54ce9bbaa19ee91ebfffcadc Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 5 Apr 2019 21:10:48 -0400 Subject: [PATCH 5/7] Added a unittest for ErofusRipper --- .../ripme/tst/ripper/rippers/ErofusRipperTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErofusRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErofusRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErofusRipperTest.java new file mode 100644 index 00000000..5c294d56 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErofusRipperTest.java @@ -0,0 +1,13 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URL; + +import com.rarchives.ripme.ripper.rippers.ErofusRipper; + +public class CfakeRipperTest extends RippersTest { + public void testRip() throws IOException { + ErofusRipper ripper = new ErofusRipper(new URL("https://www.erofus.com/comics/be-story-club-comics/a-kiss/issue-1")); + testRipper(ripper); + } +} From 6d83756f9dced70963bbc8f662a708ab9602a49c Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 5 Apr 2019 21:27:54 -0400 Subject: [PATCH 6/7] Fixed typo --- .../rarchives/ripme/tst/ripper/rippers/ErofusRipperTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErofusRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErofusRipperTest.java index 5c294d56..bd210c13 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErofusRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErofusRipperTest.java @@ -5,7 +5,7 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.ErofusRipper; -public class CfakeRipperTest extends RippersTest { +public class ErofusRipperTest extends RippersTest { public void testRip() throws IOException { ErofusRipper ripper = new ErofusRipper(new URL("https://www.erofus.com/comics/be-story-club-comics/a-kiss/issue-1")); testRipper(ripper); From 510c5ee4412d9965155939ca5a1cfb1cd1562954 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Fri, 5 Apr 2019 21:30:38 -0400 Subject: [PATCH 7/7] Added a getGID unittest to the erofus unittest --- .../rarchives/ripme/tst/ripper/rippers/ErofusRipperTest.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErofusRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErofusRipperTest.java index bd210c13..ab497433 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErofusRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ErofusRipperTest.java @@ -10,4 +10,9 @@ public class ErofusRipperTest extends RippersTest { ErofusRipper ripper = new ErofusRipper(new URL("https://www.erofus.com/comics/be-story-club-comics/a-kiss/issue-1")); testRipper(ripper); } + + public void testGetGID() throws IOException { + ErofusRipper ripper = new ErofusRipper(new URL("https://www.erofus.com/comics/be-story-club-comics/a-kiss/issue-1")); + assertEquals("be-story-club-comics", ripper.getGID(new URL("https://www.erofus.com/comics/be-story-club-comics/a-kiss/issue-1"))); + } }