From ba76d3d8b8dcc42d5a57a0c95b6e60158c5b8f62 Mon Sep 17 00:00:00 2001 From: metaprime Date: Sat, 4 Jan 2025 01:39:05 -0800 Subject: [PATCH] Implement FapDungeonRipper (Issue #1979) (#2056) --- .../ripper/rippers/FapDungeonRipper.java | 157 ++++++++++++++++++ .../ripper/rippers/FapDungeonRipperTest.java | 28 ++++ 2 files changed, 185 insertions(+) create mode 100644 src/main/java/com/rarchives/ripme/ripper/rippers/FapDungeonRipper.java create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/FapDungeonRipperTest.java diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/FapDungeonRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/FapDungeonRipper.java new file mode 100644 index 00000000..70f6322f --- /dev/null +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/FapDungeonRipper.java @@ -0,0 +1,157 @@ +package com.rarchives.ripme.ripper.rippers; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import com.rarchives.ripme.ripper.AbstractHTMLRipper; + +public class FapDungeonRipper extends AbstractHTMLRipper { + + private static final Logger logger = LogManager.getLogger(FapDungeonRipper.class); + + private static final String HOST = "fapdungeon"; + + private static final Pattern pagePattern = Pattern + .compile("^https?://[wm\\.]*fapdungeon\\.com/([a-zA-Z0-9_-]+)/(.+)/?$"); + + protected class FapDungeonDownloadItem { + public String url; + public String title; + + public FapDungeonDownloadItem(String url, String title) { + this.url = url; + this.title = title; + } + } + + public FapDungeonRipper(URL url) throws IOException { + super(url); + } + + @Override + public String getHost() { + return HOST; + } + + @Override + public String getDomain() { + return HOST + ".com"; + } + + @Override + public boolean canRip(URL url) { + Matcher m = pagePattern.matcher(url.toExternalForm()); + if (m.matches()) { + return true; + } + return false; + } + + @Override + public String getGID(URL url) throws MalformedURLException { + Matcher m = pagePattern.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } + + throw new MalformedURLException( + "Expected fapdungeon format:" + + "fapdungeon.com/category/albumname/" + + " Got: " + url); + } + + public String returnLargestImgUrlFromSrcAndSrcset(String src, String sourceSet) { + String[] parts = sourceSet.split(","); + logger.info("While ripping url: " + this.url + " img src: " + src + " has sourceSet: " + sourceSet); + + Optional largestImgUrl = Optional.empty(); + int maxWidthSoFar = 0; + + for (String part : parts) { + String[] subParts = part.strip().split(" "); + if (subParts.length == 2) { + String imgUrlPart = subParts[0].strip(); + + // parse the integer out of values like "1080w" + String widthStringPart = subParts[1].strip(); + String widthNumberString = widthStringPart.substring(0, widthStringPart.length() - 1); + int width = Integer.parseInt(widthNumberString); + + if (width > maxWidthSoFar) { + logger.info("Found larger image: " + part); + largestImgUrl = Optional.of(imgUrlPart); + maxWidthSoFar = width; + } + } + } + + if (largestImgUrl.isPresent()) { + String imgUrl = largestImgUrl.get(); + logger.info("For img src " + src + " with srcset, using largestImgUrl: " + imgUrl); + return imgUrl; + } else { + logger.info("Using img src: " + src); + return src; + } + } + + @Override + public List getURLsFromPage(Document doc) { + List results = new ArrayList<>(); + + Matcher m = pagePattern.matcher(url.toExternalForm()); + if (m.matches()) { + Element content = doc.select("div.entry-content").get(0); + + // Debug this selector on the page itself in Dev Tools with + // Array.from(document.querySelectorAll("div.entry-content img")).map((x) => + // x['srcset']) + Elements pictures = content.select("img"); + for (Element e : pictures) { + String imageSrc = e.attr("src"); // fallback on value + String imageSourceSet = e.attr("srcset"); // get the largest resolution from this srcset + String imageUrl = returnLargestImgUrlFromSrcAndSrcset(imageSrc, imageSourceSet); + results.add(imageUrl); + } + + // Debug this selector on the page itself in Dev Tools with + // Array.from(document.querySelectorAll("div.entry-content video + // source")).map((x) => x['src']) + Elements videos = content.select("video source"); + for (Element e : videos) { + results.add(e.attr("src")); + } + } + + return results; + } + + @Override + public void downloadURL(URL url, int index) { + sleep(1000); + addURLToDownload(url, getPrefix(index)); + } + + @Override + public String getAlbumTitle(URL url) throws MalformedURLException, URISyntaxException { + Matcher m = pagePattern.matcher(url.toExternalForm()); + if (m.matches()) { + return getHost() + "_" + m.group(1) + "_" + m.group(2); + } else { + return super.getAlbumTitle(url); + } + } +} diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FapDungeonRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FapDungeonRipperTest.java new file mode 100644 index 00000000..e75b0b75 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/FapDungeonRipperTest.java @@ -0,0 +1,28 @@ +package com.rarchives.ripme.tst.ripper.rippers; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; + +import com.rarchives.ripme.ripper.rippers.FapDungeonRipper; +import org.junit.jupiter.api.Test; + +public class FapDungeonRipperTest extends RippersTest { + @Test + public void testFapDungeon1() throws IOException, URISyntaxException { + FapDungeonRipper ripper = new FapDungeonRipper(new URI("https://fapdungeon.com/white/thegorillagrip-busty-cutie-onlyfans-nudes/").toURL()); + testRipper(ripper); + } + + @Test + public void testFapDungeon2() throws IOException, URISyntaxException { + FapDungeonRipper ripper = new FapDungeonRipper(new URI("https://fapdungeon.com/asian/joythailia-sexy-asian-petite-onlyfans-nudes/").toURL()); + testRipper(ripper); + } + + @Test + public void testFapDungeon3() throws IOException, URISyntaxException { + FapDungeonRipper ripper = new FapDungeonRipper(new URI("https://fapdungeon.com/black/jaaden-kyrelle-busty-ebony-onlyfans-sextapes-nudes/").toURL()); + testRipper(ripper); + } +}