From d26b73cdecae8efd2a323c352089941f2378a582 Mon Sep 17 00:00:00 2001 From: Frank Castle Date: Sat, 28 Sep 2019 14:21:32 +0530 Subject: [PATCH 1/2] Fixed xhamster ripper not downloading after 30 images --- .../rarchives/ripme/ripper/rippers/XhamsterRipper.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java index 46ec7c18..7ae570f3 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java @@ -140,8 +140,11 @@ public class XhamsterRipper extends AbstractHTMLRipper { @Override public Document getNextPage(Document doc) throws IOException { if (doc.select("a[data-page=next]").first() != null) { - if (doc.select("a[data-page=next]").first().attr("href").startsWith("http")) { - return Http.url(doc.select("a[data-page=next]").first().attr("href")).get(); + String nextPageUrl = doc.select("a[data-page=next]").first().attr("href"); + if (nextPageUrl.startsWith("http")) { + nextPageUrl = nextPageUrl.replaceAll("https?://\\w?\\w?\\.?xhamster\\.", "https://m.xhamster."); + nextPageUrl = nextPageUrl.replaceAll("https?://xhamster2\\.", "https://m.xhamster2."); + return Http.url(nextPageUrl).get(); } } throw new IOException("No more pages"); @@ -153,7 +156,7 @@ public class XhamsterRipper extends AbstractHTMLRipper { LOGGER.debug("Checking for urls"); List result = new ArrayList<>(); if (!isVideoUrl(url)) { - for (Element page : doc.select("div.items > div.item-container > a.item")) { + for (Element page : doc.select("div.picture_view > div.pictures_block > div.items > div.item-container > a.item")) { // Make sure we don't waste time running the loop if the ripper has been stopped if (isStopped()) { break; From 65ff5274faef2e886c71404dde2276ea73269717 Mon Sep 17 00:00:00 2001 From: Frank Castle Date: Sun, 29 Sep 2019 16:27:06 +0530 Subject: [PATCH 2/2] Added xvideos ripper album downloading --- .../ripme/ripper/rippers/XvideosRipper.java | 55 +++++++++++++++---- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java index ff38b347..0fdef868 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XvideosRipper.java @@ -43,7 +43,15 @@ public class XvideosRipper extends AbstractSingleFileRipper { public boolean canRip(URL url) { Pattern p = Pattern.compile("^https?://[wm.]*xvideos\\.com/video[0-9]+.*$"); Matcher m = p.matcher(url.toExternalForm()); - return m.matches(); + if (m.matches()) { + return true; + } + p = Pattern.compile("^https?://[wm.]*xvideos\\.com/profiles/[a-zA-Z0-9_-]+/photos/\\d+/[a-zA-Z0-9_-]+$"); + m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return true; + } + return false; } @Override @@ -53,6 +61,11 @@ public class XvideosRipper extends AbstractSingleFileRipper { if (m.matches()) { return m.group(1); } + p = Pattern.compile("^https?://[wm.]*xvideos\\.com/profiles/[a-zA-Z0-9_-]+/photos/(\\d+)/[a-zA-Z0-9_-]+$"); + m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return m.group(1); + } throw new MalformedURLException( "Expected xvideo format:" @@ -63,18 +76,29 @@ public class XvideosRipper extends AbstractSingleFileRipper { @Override public List getURLsFromPage(Document doc) { List results = new ArrayList<>(); - Elements scripts = doc.select("script"); - for (Element e : scripts) { - if (e.html().contains("html5player.setVideoUrlHigh")) { - LOGGER.info("Found the right script"); - String[] lines = e.html().split("\n"); - for (String line: lines) { - if (line.contains("html5player.setVideoUrlHigh")) { - String videoURL = line.replaceAll("\t", "").replaceAll("html5player.setVideoUrlHigh\\(", "").replaceAll("\'", "").replaceAll("\\);", ""); - results.add(videoURL); + Pattern p = Pattern.compile("^https?://[wm.]*xvideos\\.com/video([0-9]+).*$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + Elements scripts = doc.select("script"); + for (Element e : scripts) { + if (e.html().contains("html5player.setVideoUrlHigh")) { + LOGGER.info("Found the right script"); + String[] lines = e.html().split("\n"); + for (String line : lines) { + if (line.contains("html5player.setVideoUrlHigh")) { + String videoURL = line.replaceAll("\t", "").replaceAll("html5player.setVideoUrlHigh\\(", "").replaceAll("\'", "").replaceAll("\\);", ""); + results.add(videoURL); + } } } } + } else { + for (Element e : doc.select("div.thumb > a")) { + results.add(e.attr("href")); + if (isThisATest()) { + break; + } + } } return results; } @@ -83,4 +107,15 @@ public class XvideosRipper extends AbstractSingleFileRipper { public void downloadURL(URL url, int index) { addURLToDownload(url, getPrefix(index)); } + + @Override + public String getAlbumTitle(URL url) throws MalformedURLException { + Pattern p = Pattern.compile("^https?://[wm.]*xvideos\\.com/profiles/([a-zA-Z0-9_-]+)/photos/(\\d+)/([a-zA-Z0-9_-]+)$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return getHost() + "_" + m.group(1) + "_" + m.group(3) + "_" + m.group(2); + } else { + return super.getAlbumTitle(url); + } + } } \ No newline at end of file