From 3414364ce6e90a299df2521b8ab91ffa7b4bc50d Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 13 Aug 2019 19:41:16 -0400 Subject: [PATCH 1/4] Fixed Xhamster ripper; Added support xhamster2.com and xhamster.desi --- .../ripme/ripper/rippers/XhamsterRipper.java | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java index 2eaa5c9e..9faf5c09 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java @@ -17,7 +17,7 @@ import com.rarchives.ripme.utils.Http; // WARNING // This ripper changes all requests to use the MOBILE version of the site -// If you're chaning anything be sure to use the mobile sites html/css or you\re just wasting your time! +// If you're changing anything be sure to use the mobile sites html/css or you're just wasting your time! // WARNING public class XhamsterRipper extends AbstractHTMLRipper { @@ -45,9 +45,9 @@ public class XhamsterRipper extends AbstractHTMLRipper { @Override public URL sanitizeURL(URL url) throws MalformedURLException { String URLToReturn = url.toExternalForm(); - URLToReturn = URLToReturn.replaceAll("xhamster.one", "xhamster.com"); - URLToReturn = URLToReturn.replaceAll("m.xhamster.com", "xhamster.com"); - URLToReturn = URLToReturn.replaceAll("\\w\\w.xhamster.com", "xhamster.com"); + URLToReturn = URLToReturn.replaceAll("xhamster\\.(com|one|desi)", "xhamster.com"); + URLToReturn = URLToReturn.replaceAll("m.xhamster\\.(com|one|desi)", "xhamster.com"); + URLToReturn = URLToReturn.replaceAll("\\w\\w\\.xhamster\\.(com|one|desi)", "xhamster.com"); if (!isVideoUrl(url)) { URLToReturn = URLToReturn.replaceAll("xhamster.com", "m.xhamster.com"); } @@ -114,17 +114,17 @@ public class XhamsterRipper extends AbstractHTMLRipper { @Override public boolean canRip(URL url) { - Pattern p = Pattern.compile("^https?://([\\w\\w]*\\.)?xhamster\\.(com|one)/photos/gallery/.*?(\\d+)$"); + Pattern p = Pattern.compile("^https?://([\\w\\w]*\\.)?xhamster2?\\.(com|one|desi)/photos/gallery/.*?(\\d+)$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { return true; } - p = Pattern.compile("^https?://[\\w\\w.]*xhamster\\.(com|one)/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); + p = Pattern.compile("^https?://[\\w\\w.]*xhamster2?\\.(com|one|desi)/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); m = p.matcher(url.toExternalForm()); if (m.matches()) { return true; } - p = Pattern.compile("^https?://.*xhamster\\.(com|one)/(movies|videos)/.*$"); + p = Pattern.compile("^https?://.*xhamster2?\\.(com|one|desi)/(movies|videos)/.*$"); m = p.matcher(url.toExternalForm()); if (m.matches()) { return true; @@ -133,7 +133,7 @@ public class XhamsterRipper extends AbstractHTMLRipper { } private boolean isVideoUrl(URL url) { - Pattern p = Pattern.compile("^https?://.*xhamster\\.com/(movies|videos)/.*$"); + Pattern p = Pattern.compile("^https?://.*xhamster2?\\.(com|one|desi)/(movies|videos)/.*$"); Matcher m = p.matcher(url.toExternalForm()); return m.matches(); } @@ -155,9 +155,16 @@ public class XhamsterRipper extends AbstractHTMLRipper { List result = new ArrayList<>(); if (!isVideoUrl(url)) { for (Element page : doc.select("div.items > div.item-container > a.item")) { + // Make sure we don't waste time running the loop if the ripper has been stopped + if (isStopped()) { + break; + } String pageWithImageUrl = page.attr("href"); try { - String image = Http.url(new URL(pageWithImageUrl)).get().select("div.picture_container > a > img").attr("src"); + // This works around some redirect fuckery xhamster likes to do where visiting m.xhamster.com sends to + // the page chamster.com but displays the mobile site from m.xhamster.com + pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster\\.", "://m.xhamster."); + String image = Http.url(new URL(pageWithImageUrl)).get().select("a > img#photoCurr").attr("src"); downloadFile(image); } catch (IOException e) { LOGGER.error("Was unable to load page " + pageWithImageUrl); From 6b8f379e9de701f2dd2dd0038dd0258ea568179f Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 13 Aug 2019 19:47:10 -0400 Subject: [PATCH 2/4] Added support for xhamster2.com domain --- .../rarchives/ripme/ripper/rippers/XhamsterRipper.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java index 9faf5c09..2dbcec2a 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java @@ -49,7 +49,7 @@ public class XhamsterRipper extends AbstractHTMLRipper { URLToReturn = URLToReturn.replaceAll("m.xhamster\\.(com|one|desi)", "xhamster.com"); URLToReturn = URLToReturn.replaceAll("\\w\\w\\.xhamster\\.(com|one|desi)", "xhamster.com"); if (!isVideoUrl(url)) { - URLToReturn = URLToReturn.replaceAll("xhamster.com", "m.xhamster.com"); + URLToReturn = URLToReturn.replaceAll("xhamster2?.com", "m.xhamster.com"); } URL san_url = new URL(URLToReturn); LOGGER.info("sanitized URL is " + san_url.toExternalForm()); @@ -58,17 +58,17 @@ public class XhamsterRipper extends AbstractHTMLRipper { @Override public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://[\\w\\w.]*xhamster\\.com/photos/gallery/.*?(\\d+)$"); + Pattern p = Pattern.compile("^https?://[\\w\\w.]*xhamster2?\\.com/photos/gallery/.*?(\\d+)$"); Matcher m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(1); } - p = Pattern.compile("^https?://[\\w\\w.]*xhamster\\.com/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); + p = Pattern.compile("^https?://[\\w\\w.]*xhamster2?\\.com/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); m = p.matcher(url.toExternalForm()); if (m.matches()) { return "user_" + m.group(1); } - p = Pattern.compile("^https?://.*xhamster\\.com/(movies|videos)/(.*)$"); + p = Pattern.compile("^https?://.*xhamster2?\\.com/(movies|videos)/(.*)$"); m = p.matcher(url.toExternalForm()); if (m.matches()) { return m.group(2); @@ -164,6 +164,7 @@ public class XhamsterRipper extends AbstractHTMLRipper { // This works around some redirect fuckery xhamster likes to do where visiting m.xhamster.com sends to // the page chamster.com but displays the mobile site from m.xhamster.com pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster\\.", "://m.xhamster."); + pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster2\\.", "://m.xhamster."); String image = Http.url(new URL(pageWithImageUrl)).get().select("a > img#photoCurr").attr("src"); downloadFile(image); } catch (IOException e) { From 6f6d1b9094486a12b4f35706e6a8343a523a1db0 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 13 Aug 2019 19:47:42 -0400 Subject: [PATCH 3/4] Marked tests as @Test and added tests for newly support domains --- .../ripper/rippers/XhamsterRipperTest.java | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java index 3ccccb49..96a9295e 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/XhamsterRipperTest.java @@ -5,24 +5,41 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.XhamsterRipper; import org.jsoup.nodes.Document; +import org.junit.jupiter.api.Test; + public class XhamsterRipperTest extends RippersTest { - + @Test public void testXhamsterAlbum1() throws IOException { XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.com/photos/gallery/sexy-preggo-girls-9026608")); testRipper(ripper); } - + @Test + public void testXhamster2Album() throws IOException { + XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster2.com/photos/gallery/sexy-preggo-girls-9026608")); + testRipper(ripper); + } + @Test public void testXhamsterAlbum2() throws IOException { XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.com/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664")); testRipper(ripper); } - + @Test + public void testXhamsterAlbumOneDomain() throws IOException { + XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.one/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664")); + testRipper(ripper); + } + @Test + public void testXhamsterAlbumDesiDomain() throws IOException { + XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.desi/photos/gallery/japanese-dolls-4-asahi-mizuno-7254664")); + testRipper(ripper); + } + @Test public void testXhamsterVideo() throws IOException { XhamsterRipper ripper = new XhamsterRipper(new URL("https://xhamster.com/videos/brazzers-busty-big-booty-milf-lisa-ann-fucks-her-masseur-1492828")); testRipper(ripper); } - + @Test public void testBrazilianXhamster() throws IOException { XhamsterRipper ripper = new XhamsterRipper(new URL("https://pt.xhamster.com/photos/gallery/silvana-7105696")); testRipper(ripper); @@ -33,7 +50,7 @@ public class XhamsterRipperTest extends RippersTest { XhamsterRipper ripper = new XhamsterRipper(url); assertEquals("7254664", ripper.getGID(url)); } - + @Test public void testGetNextPage() throws IOException { XhamsterRipper ripper = new XhamsterRipper(new URL("https://pt.xhamster.com/photos/gallery/mega-compil-6-10728626")); Document doc = ripper.getFirstPage(); From 1402f3b8dde95da49810f1794ed39b3bf1b77d8f Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 17 Aug 2019 20:04:14 -0400 Subject: [PATCH 4/4] xhamster no longer changes url domain when sanitizing --- .../ripme/ripper/rippers/XhamsterRipper.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java index 2dbcec2a..46ec7c18 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java @@ -44,13 +44,12 @@ public class XhamsterRipper extends AbstractHTMLRipper { @Override public URL sanitizeURL(URL url) throws MalformedURLException { - String URLToReturn = url.toExternalForm(); - URLToReturn = URLToReturn.replaceAll("xhamster\\.(com|one|desi)", "xhamster.com"); - URLToReturn = URLToReturn.replaceAll("m.xhamster\\.(com|one|desi)", "xhamster.com"); - URLToReturn = URLToReturn.replaceAll("\\w\\w\\.xhamster\\.(com|one|desi)", "xhamster.com"); - if (!isVideoUrl(url)) { - URLToReturn = URLToReturn.replaceAll("xhamster2?.com", "m.xhamster.com"); + if (isVideoUrl(url)) { + return url; } + String URLToReturn = url.toExternalForm(); + URLToReturn = URLToReturn.replaceAll("https?://\\w?\\w?\\.?xhamster\\.", "https://m.xhamster."); + URLToReturn = URLToReturn.replaceAll("https?://xhamster2\\.", "https://m.xhamster2."); URL san_url = new URL(URLToReturn); LOGGER.info("sanitized URL is " + san_url.toExternalForm()); return san_url;