mirror/ripme
1
0
Fork 0
mirror of https://github.com/RipMeApp/ripme.git synced 2025-08-21 05:01:30 +02:00
Code Issues Releases Wiki Activity

Fix #1837 (Xhamster Junit tests)

Browse Source 
+ added support for domain endings "one" and "desi"
This commit is contained in:
saladzic
2021-02-12 22:07:36 +01:00
parent b94f64652d
commit 8a86139759
1 changed files with 46 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
src/main/java/com/rarchives/ripme/ripper/rippers/XhamsterRipper.java
View File

@@ -57,23 +57,23 @@ public class XhamsterRipper extends AbstractHTMLRipper {
@Override @Override
public String getGID(URL url) throws MalformedURLException { public String getGID(URL url) throws MalformedURLException {
Pattern p = Pattern.compile("^https?://[\\w\\w.]*xhamster2?\\.com/photos/gallery/.*?(\\d+)$"); Pattern p = Pattern.compile("^https?://[\\w\\w.]*xhamster2?\\.(com|one|desi)/photos/gallery/.*?(\\d+)$");
Matcher m = p.matcher(url.toExternalForm()); Matcher m = p.matcher(url.toExternalForm());
if (m.matches()) { if (m.matches()) {
return m.group(1); return m.group(1);
} }
p = Pattern.compile("^https?://[\\w\\w.]*xhamster2?\\.com/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); p = Pattern.compile("^https?://[\\w\\w.]*xhamster2?\\.(com|one|desi)/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?");
m = p.matcher(url.toExternalForm()); m = p.matcher(url.toExternalForm());
if (m.matches()) { if (m.matches()) {
return "user_" + m.group(1); return "user_" + m.group(1);
} }
p = Pattern.compile("^https?://.*xhamster2?\\.com/(movies|videos)/(.*)$"); p = Pattern.compile("^https?://.*xhamster2?\\.(com|one|desi)/(movies|videos)/(.*)$");
m = p.matcher(url.toExternalForm()); m = p.matcher(url.toExternalForm());
if (m.matches()) { if (m.matches()) {
return m.group(2); return m.group(2);
} }
throw new MalformedURLException( throw new MalformedURLException(
"Expected xhamster.com gallery formats: " "Expected xhamster.com gallery formats: "
+ "xhamster.com/photos/gallery/xxxxx-#####" + "xhamster.com/photos/gallery/xxxxx-#####"
+ " Got: " + url); + " Got: " + url);
@@ -97,7 +97,7 @@ public class XhamsterRipper extends AbstractHTMLRipper {
@Override @Override
public boolean pageContainsAlbums(URL url) { public boolean pageContainsAlbums(URL url) {
Pattern p = Pattern.compile("^https?://[\\w\\w.]*xhamster\\.com/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?"); Pattern p = Pattern.compile("^https?://[\\w\\w.]*xhamster2?\\.(com|one|desi)/users/([a-zA-Z0-9_-]+)/(photos|videos)(/\\d+)?");
Matcher m = p.matcher(url.toExternalForm()); Matcher m = p.matcher(url.toExternalForm());
LOGGER.info("Checking if page has albums"); LOGGER.info("Checking if page has albums");
LOGGER.info(m.matches()); LOGGER.info(m.matches());
@@ -139,8 +139,9 @@ public class XhamsterRipper extends AbstractHTMLRipper {
@Override @Override
public Document getNextPage(Document doc) throws IOException { public Document getNextPage(Document doc) throws IOException {
if (doc.select("a[data-page=next]").first() != null) { if (doc.select("a.prev-next-list-link").first() != null) {
String nextPageUrl = doc.select("a[data-page=next]").first().attr("href"); String nextPageUrl = doc.select("a.prev-next-list-link").first().attr("href");
System.out.println(nextPageUrl);
if (nextPageUrl.startsWith("http")) { if (nextPageUrl.startsWith("http")) {
nextPageUrl = nextPageUrl.replaceAll("https?://\\w?\\w?\\.?xhamster\\.", "https://m.xhamster."); nextPageUrl = nextPageUrl.replaceAll("https?://\\w?\\w?\\.?xhamster\\.", "https://m.xhamster.");
nextPageUrl = nextPageUrl.replaceAll("https?://xhamster2\\.", "https://m.xhamster2."); nextPageUrl = nextPageUrl.replaceAll("https?://xhamster2\\.", "https://m.xhamster2.");
@@ -156,25 +157,45 @@ public class XhamsterRipper extends AbstractHTMLRipper {
LOGGER.debug("Checking for urls"); LOGGER.debug("Checking for urls");
List<String> result = new ArrayList<>(); List<String> result = new ArrayList<>();
if (!isVideoUrl(url)) { if (!isVideoUrl(url)) {
for (Element page : doc.select("div.picture_view > div.pictures_block > div.items > div.item-container > a.item")) { if (!doc.select("div.picture_view > div.pictures_block > div.items > div.item-container > a.item").isEmpty()) {
// Make sure we don't waste time running the loop if the ripper has been stopped // Old HTML structure is still present at some places
if (isStopped()) { for (Element page : doc.select("div.picture_view > div.pictures_block > div.items > div.item-container > a.item")) {
break; // Make sure we don't waste time running the loop if the ripper has been stopped
} if (isStopped()) {
String pageWithImageUrl = page.attr("href"); break;
try { }
// This works around some redirect fuckery xhamster likes to do where visiting m.xhamster.com sends to String pageWithImageUrl = page.attr("href");
// the page chamster.com but displays the mobile site from m.xhamster.com try {
pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster\\.", "://m.xhamster."); // This works around some redirect fuckery xhamster likes to do where visiting m.xhamster.com sends to
pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster2\\.", "://m.xhamster."); // the page chamster.com but displays the mobile site from m.xhamster.com
String image = Http.url(new URL(pageWithImageUrl)).get().select("a > img#photoCurr").attr("src"); pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster\\.", "://m.xhamster.");
downloadFile(image); pageWithImageUrl = pageWithImageUrl.replaceAll("://xhamster2\\.", "://m.xhamster.");
} catch (IOException e) { String image = Http.url(new URL(pageWithImageUrl)).get().select("a > img#photoCurr").attr("src");
LOGGER.error("Was unable to load page " + pageWithImageUrl); result.add(image);
} downloadFile(image);
} } catch (IOException e) {
LOGGER.error("Was unable to load page " + pageWithImageUrl);
}
}
} else {
// New HTML structure
for (Element page : doc.select("div#photo-slider > div#photo_slider > a")) {
// Make sure we don't waste time running the loop if the ripper has been stopped
if (isStopped()) {
break;
}
String image = page.attr("href");
// This works around some redirect fuckery xhamster likes to do where visiting m.xhamster.com sends to
// the page chamster.com but displays the mobile site from m.xhamster.com
image = image.replaceAll("://xhamster\\.", "://m.xhamster.");
image = image.replaceAll("://xhamster2\\.", "://m.xhamster.");
result.add(image);
downloadFile(image);
}
}
} else { } else {
String imgUrl = doc.select("div.player-container > a").attr("href"); String imgUrl = doc.select("div.player-container > a").attr("href");
result.add(imgUrl);
downloadFile(imgUrl); downloadFile(imgUrl);
} }
return result; return result;