mirror/ripme
1
0
Fork 0
mirror of https://github.com/RipMeApp/ripme.git synced 2025-08-25 23:06:22 +02:00
Code Issues Releases Wiki Activity

RedgifsRipper: Fixes to redgifs ripper

Browse Source 
- Changed redgifs api URL to reflect updated change
- Added check in getNextPage to see if page has any URLs else finishes
  rather than throwing error
- Added gifdeliverynetwork.com as knwon host as it redirects to redgifs
  page
- Added reddit compatibility by adding as known host in RipUtils reddit
  ripper
- Unit test to cover above changes
This commit is contained in:
borderline232
2020-06-07 00:07:09 -04:00
parent 9a013c74ee
commit 689968948c
3 changed files with 54 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java
View File

@@ -19,6 +19,7 @@ import java.util.regex.Pattern;
public class RedgifsRipper extends AbstractHTMLRipper {
private static final String HOST = "redgifs.com";
private static final String HOST_2 = "gifdeliverynetwork.com";
String username = "";
String cursor = "";
String count = "100";
@@ -41,7 +42,7 @@ public class RedgifsRipper extends AbstractHTMLRipper {
@Override
public boolean canRip(URL url) {
return url.getHost().endsWith(HOST);
return url.getHost().endsWith(HOST) || url.getHost().endsWith(HOST_2);
}
@Override
@@ -49,6 +50,7 @@ public class RedgifsRipper extends AbstractHTMLRipper {
String sUrl = url.toExternalForm();
sUrl = sUrl.replace("/gifs/detail", "");
sUrl = sUrl.replace("/amp", "");
sUrl = sUrl.replace("gifdeliverynetwork.com", "redgifs.com/watch");
return new URL(sUrl);
}
@@ -74,10 +76,10 @@ public class RedgifsRipper extends AbstractHTMLRipper {
} else if (isSearch().matches()) {
searchText = getGID(url).replace("-", " ");
return Http.url(
new URL("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText + "&count=" + searchCount + "&start=" + searchStart*searchCount)).ignoreContentType().get();
new URL("https://napi.redgifs.com/v1/gfycats/search?search_text=" + searchText + "&count=" + searchCount + "&start=" + searchStart*searchCount)).ignoreContentType().get();
} else {
username = getGID(url);
return Http.url(new URL("https://api.redgifs.com/v1/users/" + username + "/gfycats?count=" + count))
return Http.url(new URL("https://napi.redgifs.com/v1/users/" + username + "/gfycats?count=" + count))
.ignoreContentType().get();
}
}
@@ -122,16 +124,18 @@ public class RedgifsRipper extends AbstractHTMLRipper {
@Override
public Document getNextPage(Document doc) throws IOException {
if (!isProfile().matches()) {
return Http.url(
new URL("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText
if (isSearch().matches()) {
Document d = Http.url(
new URL("https://napi.redgifs.com/v1/gfycats/search?search_text=" + searchText
+ "&count=" + searchCount + "&start=" + searchCount*++searchStart))
.ignoreContentType().get();
return (hasURLs(d).isEmpty()) ? null : d;
} else {
if (cursor.equals("")) {
throw new IOException("No more pages");
return null;
} else {
return Http.url(new URL("https://api.redgifs.com/v1/users/" + username + "/gfycats?count=" + count + "&cursor=" + cursor)).ignoreContentType().get();
Document d = Http.url(new URL("https://napi.redgifs.com/v1/users/" + username + "/gfycats?count=" + count + "&cursor=" + cursor)).ignoreContentType().get();
return (hasURLs(d).isEmpty()) ? null : d;
}
}
}
@@ -140,12 +144,7 @@ public class RedgifsRipper extends AbstractHTMLRipper {
public List<String> getURLsFromPage(Document doc) {
List<String> result = new ArrayList<>();
if (isProfile().matches() || isSearch().matches()) {
JSONObject page = new JSONObject(stripHTMLTags(doc.html()));
JSONArray content = page.getJSONArray("gfycats");
for (int i = 0; i < content.length(); i++) {
result.add(content.getJSONObject(i).getString("mp4Url"));
}
cursor = page.getString("cursor");
result = hasURLs(doc);
} else {
Elements videos = doc.select("script");
for (Element el : videos) {
@@ -159,6 +158,22 @@ public class RedgifsRipper extends AbstractHTMLRipper {
return result;
}
/**
* Helper method for retrieving URLs.
* @param doc Document of the URL page to look through
* @return List of URLs to download
*/
public List<String> hasURLs(Document doc) {
List<String> result = new ArrayList<>();
JSONObject page = new JSONObject(stripHTMLTags(doc.html()));
JSONArray content = page.getJSONArray("gfycats");
for (int i = 0; i < content.length(); i++) {
result.add(content.getJSONObject(i).getString("mp4Url"));
}
cursor = page.getString("cursor");
return result;
}
/**
* Helper method for retrieving video URLs.
* @param url URL to gfycat page

13
src/main/java/com/rarchives/ripme/utils/RipUtils.java
View File

@@ -11,6 +11,7 @@ import com.rarchives.ripme.ripper.AbstractRipper;
import com.rarchives.ripme.ripper.rippers.EroShareRipper;
import com.rarchives.ripme.ripper.rippers.EromeRipper;
import com.rarchives.ripme.ripper.rippers.ImgurRipper;
import com.rarchives.ripme.ripper.rippers.RedgifsRipper;
import com.rarchives.ripme.ripper.rippers.VidbleRipper;
import com.rarchives.ripme.ripper.rippers.GfycatRipper;
import org.apache.commons.lang.math.NumberUtils;
@@ -76,6 +77,18 @@ public class RipUtils {
}
return result;
}
else if (url.getHost().endsWith("redgifs.com") || url.getHost().endsWith("gifdeliverynetwork.com")) {
try {
logger.debug("Fetching redgifs page " + url);
String videoURL = RedgifsRipper.getVideoURL(url);
logger.debug("Got redgifs URL: " + videoURL);
result.add(new URL(videoURL));
} catch (IOException e) {
// Do nothing
logger.warn("Exception while retrieving redgifs page:", e);
}
return result;
}
else if (url.toExternalForm().contains("vidble.com/album/") || url.toExternalForm().contains("vidble.com/show/")) {
try {
logger.info("Getting vidble album " + url);

16
src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java
View File

@@ -1,6 +1,5 @@
package com.rarchives.ripme.tst.ripper.rippers;
import com.rarchives.ripme.ripper.rippers.GfycatRipper;
import com.rarchives.ripme.ripper.rippers.RedgifsRipper;
import org.jsoup.nodes.Document;
import org.junit.jupiter.api.*;
@@ -16,10 +15,19 @@ public class RedgifsRipperTest extends RippersTest {
*/
@Test
public void testRedgifsGoodURL() throws IOException{
RedgifsRipper ripper = new RedgifsRipper(new URL("https://redgifs.com/watch/blaringbonyfulmar-panty-peel"));
RedgifsRipper ripper = new RedgifsRipper(new URL("https://www.redgifs.com/watch/talkativewarpeddragon-petite"));
testRipper(ripper);
}
/**
* Rips gifdeliverynetwork URL's by redirecting them to proper redgifs url
* @throws IOException
*/
@Test
public void testRedgifsBadRL() throws IOException{
RedgifsRipper ripper = new RedgifsRipper(new URL("https://www.gifdeliverynetwork.com/foolishelasticchimpanzee"));
testRipper(ripper);
}
/**
* Rips a Redifs profile
@@ -41,8 +49,8 @@ public class RedgifsRipperTest extends RippersTest {
Document doc = ripper.getFirstPage();
doc = ripper.getNextPage(doc);
assertTrue("https://api.redgifs.com/v1/gfycats/search?search_text=little%20caprice&count=150&start=150".equalsIgnoreCase(doc.location()));
assertTrue("https://napi.redgifs.com/v1/gfycats/search?search_text=little%20caprice&count=150&start=150".equalsIgnoreCase(doc.location()));
doc = ripper.getNextPage(doc);
assertTrue("https://api.redgifs.com/v1/gfycats/search?search_text=little%20caprice&count=150&start=300".equalsIgnoreCase(doc.location()));
assertTrue("https://napi.redgifs.com/v1/gfycats/search?search_text=little%20caprice&count=150&start=300".equalsIgnoreCase(doc.location()));
}
}