diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java index 9d6152b9..17105ee4 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedgifsRipper.java @@ -19,6 +19,7 @@ import java.util.regex.Pattern; public class RedgifsRipper extends AbstractHTMLRipper { private static final String HOST = "redgifs.com"; + private static final String HOST_2 = "gifdeliverynetwork.com"; String username = ""; String cursor = ""; String count = "100"; @@ -41,7 +42,7 @@ public class RedgifsRipper extends AbstractHTMLRipper { @Override public boolean canRip(URL url) { - return url.getHost().endsWith(HOST); + return url.getHost().endsWith(HOST) || url.getHost().endsWith(HOST_2); } @Override @@ -49,6 +50,7 @@ public class RedgifsRipper extends AbstractHTMLRipper { String sUrl = url.toExternalForm(); sUrl = sUrl.replace("/gifs/detail", ""); sUrl = sUrl.replace("/amp", ""); + sUrl = sUrl.replace("gifdeliverynetwork.com", "redgifs.com/watch"); return new URL(sUrl); } @@ -74,10 +76,10 @@ public class RedgifsRipper extends AbstractHTMLRipper { } else if (isSearch().matches()) { searchText = getGID(url).replace("-", " "); return Http.url( - new URL("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText + "&count=" + searchCount + "&start=" + searchStart*searchCount)).ignoreContentType().get(); + new URL("https://napi.redgifs.com/v1/gfycats/search?search_text=" + searchText + "&count=" + searchCount + "&start=" + searchStart*searchCount)).ignoreContentType().get(); } else { username = getGID(url); - return Http.url(new URL("https://api.redgifs.com/v1/users/" + username + "/gfycats?count=" + count)) + return Http.url(new URL("https://napi.redgifs.com/v1/users/" + username + "/gfycats?count=" + count)) .ignoreContentType().get(); } } @@ -122,16 +124,18 @@ public class RedgifsRipper extends AbstractHTMLRipper { @Override public Document getNextPage(Document doc) throws IOException { - if (!isProfile().matches()) { - return Http.url( - new URL("https://api.redgifs.com/v1/gfycats/search?search_text=" + searchText + if (isSearch().matches()) { + Document d = Http.url( + new URL("https://napi.redgifs.com/v1/gfycats/search?search_text=" + searchText + "&count=" + searchCount + "&start=" + searchCount*++searchStart)) .ignoreContentType().get(); + return (hasURLs(d).isEmpty()) ? null : d; } else { if (cursor.equals("")) { - throw new IOException("No more pages"); + return null; } else { - return Http.url(new URL("https://api.redgifs.com/v1/users/" + username + "/gfycats?count=" + count + "&cursor=" + cursor)).ignoreContentType().get(); + Document d = Http.url(new URL("https://napi.redgifs.com/v1/users/" + username + "/gfycats?count=" + count + "&cursor=" + cursor)).ignoreContentType().get(); + return (hasURLs(d).isEmpty()) ? null : d; } } } @@ -140,12 +144,7 @@ public class RedgifsRipper extends AbstractHTMLRipper { public List getURLsFromPage(Document doc) { List result = new ArrayList<>(); if (isProfile().matches() || isSearch().matches()) { - JSONObject page = new JSONObject(stripHTMLTags(doc.html())); - JSONArray content = page.getJSONArray("gfycats"); - for (int i = 0; i < content.length(); i++) { - result.add(content.getJSONObject(i).getString("mp4Url")); - } - cursor = page.getString("cursor"); + result = hasURLs(doc); } else { Elements videos = doc.select("script"); for (Element el : videos) { @@ -159,6 +158,22 @@ public class RedgifsRipper extends AbstractHTMLRipper { return result; } + /** + * Helper method for retrieving URLs. + * @param doc Document of the URL page to look through + * @return List of URLs to download + */ + public List hasURLs(Document doc) { + List result = new ArrayList<>(); + JSONObject page = new JSONObject(stripHTMLTags(doc.html())); + JSONArray content = page.getJSONArray("gfycats"); + for (int i = 0; i < content.length(); i++) { + result.add(content.getJSONObject(i).getString("mp4Url")); + } + cursor = page.getString("cursor"); + return result; + } + /** * Helper method for retrieving video URLs. * @param url URL to gfycat page diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java index 03a480cf..3fcb71c2 100644 --- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java +++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java @@ -11,6 +11,7 @@ import com.rarchives.ripme.ripper.AbstractRipper; import com.rarchives.ripme.ripper.rippers.EroShareRipper; import com.rarchives.ripme.ripper.rippers.EromeRipper; import com.rarchives.ripme.ripper.rippers.ImgurRipper; +import com.rarchives.ripme.ripper.rippers.RedgifsRipper; import com.rarchives.ripme.ripper.rippers.VidbleRipper; import com.rarchives.ripme.ripper.rippers.GfycatRipper; import org.apache.commons.lang.math.NumberUtils; @@ -76,6 +77,18 @@ public class RipUtils { } return result; } + else if (url.getHost().endsWith("redgifs.com") || url.getHost().endsWith("gifdeliverynetwork.com")) { + try { + logger.debug("Fetching redgifs page " + url); + String videoURL = RedgifsRipper.getVideoURL(url); + logger.debug("Got redgifs URL: " + videoURL); + result.add(new URL(videoURL)); + } catch (IOException e) { + // Do nothing + logger.warn("Exception while retrieving redgifs page:", e); + } + return result; + } else if (url.toExternalForm().contains("vidble.com/album/") || url.toExternalForm().contains("vidble.com/show/")) { try { logger.info("Getting vidble album " + url); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java index 46624303..922dbaf8 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedgifsRipperTest.java @@ -1,6 +1,5 @@ package com.rarchives.ripme.tst.ripper.rippers; -import com.rarchives.ripme.ripper.rippers.GfycatRipper; import com.rarchives.ripme.ripper.rippers.RedgifsRipper; import org.jsoup.nodes.Document; import org.junit.jupiter.api.*; @@ -16,10 +15,19 @@ public class RedgifsRipperTest extends RippersTest { */ @Test public void testRedgifsGoodURL() throws IOException{ - RedgifsRipper ripper = new RedgifsRipper(new URL("https://redgifs.com/watch/blaringbonyfulmar-panty-peel")); + RedgifsRipper ripper = new RedgifsRipper(new URL("https://www.redgifs.com/watch/talkativewarpeddragon-petite")); testRipper(ripper); } + /** + * Rips gifdeliverynetwork URL's by redirecting them to proper redgifs url + * @throws IOException + */ + @Test + public void testRedgifsBadRL() throws IOException{ + RedgifsRipper ripper = new RedgifsRipper(new URL("https://www.gifdeliverynetwork.com/foolishelasticchimpanzee")); + testRipper(ripper); + } /** * Rips a Redifs profile @@ -41,8 +49,8 @@ public class RedgifsRipperTest extends RippersTest { Document doc = ripper.getFirstPage(); doc = ripper.getNextPage(doc); - assertTrue("https://api.redgifs.com/v1/gfycats/search?search_text=little%20caprice&count=150&start=150".equalsIgnoreCase(doc.location())); + assertTrue("https://napi.redgifs.com/v1/gfycats/search?search_text=little%20caprice&count=150&start=150".equalsIgnoreCase(doc.location())); doc = ripper.getNextPage(doc); - assertTrue("https://api.redgifs.com/v1/gfycats/search?search_text=little%20caprice&count=150&start=300".equalsIgnoreCase(doc.location())); + assertTrue("https://napi.redgifs.com/v1/gfycats/search?search_text=little%20caprice&count=150&start=300".equalsIgnoreCase(doc.location())); } }