diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java deleted file mode 100644 index c542c6dc..00000000 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/GfycatRipper.java +++ /dev/null @@ -1,160 +0,0 @@ -package com.rarchives.ripme.ripper.rippers; - - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import com.rarchives.ripme.ripper.AbstractHTMLRipper; -import org.json.JSONArray; -import org.json.JSONObject; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -import com.rarchives.ripme.utils.Http; - - -public class GfycatRipper extends AbstractHTMLRipper { - - private static final String HOST = "gfycat.com"; - String username = ""; - String cursor = ""; - String count = "30"; - String REFERRER = "www.reddit.com"; - - - - public GfycatRipper(URL url) throws IOException { - super(new URL(url.toExternalForm().split("-")[0].replace("thumbs.", ""))); - } - - @Override - public String getDomain() { - return "gfycat.com"; - } - - @Override - public String getHost() { - return "gfycat"; - } - - @Override - public boolean canRip(URL url) { - return url.getHost().endsWith(HOST); - } - - @Override - public URL sanitizeURL(URL url) throws MalformedURLException { - String sUrl = url.toExternalForm(); - sUrl = sUrl.replace("/gifs/detail", ""); - sUrl = sUrl.replace("/amp", ""); - return new URL(sUrl); - } - - public boolean isProfile() { - Pattern p = Pattern.compile("^https?://[wm.]*gfycat\\.com/@([a-zA-Z0-9\\.\\-\\_]+).*$"); - Matcher m = p.matcher(url.toExternalForm()); - return m.matches(); - } - - @Override - public Document getFirstPage() throws IOException { - if (!isProfile()) { - return Http.url(url).referrer(REFERRER).get(); - } else { - username = getGID(url); - return Http.url(new URL("https://api.gfycat.com/v1/users/" + username + "/gfycats")).referrer((REFERRER)).ignoreContentType().get(); - } - } - - @Override - public void downloadURL(URL url, int index) { - addURLToDownload(url, getPrefix(index)); - } - - @Override - public String getGID(URL url) throws MalformedURLException { - Pattern p = Pattern.compile("^https?://(?:thumbs\\.|[wm\\.]*)gfycat\\.com/@?([a-zA-Z0-9\\.\\-\\_]+).*$"); - Matcher m = p.matcher(url.toExternalForm()); - - if (m.matches()) - return m.group(1); - - throw new MalformedURLException( - "Expected gfycat.com format: " - + "gfycat.com/id or " - + "thumbs.gfycat.com/id.gif" - + " Got: " + url); - } - - private String stripHTMLTags(String t) { - t = t.replaceAll("\n" + - " \n" + - " ", ""); - t = t.replaceAll("\n" + - "", ""); - t = t.replaceAll("\n", ""); - t = t.replaceAll("=\"\"", ""); - return t; - } - - @Override - public Document getNextPage(Document doc) throws IOException { - if (cursor.equals("")) { - throw new IOException("No more pages"); - } - return Http.url(new URL("https://api.gfycat.com/v1/users/" + username + "/gfycats?count=" + count + "&cursor=" + cursor)).ignoreContentType().get(); - } - - @Override - public List getURLsFromPage(Document doc) { - List result = new ArrayList<>(); - if (isProfile()) { - JSONObject page = new JSONObject(stripHTMLTags(doc.html())); - JSONArray content = page.getJSONArray("gfycats"); - for (int i = 0; i < content.length(); i++) { - result.add(content.getJSONObject(i).getString("mp4Url")); - } - cursor = page.getString("cursor"); - } else { - Elements videos = doc.select("script"); - for (Element el : videos) { - String json = el.html(); - if (json.startsWith("{")) { - JSONObject page = new JSONObject(json); - result.add(page.getJSONObject("video").getString("contentUrl")); - } - } - } - return result; - } - - /** - * Helper method for retrieving video URLs. - * @param url URL to gfycat page - * @return URL to video - * @throws IOException - */ - public static String getVideoURL(URL url) throws IOException { - LOGGER.info("Retrieving " + url.toExternalForm()); - - //Sanitize the URL first - url = new URL(url.toExternalForm().replace("/gifs/detail", "")); - - Document doc = Http.url(url).get(); - Elements videos = doc.select("script"); - for (Element el : videos) { - String json = el.html(); - if (json.startsWith("{")) { - JSONObject page = new JSONObject(json); - return page.getJSONObject("video").getString("contentUrl"); - } - } - throw new IOException(); - } -} diff --git a/src/main/java/com/rarchives/ripme/utils/RipUtils.java b/src/main/java/com/rarchives/ripme/utils/RipUtils.java index 14aa6dce..85d092e8 100644 --- a/src/main/java/com/rarchives/ripme/utils/RipUtils.java +++ b/src/main/java/com/rarchives/ripme/utils/RipUtils.java @@ -15,7 +15,6 @@ import com.rarchives.ripme.ripper.rippers.EromeRipper; import com.rarchives.ripme.ripper.rippers.ImgurRipper; import com.rarchives.ripme.ripper.rippers.RedgifsRipper; import com.rarchives.ripme.ripper.rippers.VidbleRipper; -import com.rarchives.ripme.ripper.rippers.GfycatRipper; import com.rarchives.ripme.ripper.rippers.SoundgasmRipper; import org.apache.commons.lang.math.NumberUtils; import org.apache.logging.log4j.LogManager; @@ -69,18 +68,6 @@ public class RipUtils { return result; } - else if (url.getHost().endsWith("gfycat.com")) { - try { - logger.debug("Fetching gfycat page " + url); - String videoURL = GfycatRipper.getVideoURL(url); - logger.debug("Got gfycat URL: " + videoURL); - result.add(new URI(videoURL).toURL()); - } catch (IOException | URISyntaxException e) { - // Do nothing - logger.warn("Exception while retrieving gfycat page:", e); - } - return result; - } else if (url.getHost().endsWith("redgifs.com") || url.getHost().endsWith("gifdeliverynetwork.com")) { try { logger.debug("Fetching redgifs page " + url); diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java deleted file mode 100644 index c8400d56..00000000 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/GfycatRipperTest.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.rarchives.ripme.tst.ripper.rippers; - -import com.rarchives.ripme.ripper.rippers.GfycatRipper; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; - - -public class GfycatRipperTest extends RippersTest { - - /** - * Rips correctly formatted URL directly from Gfycat - */ - @Test - public void testGfycatGoodURL() throws IOException, URISyntaxException { - GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/TemptingExcellentIchthyosaurs").toURL()); - testRipper(ripper); - } - /** - * Rips badly formatted URL directly from Gfycat - */ - public void testGfycatBadURL() throws IOException, URISyntaxException { - GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/gifs/detail/limitedtestyamericancrow").toURL()); - testRipper(ripper); - } - - /** - * Rips a Gfycat profile - */ - public void testGfycatProfile() throws IOException, URISyntaxException { - GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/@golbanstorage").toURL()); - testRipper(ripper); - } - - /** - * Rips a Gfycat amp link - * @throws IOException - */ - public void testGfycatAmp() throws IOException, URISyntaxException { - GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/amp/TemptingExcellentIchthyosaurs").toURL()); - testRipper(ripper); - } - - /** - * Rips a Gfycat profile with special characters in username - */ - public void testGfycatSpecialChar() throws IOException, URISyntaxException { - GfycatRipper ripper = new GfycatRipper(new URI("https://gfycat.com/@rsss.kr").toURL()); - testRipper(ripper); - } -}