From 39ba3349bb8501289b304dc0dd140f5a2cacb8d7 Mon Sep 17 00:00:00 2001 From: Kevin Jiang Date: Sat, 13 Oct 2018 20:48:26 -0400 Subject: [PATCH 1/2] Fixes TheChive bug so that it can now rip gifs. --- .../ripme/ripper/rippers/ThechiveRipper.java | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ThechiveRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ThechiveRipper.java index 8606ec42..7d1a38bc 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ThechiveRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ThechiveRipper.java @@ -1,4 +1,3 @@ - package com.rarchives.ripme.ripper.rippers; import com.rarchives.ripme.ripper.AbstractHTMLRipper; @@ -17,7 +16,7 @@ import org.jsoup.nodes.Element; public class ThechiveRipper extends AbstractHTMLRipper { public ThechiveRipper(URL url) throws IOException { - super(url); + super(url); } @Override @@ -38,8 +37,8 @@ public class ThechiveRipper extends AbstractHTMLRipper { boolean isTag = false; return m.group(1); } - throw new MalformedURLException("Expected thechive.com URL format: " + - "thechive.com/YEAR/MONTH/DAY/POSTTITLE/ - got " + url + " instead"); + throw new MalformedURLException("Expected thechive.com URL format: " + + "thechive.com/YEAR/MONTH/DAY/POSTTITLE/ - got " + url + " instead"); } @Override @@ -52,11 +51,19 @@ public class ThechiveRipper extends AbstractHTMLRipper { public List getURLsFromPage(Document doc) { List result = new ArrayList<>(); for (Element el : doc.select("img.attachment-gallery-item-full")) { - String imageSource = el.attr("src"); + String imageSource; + if (el.attr("data-gifsrc").isEmpty()) { //If it's not a gif + imageSource = el.attr("src"); + } else { //If it is a gif + imageSource = el.attr("data-gifsrc") //from data-gifsrc attribute + .replaceAll("\\?w=\\d{3}", ""); //remove the width modifier at the end to get highest resolution + //May need to replace the regex's {3} later on if website starts giving higher-res photos by default. + } + // We replace thumbs with resizes so we can the full sized images imageSource = imageSource.replace("thumbs", "resizes"); result.add(imageSource); - } + } return result; } @@ -65,5 +72,4 @@ public class ThechiveRipper extends AbstractHTMLRipper { addURLToDownload(url, getPrefix(index)); } - } From c664c9c58d94b76f03377b24beacfadc8307c1d6 Mon Sep 17 00:00:00 2001 From: Kevin Jiang Date: Sat, 13 Oct 2018 21:23:36 -0400 Subject: [PATCH 2/2] Add theChive testing --- .../ripper/rippers/ThechiveRipperTest.java | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 src/test/java/com/rarchives/ripme/tst/ripper/rippers/ThechiveRipperTest.java diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ThechiveRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ThechiveRipperTest.java new file mode 100644 index 00000000..274dc551 --- /dev/null +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/ThechiveRipperTest.java @@ -0,0 +1,80 @@ +/* + * The MIT License + * + * Copyright 2018 Kevin Jiang . + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package com.rarchives.ripme.tst.ripper.rippers; + +import com.rarchives.ripme.ripper.rippers.ThechiveRipper; +import java.io.IOException; +import java.net.URL; +import org.jsoup.nodes.Attributes; +import org.jsoup.nodes.Element; +import org.jsoup.parser.Tag; + +/** + * + * @author Kevin Jiang + */ +public class ThechiveRipperTest extends RippersTest { + + /** + * Tests general ripping for The Chive + * + * @throws IOException + */ + public void theChiveRip() throws IOException { + ThechiveRipper ripper = new ThechiveRipper(new URL("https://thechive.com/2018/10/03/the-definitive-list-of-the-hottest-horror-movie-babes/")); + testRipper(ripper); + } + + /* + + //If anyone figures out how to get JSOUP Elements mocked up, we can use the following methods to test both jpeg + gif ripping. + + public void testGifRip() throws IOException { + String elementInString = "" + + Element el = new Element( + new Tag("img"), + "",//URI + new Attributes()); + String URL = ThechiveRipper.getImageSource(el); + assertTrue(URL.equals("https://thechive.files.wordpress.com/2018/10/american_mary_crimson_quill-1.gif")); + } + + public void testGifRip() throws IOException { + String elementInString = ""; + Element el = new Element( + new Tag("img"), + "",//URI + new Attributes()); + String URL = ThechiveRipper.getImageSource(el); + assertTrue(URL.equals("https://thechive.files.wordpress.com/2018/10/the-definitive-list-of-the-hottest-horror-movie-babes-11.jpg")); + } + */ +}