From c489d8619238371b23ae59f2ee3bd6bfce077b3f Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 22 Sep 2018 23:43:01 -0400 Subject: [PATCH 1/4] Added support for downloading embedded images from tumblrs --- .../ripme/ripper/rippers/TumblrRipper.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java index 58d086b7..8f8cde76 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java @@ -19,6 +19,9 @@ import com.rarchives.ripme.ripper.AlbumRipper; import com.rarchives.ripme.ui.RipStatusMessage.STATUS; import com.rarchives.ripme.utils.Http; import com.rarchives.ripme.utils.Utils; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; public class TumblrRipper extends AlbumRipper { @@ -235,7 +238,6 @@ public class TumblrRipper extends AlbumRipper { for (int j = 0; j < photos.length(); j++) { photo = photos.getJSONObject(j); try { - // If the url is shorter than 65 chars long we skip it because it's those images don't support grabbing them in fullsize fileURL = new URL(photo.getJSONObject("original_size").getString("url").replaceAll("http:", "https:")); m = p.matcher(fileURL.toString()); @@ -257,6 +259,16 @@ public class TumblrRipper extends AlbumRipper { LOGGER.error("[!] Error while parsing video in " + post, e); return true; } + } else if (post.has("body")) { + Document d = Jsoup.parse(post.getString("body")); + if (!d.select("img").attr("src").isEmpty()) { + try { + addURLToDownload(new URL(d.select("img").attr("src"))); + } catch (MalformedURLException e) { + LOGGER.error("[!] Error while getting embedded image at " + post, e); + return true; + } + } } if (albumType == ALBUM_TYPE.POST) { return false; From 89ded2936b65e724b88d14566b372471e7c36e02 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sat, 22 Sep 2018 23:58:46 -0400 Subject: [PATCH 2/4] Added unit test for tumblr embedded image ripping --- .../ripme/tst/ripper/rippers/TumblrRipperTest.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TumblrRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TumblrRipperTest.java index eb616c4a..2bda2daf 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TumblrRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TumblrRipperTest.java @@ -7,19 +7,21 @@ import java.net.URL; import com.rarchives.ripme.ripper.rippers.TumblrRipper; public class TumblrRipperTest extends RippersTest { - // https://github.com/RipMeApp/ripme/issues/250 - /* + public void testTumblrFullRip() throws IOException { - TumblrRipper ripper = new TumblrRipper(new URL("http://wrouinr.tumblr.com/archive")); + TumblrRipper ripper = new TumblrRipper(new URL("http://wrouinr.tumblr.com")); testRipper(ripper); } public void testTumblrTagRip() throws IOException { - TumblrRipper ripper = new TumblrRipper(new URL("http://topinstagirls.tumblr.com/tagged/berlinskaya")); + TumblrRipper ripper = new TumblrRipper(new URL("https://these-are-my-b-sides.tumblr.com/tagged/boobs")); testRipper(ripper); } public void testTumblrPostRip() throws IOException { TumblrRipper ripper = new TumblrRipper(new URL("http://sadbaffoon.tumblr.com/post/132045920789/what-a-hoe")); testRipper(ripper); } - */ + public void testEmbeddedImage() throws IOException { + TumblrRipper ripper = new TumblrRipper(new URL("https://these-are-my-b-sides.tumblr.com/post/178225921524/this-was-fun")); + testRipper(ripper); + } } From 54502f3310c55bc3cae42a0c8c31a13d6c5b2188 Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Sun, 23 Sep 2018 00:35:32 -0400 Subject: [PATCH 3/4] Tumblr ripper unit tests now have their own api key --- .../java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java index 8f8cde76..93e7b318 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java @@ -49,6 +49,10 @@ public class TumblrRipper extends AlbumRipper { * @return Tumblr API key */ public static String getApiKey() { + // Use a different api ket for unit tests so we don't get 429 errors + if (isThisATest()) { + return "UHpRFx16HFIRgQjtjJKgfVIcwIeb71BYwOQXTMtiCvdSEPjV7N"; + } if (API_KEY == null) { API_KEY = pickRandomApiKey(); } From 7a92014dece644f00434de09cd9fc49a9b510afc Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Mon, 24 Sep 2018 20:01:18 -0400 Subject: [PATCH 4/4] Commented out slaky unit tests --- .../tst/ripper/rippers/TumblrRipperTest.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TumblrRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TumblrRipperTest.java index 2bda2daf..2de62e51 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TumblrRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/TumblrRipperTest.java @@ -8,18 +8,18 @@ import com.rarchives.ripme.ripper.rippers.TumblrRipper; public class TumblrRipperTest extends RippersTest { - public void testTumblrFullRip() throws IOException { - TumblrRipper ripper = new TumblrRipper(new URL("http://wrouinr.tumblr.com")); - testRipper(ripper); - } - public void testTumblrTagRip() throws IOException { - TumblrRipper ripper = new TumblrRipper(new URL("https://these-are-my-b-sides.tumblr.com/tagged/boobs")); - testRipper(ripper); - } - public void testTumblrPostRip() throws IOException { - TumblrRipper ripper = new TumblrRipper(new URL("http://sadbaffoon.tumblr.com/post/132045920789/what-a-hoe")); - testRipper(ripper); - } +// public void testTumblrFullRip() throws IOException { +// TumblrRipper ripper = new TumblrRipper(new URL("http://wrouinr.tumblr.com")); +// testRipper(ripper); +// } +// public void testTumblrTagRip() throws IOException { +// TumblrRipper ripper = new TumblrRipper(new URL("https://these-are-my-b-sides.tumblr.com/tagged/boobs")); +// testRipper(ripper); +// } +// public void testTumblrPostRip() throws IOException { +// TumblrRipper ripper = new TumblrRipper(new URL("http://sadbaffoon.tumblr.com/post/132045920789/what-a-hoe")); +// testRipper(ripper); +// } public void testEmbeddedImage() throws IOException { TumblrRipper ripper = new TumblrRipper(new URL("https://these-are-my-b-sides.tumblr.com/post/178225921524/this-was-fun")); testRipper(ripper);