From 4021a1b323e612cdb258f74bd751b3f61aae134a Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 4 Dec 2018 16:36:45 -0500 Subject: [PATCH 1/3] tumblr ripper now respects download.save_order --- .../ripme/ripper/rippers/TumblrRipper.java | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java index 53225927..0af3a2a5 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java @@ -25,6 +25,8 @@ import org.jsoup.nodes.Element; public class TumblrRipper extends AlbumRipper { + int index = 1; + private static final String DOMAIN = "tumblr.com", HOST = "tumblr", IMAGE_PATTERN = "([^\\s]+(\\.(?i)(jpg|png|gif|bmp))$)"; @@ -252,10 +254,10 @@ public class TumblrRipper extends AlbumRipper { m = p.matcher(fileURL.toString()); if (m.matches()) { - addURLToDownload(fileURL); + downloadURL(fileURL); } else { URL redirectedURL = Http.url(fileURL).ignoreContentType().response().url(); - addURLToDownload(redirectedURL); + downloadURL(redirectedURL); } } catch (Exception e) { LOGGER.error("[!] Error while parsing photo in " + photo, e); @@ -264,7 +266,7 @@ public class TumblrRipper extends AlbumRipper { } else if (post.has("video_url")) { try { fileURL = new URL(post.getString("video_url").replaceAll("http:", "https:")); - addURLToDownload(fileURL); + downloadURL(fileURL); } catch (Exception e) { LOGGER.error("[!] Error while parsing video in " + post, e); return true; @@ -273,7 +275,7 @@ public class TumblrRipper extends AlbumRipper { Document d = Jsoup.parse(post.getString("body")); if (!d.select("img").attr("src").isEmpty()) { try { - addURLToDownload(new URL(d.select("img").attr("src"))); + downloadURL(new URL(d.select("img").attr("src"))); } catch (MalformedURLException e) { LOGGER.error("[!] Error while getting embedded image at " + post, e); return true; @@ -355,4 +357,17 @@ public class TumblrRipper extends AlbumRipper { throw new MalformedURLException("Expected format: http://subdomain[.tumblr.com][/tagged/tag|/post/postno]"); } + private String getPrefix(int i) { + String prefix = ""; + if (Utils.getConfigBoolean("download.save_order", true)) { + prefix = String.format("%03d_", i); + } + return prefix; + } + + public void downloadURL(URL url) { + addURLToDownload(url, getPrefix(index)); + index++; + } + } From f81ddb8b3d06c5c8991df6a023f07c977976e6fa Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 4 Dec 2018 20:35:21 -0500 Subject: [PATCH 2/3] Tumblr ripper now can rip from liked pages --- .../ripme/ripper/rippers/TumblrRipper.java | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java index 0af3a2a5..2d3f992f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java @@ -34,7 +34,8 @@ public class TumblrRipper extends AlbumRipper { private enum ALBUM_TYPE { SUBDOMAIN, TAG, - POST + POST, + LIKED } private ALBUM_TYPE albumType; private String subdomain, tagName, postNumber; @@ -237,7 +238,12 @@ public class TumblrRipper extends AlbumRipper { URL fileURL; - posts = json.getJSONObject("response").getJSONArray("posts"); + if (albumType == ALBUM_TYPE.LIKED) { + posts = json.getJSONObject("response").getJSONArray("liked_posts"); + + } else { + posts = json.getJSONObject("response").getJSONArray("posts"); + } if (posts.length() == 0) { LOGGER.info(" Zero posts returned."); return false; @@ -291,6 +297,16 @@ public class TumblrRipper extends AlbumRipper { private String getTumblrApiURL(String mediaType, int offset) { StringBuilder sb = new StringBuilder(); + if (albumType == ALBUM_TYPE.LIKED) { + sb.append("http://api.tumblr.com/v2/blog/") + .append(subdomain) + .append("/likes") + .append("?api_key=") + .append(getApiKey()) + .append("&offset=") + .append(offset); + return sb.toString(); + } if (albumType == ALBUM_TYPE.POST) { sb.append("http://api.tumblr.com/v2/blog/") .append(subdomain) @@ -312,6 +328,7 @@ public class TumblrRipper extends AlbumRipper { sb.append("&tag=") .append(tagName); } + return sb.toString(); } @@ -354,6 +371,15 @@ public class TumblrRipper extends AlbumRipper { this.subdomain = m.group(1); return this.subdomain; } + // Likes url + p = Pattern.compile("https?://([a-z0-9_-]+).tumblr.com/likes"); + m = p.matcher(url.toExternalForm()); + if (m.matches()) { + this.albumType = ALBUM_TYPE.LIKED; + this.subdomain = m.group(1); + return this.subdomain + "_liked"; + } + throw new MalformedURLException("Expected format: http://subdomain[.tumblr.com][/tagged/tag|/post/postno]"); } From f01cd074962b9377eab2e199c3a848abfa47be3a Mon Sep 17 00:00:00 2001 From: cyian-1756 Date: Tue, 4 Dec 2018 20:48:13 -0500 Subject: [PATCH 3/3] Added support for the other likd url format --- .../rarchives/ripme/ripper/rippers/TumblrRipper.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java index 2d3f992f..ac99200f 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/TumblrRipper.java @@ -240,7 +240,6 @@ public class TumblrRipper extends AlbumRipper { if (albumType == ALBUM_TYPE.LIKED) { posts = json.getJSONObject("response").getJSONArray("liked_posts"); - } else { posts = json.getJSONObject("response").getJSONArray("posts"); } @@ -380,6 +379,15 @@ public class TumblrRipper extends AlbumRipper { return this.subdomain + "_liked"; } + // Likes url different format + p = Pattern.compile("https://www.tumblr.com/liked/by/([a-z0-9_-]+)"); + m = p.matcher(url.toExternalForm()); + if (m.matches()) { + this.albumType = ALBUM_TYPE.LIKED; + this.subdomain = m.group(1); + return this.subdomain + "_liked"; + } + throw new MalformedURLException("Expected format: http://subdomain[.tumblr.com][/tagged/tag|/post/postno]"); }