From eae9ecf37b8725bd2362c659d4581e0277d059b0 Mon Sep 17 00:00:00 2001 From: Sheefip <61481253+Sheefip@users.noreply.github.com> Date: Sun, 2 Aug 2020 14:22:20 +0200 Subject: [PATCH 1/2] RedditRipper: added gallery support A gallery is a special kind of reddit post, which can have up to 20 images. They can be recognized by the is_gallery field in the JSON. The gallery_data array describes the structure of the gallery by referencing the image ids, it also contains the captions and links, which are ignored by this implementation. The s field of each entry in media_metadata array appears to reference the highest quality image. getJsonURL and getGID were modified to handle reddit.com/gallery/id links. parseJsonChild was modified to handle the gallery case. The new method handleGallery parses gallery_data and media_metadata and adds the images to download. A test for a gallery was added. --- .../ripme/ripper/rippers/RedditRipper.java | 44 ++++++++++++++++++- .../tst/ripper/rippers/RedditRipperTest.java | 7 +++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index e68e477d..3d2fe22c 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -10,6 +10,7 @@ import java.util.regex.Pattern; import com.rarchives.ripme.ui.RipStatusMessage; import org.json.JSONArray; +import org.json.JSONException; import org.json.JSONObject; import org.json.JSONTokener; @@ -54,6 +55,13 @@ public class RedditRipper extends AlbumRipper { } private URL getJsonURL(URL url) throws MalformedURLException { + // Convert gallery to post link and append ".json" + Pattern p = Pattern.compile("^https?://[a-zA-Z0-9.]{0,4}reddit\\.com/gallery/([a-zA-Z0-9]+).*$"); + Matcher m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return new URL("https://reddit.com/" +m.group(m.groupCount())+ ".json"); + } + // Append ".json" to URL in appropriate location. String result = url.getProtocol() + "://" + url.getHost() + url.getPath() + ".json"; if (url.getQuery() != null) { @@ -188,6 +196,8 @@ public class RedditRipper extends AlbumRipper { if (data.getBoolean("is_self")) { // TODO Parse self text handleBody(data.getString("selftext"), data.getString("id"), data.getString("title")); + } else if (data.has("is_gallery") && data.getBoolean("is_gallery")) { + handleGallery(data.getJSONObject("gallery_data").getJSONArray("items"), data.getJSONObject("media_metadata"), data.getString("id"), data.getString("title")); } else { // Get link handleURL(data.getString("url"), data.getString("id"), data.getString("title")); @@ -291,6 +301,31 @@ public class RedditRipper extends AlbumRipper { } } + private void handleGallery(JSONArray data, JSONObject metadata, String id, String title){ + //TODO handle captions and caption urls + String subdirectory = ""; + if (Utils.getConfigBoolean("reddit.use_sub_dirs", true)) { + if (Utils.getConfigBoolean("album_titles.save", true)) { + subdirectory = title; + title = "-" + title + "-"; + } + } + for (int i = 0; i < data.length(); i++) { + JSONObject media = metadata.getJSONObject(data.getJSONObject(i).getString("media_id")); + String prefix = id + "-"; + if (Utils.getConfigBoolean("download.save_order", true)) { + //announcement says up to 20 (https://www.reddit.com/r/announcements/comments/hrrh23/now_you_can_make_posts_with_multiple_images/) + prefix += String.format("%02d-", i + 1); + } + try { + URL mediaURL = new URL(media.getJSONObject("s").getString("u").replaceAll("&", "&")); + addURLToDownload(mediaURL, prefix, subdirectory); + } catch (MalformedURLException | JSONException e) { + LOGGER.error("[!] Unable to parse gallery JSON:\ngallery_data:\n" + data +"\nmedia_metadata:\n" + metadata); + } + } + } + @Override public String getHost() { return HOST; @@ -312,6 +347,13 @@ public class RedditRipper extends AlbumRipper { return "post_" + m.group(m.groupCount()); } + // Gallery + p = Pattern.compile("^https?://[a-zA-Z0-9.]{0,4}reddit\\.com/gallery/([a-zA-Z0-9]+).*$"); + m = p.matcher(url.toExternalForm()); + if (m.matches()) { + return "post_" + m.group(m.groupCount()); + } + // Subreddit p = Pattern.compile("^https?://[a-zA-Z0-9.]{0,4}reddit\\.com/r/([a-zA-Z0-9_]+).*$"); m = p.matcher(url.toExternalForm()); @@ -319,7 +361,7 @@ public class RedditRipper extends AlbumRipper { return "sub_" + m.group(m.groupCount()); } - throw new MalformedURLException("Only accepts user pages, subreddits, or post, can't understand " + url); + throw new MalformedURLException("Only accepts user pages, subreddits, post, or gallery can't understand " + url); } } diff --git a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java index d5d9600d..972e2f9b 100644 --- a/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java +++ b/src/test/java/com/rarchives/ripme/tst/ripper/rippers/RedditRipperTest.java @@ -56,4 +56,11 @@ public class RedditRipperTest extends RippersTest { new URL("https://www.reddit.com/r/bottesting/comments/7msmhi/bad_link/")); testRipper(ripper); } + + @Test + public void testRedditGallery() throws IOException{ + RedditRipper ripper = new RedditRipper( + new URL("https://www.reddit.com/gallery/hrrh23")); + testRipper(ripper); + } } From 8118287cddec976c27f84a1c26470958a842e12d Mon Sep 17 00:00:00 2001 From: Sheefip <61481253+Sheefip@users.noreply.github.com> Date: Fri, 14 Aug 2020 10:19:36 +0200 Subject: [PATCH 2/2] RedditRipper: changed gallery condition --- .../java/com/rarchives/ripme/ripper/rippers/RedditRipper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index 3d2fe22c..65d854fb 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -196,7 +196,7 @@ public class RedditRipper extends AlbumRipper { if (data.getBoolean("is_self")) { // TODO Parse self text handleBody(data.getString("selftext"), data.getString("id"), data.getString("title")); - } else if (data.has("is_gallery") && data.getBoolean("is_gallery")) { + } else if (!data.isNull("gallery_data") && !data.isNull("media_metadata")) { handleGallery(data.getJSONObject("gallery_data").getJSONArray("items"), data.getJSONObject("media_metadata"), data.getString("id"), data.getString("title")); } else { // Get link