From 559de3b5fd266ae6af7be427a2698549f5bc6d32 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Wed, 6 Feb 2019 21:50:11 +0000 Subject: [PATCH 1/2] Closes #1143 Added support for score filtering when ripping from reddit Adds support for filtering out post/comments outside of a specific range from being downloaded. Examples with comments placed in the rip.properties file. Also added some (incomplete, but some is better than none) docs to the RedditRipper --- .../ripme/ripper/rippers/RedditRipper.java | 25 +++++++++++++++++++ src/main/resources/rip.properties | 14 +++++++++++ 2 files changed, 39 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index f0984d7d..b91a3c97 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -118,6 +118,12 @@ public class RedditRipper extends AlbumRipper { return nextURL; } + /** + * Gets a representation of the specified reddit page as a JSONArray using the reddit API + * @param url The url of the desired page + * @return A JSONArray object representation of the desired page + * @throws IOException If no response is received from the url + */ private JSONArray getJsonArrayFromURL(URL url) throws IOException { // Wait 2 seconds before the next request long timeDiff = System.currentTimeMillis() - lastRequestTime; @@ -149,9 +155,28 @@ public class RedditRipper extends AlbumRipper { return jsonArray; } + /** + * Turns child JSONObject's into usable URLs and hands them off for further processing + * Performs filtering checks based on the reddit. + * Only called from getAndParseAndReturnNext() while parsing the JSONArray returned from reddit's API + * @param child The child to process + */ private void parseJsonChild(JSONObject child) { String kind = child.getString("kind"); JSONObject data = child.getJSONObject("data"); + + //Upvote filtering + if (Utils.getConfigBoolean("reddit.rip_by_upvote", false)){ + int score = data.getInt("score"); + int maxScore = Utils.getConfigInteger("reddit.max_upvotes", Integer.MAX_VALUE); + int minScore = Utils.getConfigInteger("reddit.min_upvotes", Integer.MIN_VALUE); + + if (score > maxScore || score < minScore) { + + return; //Outside specified range, do not download + } + } + if (kind.equals("t1")) { // Comment handleBody(data.getString("body"), data.getString("id"), ""); diff --git a/src/main/resources/rip.properties b/src/main/resources/rip.properties index dd86dc1a..fd9e611a 100644 --- a/src/main/resources/rip.properties +++ b/src/main/resources/rip.properties @@ -30,3 +30,17 @@ twitter.max_requests = 10 clipboard.autorip = false download.save_order = true + +## Reddit ripper configs +# Determines whether or not to filter reddit ripping by upvote +# Enables the reddit.min_upvotes and reddit.max_upvotes properties when true +reddit.rip_by_upvote = false + +# Only rips file if the number of upvotes is equal to or greater than this value +# Requires reddit.rip_by_upvote = true +reddit.min_upvotes = 0 + +# Only rips files if the number of upvotes is less than this value +# Requires reddit.rip_by_upvote = true +reddit.max_upvotes = 10000 + From c683a0b2496fd98fe97a02eb4bd552a9ce6a8a74 Mon Sep 17 00:00:00 2001 From: Gamerick Date: Sun, 10 Feb 2019 14:31:28 +0000 Subject: [PATCH 2/2] Update src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java Add debug logging for score filtering --- .../java/com/rarchives/ripme/ripper/rippers/RedditRipper.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java index b91a3c97..082972e1 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/RedditRipper.java @@ -173,6 +173,8 @@ public class RedditRipper extends AlbumRipper { if (score > maxScore || score < minScore) { + String message = "Skipping post with score outside specified range of " + minScore + " to " + maxScore; + LOGGER.debug(message); return; //Outside specified range, do not download } }