From 72a697eda873829a4b69c348c0045ced66f6b41b Mon Sep 17 00:00:00 2001 From: PaaaulZ <46759927+PaaaulZ@users.noreply.github.com> Date: Sat, 2 Jan 2021 20:36:03 +0100 Subject: [PATCH] Refactor, removed duplicated code. Fixed bugs --- .../ripme/ripper/rippers/ScrolllerRipper.java | 146 ++++++------------ 1 file changed, 48 insertions(+), 98 deletions(-) diff --git a/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java b/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java index a333eac6..7e0c1c46 100644 --- a/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java +++ b/src/main/java/com/rarchives/ripme/ripper/rippers/ScrolllerRipper.java @@ -8,12 +8,9 @@ import java.net.*; import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; -import java.util.Locale; import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.oracle.truffle.js.nodes.access.IteratorStepNode; -import com.oracle.truffle.js.runtime.builtins.JSON; import org.java_websocket.client.WebSocketClient; import org.apache.http.NameValuePair; @@ -61,55 +58,31 @@ public class ScrolllerRipper extends AbstractJSONRipper { private JSONObject prepareQuery(String iterator, String gid, String sortByString) throws IOException, URISyntaxException { - // Prepares the JSONObject we need to pass to the GraphQL query. + String QUERY_NOSORT = "query SubredditQuery( $url: String! $filter: SubredditPostFilter $iterator: String ) { getSubreddit(url: $url) { children( limit: 50 iterator: $iterator filter: $filter ) { iterator items { __typename url title subredditTitle subredditUrl redditPath isNsfw albumUrl isFavorite mediaSources { url width height isOptimized } } } } }"; + String QUERY_SORT = "subscription SubredditSubscription( $url: String! $sortBy: SubredditSortBy $timespan: SubredditTimespan $iterator: String $limit: Int $filter: SubredditPostFilter ) { fetchSubreddit( url: $url sortBy: $sortBy timespan: $timespan iterator: $iterator limit: $limit filter: $filter ) { __typename ... on Subreddit { __typename url title secondaryTitle description createdAt isNsfw subscribers isComplete itemCount videoCount pictureCount albumCount isFollowing } ... on SubredditPost { __typename url title subredditTitle subredditUrl redditPath isNsfw albumUrl isFavorite mediaSources { url width height isOptimized } } ... on Iterator { iterator } ... on Error { message } } }"; - if (sortByString.equals("")) { - // Sorting not selected - String queryString = "query SubredditQuery( $url: String! $filter: SubredditPostFilter $iterator: String ) { getSubreddit(url: $url) { children( limit: 50 iterator: $iterator filter: $filter ) { iterator items { __typename url title subredditTitle subredditUrl redditPath isNsfw albumUrl isFavorite mediaSources { url width height isOptimized } } } } }"; - String filterString = convertFilterString(getParameter(this.url,"filter")); + String filterString = convertFilterString(getParameter(this.url,"filter")); - JSONObject variablesObject = new JSONObject().put("url", String.format("/r/%s", gid)); + JSONObject variablesObject = new JSONObject().put("url", String.format("/r/%s", gid)).put("sortBy", sortByString.toUpperCase()); + JSONObject finalQueryObject = new JSONObject().put("variables", variablesObject).put("query", sortByString.equals("") ? QUERY_NOSORT : QUERY_SORT); - if (iterator != null) { - // Iterator is not present on the first page - variablesObject.put("iterator", iterator); - } - if (!filterString.equals("NOFILTER")) { - // We could also pass filter="" but not including it if not present is cleaner - variablesObject.put("filter", filterString); - } - - JSONObject finalQueryObject = new JSONObject().put("variables", variablesObject).put("query", queryString); - - return getPosts(finalQueryObject); + if (iterator != null) { + // Iterator is not present on the first page + variablesObject.put("iterator", iterator); } - else { - - String queryString = "subscription SubredditSubscription( $url: String! $sortBy: SubredditSortBy $timespan: SubredditTimespan $iterator: String $limit: Int $filter: SubredditPostFilter ) { fetchSubreddit( url: $url sortBy: $sortBy timespan: $timespan iterator: $iterator limit: $limit filter: $filter ) { __typename ... on Subreddit { __typename url title secondaryTitle description createdAt isNsfw subscribers isComplete itemCount videoCount pictureCount albumCount isFollowing } ... on SubredditPost { __typename url title subredditTitle subredditUrl redditPath isNsfw albumUrl isFavorite mediaSources { url width height isOptimized } } ... on Iterator { iterator } ... on Error { message } } }"; - String filterString = convertFilterString(getParameter(this.url,"filter")); - - JSONObject variablesObject = new JSONObject().put("url", String.format("/r/%s", gid)).put("sortBy", sortByString.toUpperCase()); - - if (iterator != null) { - // Iterator is not present on the first page - variablesObject.put("iterator", iterator); - } - if (!filterString.equals("NOFILTER")) { - // We could also pass filter="" but not including it if not present is cleaner - variablesObject.put("filter", filterString); - } - - JSONObject finalQueryObject = new JSONObject().put("variables", variablesObject).put("query", queryString); - - return getPostsSorted(finalQueryObject); + if (!filterString.equals("NOFILTER")) { + variablesObject.put("filter", filterString); } + + return sortByString.equals("") ? getPosts(finalQueryObject) : getPostsSorted(finalQueryObject); + } public String convertFilterString(String filterParameter) { // Converts the ?filter= parameter of the URL to one that can be used in the GraphQL query // I could basically remove the last "s" and call toUpperCase instead of this switch statement but this looks easier to read. - switch (filterParameter) { + switch (filterParameter.toLowerCase()) { case "pictures": return "PICTURE"; case "videos": @@ -134,7 +107,7 @@ public class ScrolllerRipper extends AbstractJSONRipper { // 1) arg.getName() => https://scrolller.com/r/CatsStandingUp?filter // 2) arg.getName() => sort - if (arg.getName().replace(toReplace,"").equals((parameter))) { + if (arg.getName().replace(toReplace,"").toLowerCase().equals((parameter))) { return arg.getValue(); } } @@ -198,8 +171,7 @@ public class ScrolllerRipper extends AbstractJSONRipper { @Override public void onMessage(String s) { postsJsonStrings.add(s); - if (s.contains("{\"data\":{\"fetchSubreddit\":{\"__typename\":\"Iterator\",\"iterator\":")) { - // Iterator is the last field returned, once we received it we can close the connection. + if (new JSONObject(s).getJSONObject("data").getJSONObject("fetchSubreddit").has("iterator")) { this.close(); } } @@ -230,6 +202,11 @@ public class ScrolllerRipper extends AbstractJSONRipper { } finalObject.put("posts", posts); + if (finalObject.getJSONArray("posts").length() == 1 && !finalObject.getJSONArray("posts").getJSONObject(0).getJSONObject("data").getJSONObject("fetchSubreddit").has("mediaSources")) { + // Only iterator, no posts. + return null; + } + return finalObject; @@ -244,66 +221,38 @@ public class ScrolllerRipper extends AbstractJSONRipper { @Override protected List getURLsFromJSON(JSONObject json) throws JSONException { - if (json.has("posts")) { - // If the JSONObject contains the key "posts" it's our custom JSON made after the WebSocket call. + boolean sortRequested = json.has("posts"); - JSONArray itemsList = json.getJSONArray("posts"); - int bestArea = 0; - String bestUrl = ""; - List list = new ArrayList<>(); + int bestArea = 0; + String bestUrl = ""; + List list = new ArrayList<>(); + JSONArray itemsList = sortRequested ? json.getJSONArray("posts") : json.getJSONObject("data").getJSONObject("getSubreddit").getJSONObject("children").getJSONArray("items"); - for (Object item : itemsList) { - if (((JSONObject) item).getJSONObject("data").getJSONObject("fetchSubreddit").has("mediaSources")) { - // Is it really a post? It could be the subreddit description or the iterator (first and last item) - JSONArray sourcesTMP = ((JSONObject) item).getJSONObject("data").getJSONObject("fetchSubreddit").getJSONArray("mediaSources"); - for (Object sourceTMP : sourcesTMP) - { - int widthTMP = ((JSONObject) sourceTMP).getInt("width"); - int heightTMP = ((JSONObject) sourceTMP).getInt("height"); - int areaTMP = widthTMP * heightTMP; + for (Object item : itemsList) { - if (areaTMP > bestArea) { - // Better way to determine best image? - bestArea = widthTMP; - bestUrl = ((JSONObject) sourceTMP).getString("url"); - } - } - list.add(bestUrl); - bestUrl = ""; - bestArea = 0; + if (sortRequested && !((JSONObject) item).getJSONObject("data").getJSONObject("fetchSubreddit").has("mediaSources")) { + continue; + } + JSONArray sourcesTMP = sortRequested ? ((JSONObject) item).getJSONObject("data").getJSONObject("fetchSubreddit").getJSONArray("mediaSources") : ((JSONObject) item).getJSONArray("mediaSources"); + for (Object sourceTMP : sourcesTMP) + { + int widthTMP = ((JSONObject) sourceTMP).getInt("width"); + int heightTMP = ((JSONObject) sourceTMP).getInt("height"); + int areaTMP = widthTMP * heightTMP; + + if (areaTMP > bestArea) { + bestArea = widthTMP; + bestUrl = ((JSONObject) sourceTMP).getString("url"); } } - return list; - - } else { - JSONArray itemsList = json.getJSONObject("data").getJSONObject("getSubreddit").getJSONObject("children").getJSONArray("items"); - int bestArea = 0; - String bestUrl = ""; - List list = new ArrayList<>(); - - - for (Object item : itemsList) { - JSONArray sourcesTMP = ((JSONObject) item).getJSONArray("mediaSources"); - for (Object sourceTMP : sourcesTMP) - { - int widthTMP = ((JSONObject) sourceTMP).getInt("width"); - int heightTMP = ((JSONObject) sourceTMP).getInt("height"); - int areaTMP = widthTMP * heightTMP; - - if (areaTMP > bestArea) { - // Better way to determine best image? - bestArea = widthTMP; - bestUrl = ((JSONObject) sourceTMP).getString("url"); - } - } - list.add(bestUrl); - bestUrl = ""; - bestArea = 0; - } - return list; + list.add(bestUrl); + bestUrl = ""; + bestArea = 0; } + + return list; } @Override @@ -323,12 +272,14 @@ public class ScrolllerRipper extends AbstractJSONRipper { Object iterator = null; if (source.has("iterator")) { - // sorted + // Sort requested, custom JSON. iterator = source.getJSONObject("iterator").getJSONObject("data").getJSONObject("fetchSubreddit").get("iterator"); } else { iterator = source.getJSONObject("data").getJSONObject("getSubreddit").getJSONObject("children").get("iterator"); } + if (!iterator.toString().equals("null")) { + // Need to change page. try { return prepareQuery(iterator.toString(), this.getGID(url), getParameter(url,"sort")); } catch (URISyntaxException e) { @@ -339,5 +290,4 @@ public class ScrolllerRipper extends AbstractJSONRipper { return null; } } - } \ No newline at end of file